




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
					if is_torch_available():
										import torch
					if is_tf_available():
										import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


UpperCamelCase__            =     logging.get_logger(__name__)

UpperCamelCase__            =     {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

UpperCamelCase__            =     {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

UpperCamelCase__            =     {
    """Salesforce/codegen-350M-mono""": 2048,
}







@lru_cache()
def       _a (		):
					__lowerCAmelCase							  =			(
					    list(range(ord("!"	) , ord("~"	) + 1	)	) + list(range(ord("¡"	) , ord("¬"	) + 1	)	) + list(range(ord("®"	) , ord("ÿ"	) + 1	)	)
					)
					__lowerCAmelCase							  =			bs[:]
					__lowerCAmelCase							  =			0
					for b in range(2**8	):
										if b not in bs:
															bs.append(lowerCamelCase__	)
															cs.append(2**8 + n	)
															n += 1
					__lowerCAmelCase							  =			[chr(lowerCamelCase__	) for n in cs]
					return dict(zip(lowerCamelCase__ , lowerCamelCase__	)	)







def       _a (		SCREAMING_SNAKE_CASE_				:							List[str]	):
					__lowerCAmelCase							  =			set()
					__lowerCAmelCase							  =			word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)	)
										__lowerCAmelCase							  =			char
					return pairs


class      a__       (   snake_case__ ):
						_a				:       Optional[Any]   =       VOCAB_FILES_NAMES
						_a				:       Union[str, Any]   =       PRETRAINED_VOCAB_FILES_MAP
						_a				:       Any   =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a				:       Dict   =       ['input_ids', 'attention_mask']






						def __init__(  self						,     _A						,     _A						,     _A="replace"						,     _A="<|endoftext|>"						,     _A="<|endoftext|>"						,     _A="<|endoftext|>"						,     _A=None						,     _A=False						,     _A=False						,     **_A						,     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			AddedToken(_snake_case						,     lstrip=_snake_case						,     rstrip=_snake_case     ) if isinstance(_snake_case						,     _snake_case     ) else bos_token
											__lowerCAmelCase							  =			AddedToken(_snake_case						,     lstrip=_snake_case						,     rstrip=_snake_case     ) if isinstance(_snake_case						,     _snake_case     ) else eos_token
											__lowerCAmelCase							  =			AddedToken(_snake_case						,     lstrip=_snake_case						,     rstrip=_snake_case     ) if isinstance(_snake_case						,     _snake_case     ) else unk_token
											__lowerCAmelCase							  =			AddedToken(_snake_case						,     lstrip=_snake_case						,     rstrip=_snake_case     ) if isinstance(_snake_case						,     _snake_case     ) else pad_token
											super().__init__(
											    errors=_snake_case						,     unk_token=_snake_case						,     bos_token=_snake_case						,     eos_token=_snake_case						,     pad_token=_snake_case						,     add_prefix_space=_snake_case						,     add_bos_token=_snake_case						,     **_snake_case						,     )
											__lowerCAmelCase							  =			add_bos_token

											with open(_snake_case						,     encoding="utf-8"     ) as vocab_handle:
																__lowerCAmelCase							  =			json.load(_snake_case     )
											__lowerCAmelCase							  =			{v: k for k, v in self.encoder.items()}
											__lowerCAmelCase							  =			errors  # how to handle errors in decoding
											__lowerCAmelCase							  =			bytes_to_unicode()
											__lowerCAmelCase							  =			{v: k for k, v in self.byte_encoder.items()}
											with open(_snake_case						,     encoding="utf-8"     ) as merges_handle:
																__lowerCAmelCase							  =			merges_handle.read().split("\n"     )[1:-1]
											__lowerCAmelCase							  =			[tuple(merge.split()     ) for merge in bpe_merges]
											__lowerCAmelCase							  =			dict(zip(_snake_case						,     range(len(_snake_case     )     )     )     )
											__lowerCAmelCase							  =			{}
											__lowerCAmelCase							  =			add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											__lowerCAmelCase							  =			re.compile(R"\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"     )






						@property
						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											return len(self.encoder     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											return dict(self.encoder						,     **self.added_tokens_encoder     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A     ):




											"""simple docstring"""



											if token in self.cache:
																return self.cache[token]
											__lowerCAmelCase							  =			tuple(_snake_case     )
											__lowerCAmelCase							  =			get_pairs(_snake_case     )

											if not pairs:
																return token

											while True:
																__lowerCAmelCase							  =			min(_snake_case						,     key=lambda _A     : self.bpe_ranks.get(_snake_case						,     float("inf"     )     )     )
																if bigram not in self.bpe_ranks:
																					break
																__lowerCAmelCase							  =			bigram
																__lowerCAmelCase							  =			[]
																__lowerCAmelCase							  =			0
																while i < len(_snake_case     ):
																					try:
																										__lowerCAmelCase							  =			word.index(_snake_case						,     _snake_case     )
																					except ValueError:
																										new_word.extend(word[i:]     )
																										break
																					else:
																										new_word.extend(word[i:j]     )
																										__lowerCAmelCase							  =			j

																					if word[i] == first and i < len(_snake_case     ) - 1 and word[i + 1] == second:
																										new_word.append(first + second     )
																										i += 2
																					else:
																										new_word.append(word[i]     )
																										i += 1
																__lowerCAmelCase							  =			tuple(_snake_case     )
																__lowerCAmelCase							  =			new_word
																if len(_snake_case     ) == 1:
																					break
																else:
																					__lowerCAmelCase							  =			get_pairs(_snake_case     )
											__lowerCAmelCase							  =			' '.join(_snake_case     )
											__lowerCAmelCase							  =			word
											return word






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A=None     ):




											"""simple docstring"""



											if self.add_bos_token:
																__lowerCAmelCase							  =			[self.bos_token_id]
											else:
																__lowerCAmelCase							  =			[]

											__lowerCAmelCase							  =			bos_token_ids + token_ids_a

											if token_ids_a is None:
																return output

											return output + bos_token_ids + token_ids_a






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			[]
											for token in re.findall(self.pat						,     _snake_case     ):
																__lowerCAmelCase							  =			''.join(
																    self.byte_encoder[b] for b in token.encode("utf-8"     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_snake_case     ).split(" "     )     )
											return bpe_tokens






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A     ):




											"""simple docstring"""



											return self.encoder.get(_snake_case						,     self.encoder.get(self.unk_token     )     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A     ):




											"""simple docstring"""



											return self.decoder.get(_snake_case     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			''.join(_snake_case     )
											__lowerCAmelCase							  =			bytearray([self.byte_decoder[c] for c in text]     ).decode("utf-8"						,     errors=self.errors     )
											return text






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A = None     ):




											"""simple docstring"""



											if not os.path.isdir(_snake_case     ):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
																return
											__lowerCAmelCase							  =			os.path.join(
											    _snake_case						,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )
											__lowerCAmelCase							  =			os.path.join(
											    _snake_case						,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]     )

											with open(_snake_case						,     "w"						,     encoding="utf-8"     ) as f:
																f.write(json.dumps(self.encoder						,     indent=2						,     sort_keys=_snake_case						,     ensure_ascii=_snake_case     ) + "\n"     )

											__lowerCAmelCase							  =			0
											with open(_snake_case						,     "w"						,     encoding="utf-8"     ) as writer:
																writer.write("#version: 0.2\n"     )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,     key=lambda _A     : kv[1]     ):
																					if index != token_index:
																										logger.warning(
																										    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																										    " Please check that the tokenizer is not corrupted!"     )
																										__lowerCAmelCase							  =			token_index
																					writer.write(" ".join(_snake_case     ) + "\n"     )
																					index += 1

											return vocab_file, merge_file






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A=False						,     **_A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			kwargs.pop("add_prefix_space"						,     self.add_prefix_space     )
											if is_split_into_words or add_prefix_space:
																__lowerCAmelCase							  =			' ' + text
											return (text, kwargs)






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A = False						,     _A = None						,     _A = None						,     **_A						,     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			super()._decode(
											    token_ids=_snake_case						,     skip_special_tokens=_snake_case						,     clean_up_tokenization_spaces=_snake_case						,     **_snake_case						,     )

											if truncate_before_pattern is not None and len(_snake_case     ) > 0:
																__lowerCAmelCase							  =			self.truncate(_snake_case						,     _snake_case     )

											return decoded_text






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A     ):




											"""simple docstring"""



											def find_re(_A						,     _A						,     _A     ):
																__lowerCAmelCase							  =			pattern.search(_snake_case						,     _snake_case     )
																return m.start() if m else -1

											__lowerCAmelCase							  =			[re.compile(_snake_case						,     re.MULTILINE     ) for pattern in truncate_before_pattern]

											__lowerCAmelCase							  =			list(re.finditer("^print"						,     _snake_case						,     re.MULTILINE     )     )

											if len(_snake_case     ) > 1:
																__lowerCAmelCase							  =			completion[: prints[1].start()]

											__lowerCAmelCase							  =			list(re.finditer("^def"						,     _snake_case						,     re.MULTILINE     )     )

											if len(_snake_case     ) > 1:
																__lowerCAmelCase							  =			completion[: defs[1].start()]

											__lowerCAmelCase							  =			0

											__lowerCAmelCase							  =			[
											    pos for pos in [find_re(_snake_case						,     _snake_case						,     _snake_case     ) for terminal in terminals] if pos != -1
											]

											if len(_snake_case     ) > 0:
																return completion[: min(_snake_case     )]
											else:
																return completion












def       _a   (							lowerCamelCase__				)  ->      int:
 return int(lowerCamelCase__				) if x - int(lowerCamelCase__				) >= 0 else int(lowerCamelCase__				) - 1


if __name__ == "__main__":
       import doctest

       doctest.testmod()
