




"""simple docstring"""

import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_lowerCamelCase	:      Tuple   =							logging.get_logger(__name__)

_lowerCamelCase	:      Optional[int]   =							{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_lowerCamelCase	:      int   =							{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

_lowerCamelCase	:      Any   =							{
    'Salesforce/codegen-350M-mono': 2048,
}

@lru_cache()
def 		lowercase_   (		):




	"""simple docstring"""






	A_	:					Dict						 =		(
	    list(range(ord('''!'''						)	,   ord('''~'''						) + 1						)						) + list(range(ord('''¡'''						)	,   ord('''¬'''						) + 1						)						) + list(range(ord('''®'''						)	,   ord('''ÿ'''						) + 1						)						)
	)
	A_	:					Any						 =		bs[:]
	A_	:					Union[str, Any]						 =		0
	for b in range(2**8						):
		if b not in bs:
			bs.append(_UpperCAmelCase						)
			cs.append(2**8 + n						)
			n += 1
	A_	:					str						 =		[chr(_UpperCAmelCase						) for n in cs]
	return dict(zip(_UpperCAmelCase	,   _UpperCAmelCase						)						)

def 		lowercase_   (		_UpperCAmelCase						):




	"""simple docstring"""






	A_	:					Union[str, Any]						 =		set()
	A_	:					List[str]						 =		word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)						)
		A_	:					Dict						 =		char
	return pairs

class 							lowercase   (     __UpperCAmelCase):
				__lowerCAmelCase     :     List[Any]								= VOCAB_FILES_NAMES
				__lowerCAmelCase     :     List[str]								= PRETRAINED_VOCAB_FILES_MAP
				__lowerCAmelCase     :     Tuple								= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__lowerCAmelCase     :     str								= ["""input_ids""", """attention_mask"""]







				def __init__(				self :       Dict   ,  _lowerCamelCase :       List[Any]   ,  _lowerCamelCase :       Union[str, Any]   ,  _lowerCamelCase :       List[str]="replace"   ,  _lowerCamelCase :       Dict="<|endoftext|>"   ,  _lowerCamelCase :       Optional[Any]="<|endoftext|>"   ,  _lowerCamelCase :       Union[str, Any]="<|endoftext|>"   ,  _lowerCamelCase :       Union[str, Any]=None   ,  _lowerCamelCase :       Optional[int]=False   ,  _lowerCamelCase :       str=False   ,  **_lowerCamelCase :       Any   ,  ):



					"""simple docstring"""

					A_	:					str						 =		AddedToken(_lowerCamelCase   ,  lstrip=_lowerCamelCase   ,  rstrip=_lowerCamelCase							) if isinstance(_lowerCamelCase   ,  _lowerCamelCase							) else bos_token
					A_	:					Union[str, Any]						 =		AddedToken(_lowerCamelCase   ,  lstrip=_lowerCamelCase   ,  rstrip=_lowerCamelCase							) if isinstance(_lowerCamelCase   ,  _lowerCamelCase							) else eos_token
					A_	:					int						 =		AddedToken(_lowerCamelCase   ,  lstrip=_lowerCamelCase   ,  rstrip=_lowerCamelCase							) if isinstance(_lowerCamelCase   ,  _lowerCamelCase							) else unk_token
					A_	:					str						 =		AddedToken(_lowerCamelCase   ,  lstrip=_lowerCamelCase   ,  rstrip=_lowerCamelCase							) if isinstance(_lowerCamelCase   ,  _lowerCamelCase							) else pad_token
					super().__init__(
					    errors=_lowerCamelCase   ,  unk_token=_lowerCamelCase   ,  bos_token=_lowerCamelCase   ,  eos_token=_lowerCamelCase   ,  pad_token=_lowerCamelCase   ,  add_prefix_space=_lowerCamelCase   ,  add_bos_token=_lowerCamelCase   ,  **_lowerCamelCase   ,  )
					A_	:					List[str]						 =		add_bos_token

					with open(_lowerCamelCase   ,  encoding='''utf-8'''							) as vocab_handle:
						A_	:					List[Any]						 =		json.load(_lowerCamelCase							)
					A_	:					Dict						 =		{v: k for k, v in self.encoder.items()}
					A_	:					str						 =		errors  # how to handle errors in decoding
					A_	:					str						 =		bytes_to_unicode()
					A_	:					List[str]						 =		{v: k for k, v in self.byte_encoder.items()}
					with open(_lowerCamelCase   ,  encoding='''utf-8'''							) as merges_handle:
						A_	:					Optional[int]						 =		merges_handle.read().split('''\n'''							)[1:-1]
					A_	:					List[Any]						 =		[tuple(merge.split()							) for merge in bpe_merges]
					A_	:					List[Any]						 =		dict(zip(_lowerCamelCase   ,  range(len(_lowerCamelCase							)							)							)							)
					A_	:					List[str]						 =		{}
					A_	:					Dict						 =		add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					A_	:					Optional[Any]						 =		re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''							)







				@property
				def        a_	(				self :       Any							):



					"""simple docstring"""

					return len(self.encoder							)







				def        a_	(				self :       Optional[int]							):



					"""simple docstring"""

					return dict(self.encoder   ,  **self.added_tokens_encoder							)







				def        a_	(				self :       List[Any]   ,  _lowerCamelCase :       Optional[Any]							):



					"""simple docstring"""

					if token in self.cache:
						return self.cache[token]
					A_	:					int						 =		tuple(_lowerCamelCase							)
					A_	:					List[Any]						 =		get_pairs(_lowerCamelCase							)

					if not pairs:
						return token

					while True:
						A_	:					Any						 =		min(_lowerCamelCase   ,  key=lambda _lowerCamelCase							: self.bpe_ranks.get(_lowerCamelCase   ,  float('''inf'''							)							)							)
						if bigram not in self.bpe_ranks:
							break
						A_ ,			A_	:					str						 =		bigram
						A_	:					Union[str, Any]						 =		[]
						A_	:					Any						 =		0
						while i < len(_lowerCamelCase							):
							try:
								A_	:					str						 =		word.index(_lowerCamelCase   ,  _lowerCamelCase							)
							except ValueError:
								new_word.extend(word[i:]							)
								break
							else:
								new_word.extend(word[i:j]							)
								A_	:					Optional[int]						 =		j

							if word[i] == first and i < len(_lowerCamelCase							) - 1 and word[i + 1] == second:
								new_word.append(first + second							)
								i += 2
							else:
								new_word.append(word[i]							)
								i += 1
						A_	:					List[str]						 =		tuple(_lowerCamelCase							)
						A_	:					Any						 =		new_word
						if len(_lowerCamelCase							) == 1:
							break
						else:
							A_	:					int						 =		get_pairs(_lowerCamelCase							)
					A_	:					int						 =		''' '''.join(_lowerCamelCase							)
					A_	:					List[Any]						 =		word
					return word







				def        a_	(				self :       Union[str, Any]   ,  _lowerCamelCase :       Dict   ,  _lowerCamelCase :       Union[str, Any]=None							):



					"""simple docstring"""

					if self.add_bos_token:
						A_	:					str						 =		[self.bos_token_id]
					else:
						A_	:					Optional[Any]						 =		[]

					A_	:					List[Any]						 =		bos_token_ids + token_ids_a

					if token_ids_a is None:
						return output

					return output + bos_token_ids + token_ids_a







				def        a_	(				self :       Union[str, Any]   ,  _lowerCamelCase :       Optional[int]							):



					"""simple docstring"""

					A_	:					Optional[int]						 =		[]
					for token in re.findall(self.pat   ,  _lowerCamelCase							):
						A_	:					Union[str, Any]						 =		''''''.join(
						    self.byte_encoder[b] for b in token.encode('''utf-8'''							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
						bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase							).split(''' '''							)							)
					return bpe_tokens







				def        a_	(				self :       Tuple   ,  _lowerCamelCase :       Dict							):



					"""simple docstring"""

					return self.encoder.get(_lowerCamelCase   ,  self.encoder.get(self.unk_token							)							)







				def        a_	(				self :       Tuple   ,  _lowerCamelCase :       Optional[Any]							):



					"""simple docstring"""

					return self.decoder.get(_lowerCamelCase							)







				def        a_	(				self :       List[Any]   ,  _lowerCamelCase :       Dict							):



					"""simple docstring"""

					A_	:					List[str]						 =		''''''.join(_lowerCamelCase							)
					A_	:					List[str]						 =		bytearray([self.byte_decoder[c] for c in text]							).decode('''utf-8'''   ,  errors=self.errors							)
					return text







				def        a_	(				self :       Optional[int]   ,  _lowerCamelCase :       str   ,  _lowerCamelCase :       Optional[str] = None							):



					"""simple docstring"""

					if not os.path.isdir(_lowerCamelCase							):
						logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""							)
						return
					A_	:					Any						 =		os.path.join(
					    _lowerCamelCase   ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
					A_	:					Tuple						 =		os.path.join(
					    _lowerCamelCase   ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']							)

					with open(_lowerCamelCase   ,  '''w'''   ,  encoding='''utf-8'''							) as f:
						f.write(json.dumps(self.encoder   ,  indent=2   ,  sort_keys=_lowerCamelCase   ,  ensure_ascii=_lowerCamelCase							) + '''\n'''							)

					A_	:					Union[str, Any]						 =		0
					with open(_lowerCamelCase   ,  '''w'''   ,  encoding='''utf-8'''							) as writer:
						writer.write('''#version: 0.2\n'''							)
						for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,  key=lambda _lowerCamelCase							: kv[1]							):
							if index != token_index:
								logger.warning(
								    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
								    ''' Please check that the tokenizer is not corrupted!'''							)
								A_	:					Dict						 =		token_index
							writer.write(''' '''.join(_lowerCamelCase							) + '''\n'''							)
							index += 1

					return vocab_file, merge_file







				def        a_	(				self :       List[Any]   ,  _lowerCamelCase :       List[str]   ,  _lowerCamelCase :       Tuple=False   ,  **_lowerCamelCase :       Tuple							):



					"""simple docstring"""

					A_	:					str						 =		kwargs.pop('''add_prefix_space'''   ,  self.add_prefix_space							)
					if is_split_into_words or add_prefix_space:
						A_	:					Optional[int]						 =		''' ''' + text
					return (text, kwargs)







				def        a_	(				self :       Any   ,  _lowerCamelCase :       Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]   ,  _lowerCamelCase :       bool = False   ,  _lowerCamelCase :       bool = None   ,  _lowerCamelCase :       Optional[List[str]] = None   ,  **_lowerCamelCase :       str   ,  ):



					"""simple docstring"""

					A_	:					List[str]						 =		super()._decode(
					    token_ids=_lowerCamelCase   ,  skip_special_tokens=_lowerCamelCase   ,  clean_up_tokenization_spaces=_lowerCamelCase   ,  **_lowerCamelCase   ,  )

					if truncate_before_pattern is not None and len(_lowerCamelCase							) > 0:
						A_	:					Optional[Any]						 =		self.truncate(_lowerCamelCase   ,  _lowerCamelCase							)

					return decoded_text







				def        a_	(				self :       int   ,  _lowerCamelCase :       Tuple   ,  _lowerCamelCase :       List[Any]							):



					"""simple docstring"""

					def find_re(_lowerCamelCase :       List[Any]   ,  _lowerCamelCase :       List[str]   ,  _lowerCamelCase :       Dict							):
						A_	:					Tuple						 =		pattern.search(_lowerCamelCase   ,  _lowerCamelCase							)
						return m.start() if m else -1

					A_	:					Union[str, Any]						 =		[re.compile(_lowerCamelCase   ,  re.MULTILINE							) for pattern in truncate_before_pattern]

					A_	:					List[str]						 =		list(re.finditer('''^print'''   ,  _lowerCamelCase   ,  re.MULTILINE							)							)

					if len(_lowerCamelCase							) > 1:
						A_	:					Dict						 =		completion[: prints[1].start()]

					A_	:					Any						 =		list(re.finditer('''^def'''   ,  _lowerCamelCase   ,  re.MULTILINE							)							)

					if len(_lowerCamelCase							) > 1:
						A_	:					Optional[int]						 =		completion[: defs[1].start()]

					A_	:					int						 =		0

					A_	:					Dict						 =		[
					    pos for pos in [find_re(_lowerCamelCase   ,  _lowerCamelCase   ,  _lowerCamelCase							) for terminal in terminals] if pos != -1
					]

					if len(_lowerCamelCase							) > 0:
						return completion[: min(_lowerCamelCase							)]
					else:
						return completion











"""simple docstring"""

def 		lowercase_   (		_UpperCAmelCase	,   _UpperCAmelCase						):




	"""simple docstring"""






	return a + b


if __name__ == "__main__":
		_lowerCamelCase	:      Tuple   =							5
		_lowerCamelCase	:      Union[str, Any]   =							6
		print(f'The sum of {a} + {b} is {add(a, b)}')





