



'''simple docstring'''







import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation

_UpperCamelCase  						=			logging.get_logger(__name__)

_UpperCamelCase  						=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

_UpperCamelCase  						=			{
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

_UpperCamelCase  						=			{
    """microsoft/deberta-base""": 512,
    """microsoft/deberta-large""": 512,
    """microsoft/deberta-xlarge""": 512,
    """microsoft/deberta-base-mnli""": 512,
    """microsoft/deberta-large-mnli""": 512,
    """microsoft/deberta-xlarge-mnli""": 512,
}

_UpperCamelCase  						=			{
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}
def 			_lowerCAmelCase(      ) ->							Union[str, Any]:
							lowerCAmelCase__  =   (
							    list(range(ord("""!"""						)					, ord("""~"""						) + 1						)						) + list(range(ord("""¡"""						)					, ord("""¬"""						) + 1						)						) + list(range(ord("""®"""						)					, ord("""ÿ"""						) + 1						)						)
							)
							lowerCAmelCase__  =   bs[:]
							lowerCAmelCase__  =   0
							for b in range(2**8						):
														if b not in bs:
																					bs.append(UpperCAmelCase_						)
																					cs.append(2**8 + n						)
																					n += 1
							lowerCAmelCase__  =   [chr(UpperCAmelCase_						) for n in cs]
							return dict(zip(UpperCAmelCase_					, UpperCAmelCase_						)						)
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Optional[int]						) ->							Dict:
							lowerCAmelCase__  =   set()
							lowerCAmelCase__  =   word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)						)
														lowerCAmelCase__  =   char
							return pairs






class      lowerCamelCase__     (     _A				):






	'''simple docstring'''

	A__		=	VOCAB_FILES_NAMES
	A__		=	PRETRAINED_VOCAB_FILES_MAP
	A__		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A__		=	['''input_ids''', '''attention_mask''', '''token_type_ids''']




	def __init__(  self		:  Optional[int]   ,		__A		:  str   ,		__A		:  str   ,		__A		:  Union[str, Any]="replace"   ,		__A		:  Optional[Any]="[CLS]"   ,		__A		:  Union[str, Any]="[SEP]"   ,		__A		:  Dict="[SEP]"   ,		__A		:  Optional[int]="[CLS]"   ,		__A		:  List[str]="[UNK]"   ,		__A		:  Tuple="[PAD]"   ,		__A		:  List[str]="[MASK]"   ,		__A		:  List[str]=False   ,		__A		:  Tuple=False   ,		**__A		:  List[str]   ,		)     ->			List[str]:




								'''simple docstring'''





								lowerCAmelCase__  =   AddedToken(__A   ,		lstrip=__A   ,		rstrip=__A					) if isinstance(__A   ,		__A					) else bos_token
								lowerCAmelCase__  =   AddedToken(__A   ,		lstrip=__A   ,		rstrip=__A					) if isinstance(__A   ,		__A					) else eos_token
								lowerCAmelCase__  =   AddedToken(__A   ,		lstrip=__A   ,		rstrip=__A					) if isinstance(__A   ,		__A					) else sep_token
								lowerCAmelCase__  =   AddedToken(__A   ,		lstrip=__A   ,		rstrip=__A					) if isinstance(__A   ,		__A					) else cls_token
								lowerCAmelCase__  =   AddedToken(__A   ,		lstrip=__A   ,		rstrip=__A					) if isinstance(__A   ,		__A					) else unk_token
								lowerCAmelCase__  =   AddedToken(__A   ,		lstrip=__A   ,		rstrip=__A					) if isinstance(__A   ,		__A					) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								lowerCAmelCase__  =   AddedToken(__A   ,		lstrip=__A   ,		rstrip=__A					) if isinstance(__A   ,		__A					) else mask_token

								super().__init__(
								    errors=__A   ,		bos_token=__A   ,		eos_token=__A   ,		unk_token=__A   ,		sep_token=__A   ,		cls_token=__A   ,		pad_token=__A   ,		mask_token=__A   ,		add_prefix_space=__A   ,		add_bos_token=__A   ,		**__A   ,		)
								lowerCAmelCase__  =   add_bos_token

								with open(__A   ,		encoding="""utf-8"""					) as vocab_handle:
															lowerCAmelCase__  =   json.load(__A					)
								lowerCAmelCase__  =   {v: k for k, v in self.encoder.items()}
								lowerCAmelCase__  =   errors  # how to handle errors in decoding
								lowerCAmelCase__  =   bytes_to_unicode()
								lowerCAmelCase__  =   {v: k for k, v in self.byte_encoder.items()}
								with open(__A   ,		encoding="""utf-8"""					) as merges_handle:
															lowerCAmelCase__  =   merges_handle.read().split("""\n"""					)[1:-1]
								lowerCAmelCase__  =   [tuple(merge.split()					) for merge in bpe_merges]
								lowerCAmelCase__  =   dict(zip(__A   ,		range(len(__A					)					)					)					)
								lowerCAmelCase__  =   {}
								lowerCAmelCase__  =   add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								lowerCAmelCase__  =   re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""					)




	@property
	# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
	def      lowercase__							(  self		:  str					)     ->			Any:




								'''simple docstring'''





								return len(self.encoder					)




	def      lowercase__							(  self		:  List[Any]					)     ->			int:




								'''simple docstring'''





								return dict(self.encoder   ,		**self.added_tokens_encoder					)




	def      lowercase__							(  self		:  int   ,		__A		:  Union[str, Any]					)     ->			List[str]:




								'''simple docstring'''





								if token in self.cache:
															return self.cache[token]
								lowerCAmelCase__  =   tuple(__A					)
								lowerCAmelCase__  =   get_pairs(__A					)

								if not pairs:
															return token

								while True:
															lowerCAmelCase__  =   min(__A   ,		key=lambda __A					: self.bpe_ranks.get(__A   ,		float("""inf"""					)					)					)
															if bigram not in self.bpe_ranks:
																						break
															lowerCAmelCase__  ,lowerCAmelCase__  =   bigram
															lowerCAmelCase__  =   []
															lowerCAmelCase__  =   0
															while i < len(__A					):
																						try:
																													lowerCAmelCase__  =   word.index(__A   ,		__A					)
																						except ValueError:
																													new_word.extend(word[i:]					)
																													break
																						else:
																													new_word.extend(word[i:j]					)
																													lowerCAmelCase__  =   j

																						if word[i] == first and i < len(__A					) - 1 and word[i + 1] == second:
																													new_word.append(first + second					)
																													i += 2
																						else:
																													new_word.append(word[i]					)
																													i += 1
															lowerCAmelCase__  =   tuple(__A					)
															lowerCAmelCase__  =   new_word
															if len(__A					) == 1:
																						break
															else:
																						lowerCAmelCase__  =   get_pairs(__A					)
								lowerCAmelCase__  =   """ """.join(__A					)
								lowerCAmelCase__  =   word
								return word




	def      lowercase__							(  self		:  str   ,		__A		:  List[int]   ,		__A		:  Optional[List[int]] = None					)     ->			List[int]:




								'''simple docstring'''





								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								lowerCAmelCase__  =   [self.cls_token_id]
								lowerCAmelCase__  =   [self.sep_token_id]
								return cls + token_ids_a + sep + token_ids_a + sep




	def      lowercase__							(  self		:  Dict   ,		__A		:  List[int]   ,		__A		:  Optional[List[int]] = None   ,		__A		:  bool = False					)     ->			List[int]:




								'''simple docstring'''





								if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=__A   ,		token_ids_a=__A   ,		already_has_special_tokens=__A					)

								if token_ids_a is None:
															return [1] + ([0] * len(__A					)) + [1]
								return [1] + ([0] * len(__A					)) + [1] + ([0] * len(__A					)) + [1]




	def      lowercase__							(  self		:  Optional[int]   ,		__A		:  List[int]   ,		__A		:  Optional[List[int]] = None					)     ->			List[int]:




								'''simple docstring'''





								lowerCAmelCase__  =   [self.sep_token_id]
								lowerCAmelCase__  =   [self.cls_token_id]

								if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]




	def      lowercase__							(  self		:  List[Any]   ,		__A		:  int					)     ->			str:




								'''simple docstring'''





								lowerCAmelCase__  =   []
								for token in re.findall(self.pat   ,		__A					):
															lowerCAmelCase__  =   """""".join(
															    self.byte_encoder[b] for b in token.encode("""utf-8"""					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A					).split(""" """					)					)
								return bpe_tokens




	def      lowercase__							(  self		:  Union[str, Any]   ,		__A		:  List[Any]					)     ->			Union[str, Any]:




								'''simple docstring'''





								return self.encoder.get(__A   ,		self.encoder.get(self.unk_token					)					)




	def      lowercase__							(  self		:  str   ,		__A		:  List[Any]					)     ->			int:




								'''simple docstring'''





								return self.decoder.get(__A					)




	def      lowercase__							(  self		:  Tuple   ,		__A		:  Union[str, Any]					)     ->			List[Any]:




								'''simple docstring'''





								lowerCAmelCase__  =   """""".join(__A					)
								lowerCAmelCase__  =   bytearray([self.byte_decoder[c] for c in text]					).decode("""utf-8"""   ,		errors=self.errors					)
								return text




	def      lowercase__							(  self		:  Optional[Any]   ,		__A		:  str   ,		__A		:  Optional[str] = None					)     ->			Tuple[str]:




								'''simple docstring'''





								if not os.path.isdir(__A					):
															logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
															return
								lowerCAmelCase__  =   os.path.join(
								    __A   ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
								lowerCAmelCase__  =   os.path.join(
								    __A   ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]					)

								with open(__A   ,		"""w"""   ,		encoding="""utf-8"""					) as f:
															f.write(json.dumps(self.encoder   ,		indent=2   ,		sort_keys=__A   ,		ensure_ascii=__A					) + """\n"""					)

								lowerCAmelCase__  =   0
								with open(__A   ,		"""w"""   ,		encoding="""utf-8"""					) as writer:
															writer.write("""#version: 0.2\n"""					)
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,		key=lambda __A					: kv[1]					):
																						if index != token_index:
																													logger.warning(
																													    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																													    """ Please check that the tokenizer is not corrupted!"""					)
																													lowerCAmelCase__  =   token_index
																						writer.write(""" """.join(__A					) + """\n"""					)
																						index += 1

								return vocab_file, merge_file




	def      lowercase__							(  self		:  List[str]   ,		__A		:  Union[str, Any]   ,		__A		:  int=False   ,		**__A		:  Union[str, Any]					)     ->			Optional[int]:




								'''simple docstring'''





								lowerCAmelCase__  =   kwargs.pop("""add_prefix_space"""   ,		self.add_prefix_space					)
								if (is_split_into_words or add_prefix_space) and (len(__A					) > 0 and not text[0].isspace()):
															lowerCAmelCase__  =   """ """ + text
								return (text, kwargs)




	def      lowercase__							(  self		:  Optional[int]   ,		__A		:  "Conversation"					)     ->			List[int]:




								'''simple docstring'''





								lowerCAmelCase__  =   []
								for is_user, text in conversation.iter_texts():
															input_ids.extend(self.encode(__A   ,		add_special_tokens=__A					) + [self.eos_token_id]					)
								if len(__A					) > self.model_max_length:
															lowerCAmelCase__  =   input_ids[-self.model_max_length :]
								return input_ids










'''simple docstring'''







def 			_lowerCAmelCase(      UpperCAmelCase_      :	list					, UpperCAmelCase_      :	int					, UpperCAmelCase_      :	int					, UpperCAmelCase_      :	int						) ->							int:
							lowerCAmelCase__  =   int((lower_bound + upper_bound) // 2						)
							if array[r] == value:
														return r
							if lower_bound >= upper_bound:
														return -1
							if array[r] < value:
														return binary_search(UpperCAmelCase_					, r + 1					, UpperCAmelCase_					, UpperCAmelCase_						)
							else:
														return binary_search(UpperCAmelCase_					, UpperCAmelCase_					, r - 1					, UpperCAmelCase_						)
def 			_lowerCAmelCase(      UpperCAmelCase_      :	int					, UpperCAmelCase_      :	list						) ->							list:
							lowerCAmelCase__  =   0
							if matrix[index][0] == value:
														return [index, 0]
							while index < len(UpperCAmelCase_						) and matrix[index][0] < value:
														lowerCAmelCase__  =   binary_search(matrix[index]					, 0					, len(matrix[index]						) - 1					, UpperCAmelCase_						)
														if r != -1:
																					return [index, r]
														index += 1
							return [-1, -1]


if __name__ == "__main__":
		import doctest

		doctest.testmod()





