



'''simple docstring'''






import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation

A_				:					List[str]							=      logging.get_logger(__name__)

A_				:					str							=      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

A_				:					List[str]							=      {
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

A_				:					Optional[int]							=      {
    """microsoft/deberta-base""": 5_1_2,
    """microsoft/deberta-large""": 5_1_2,
    """microsoft/deberta-xlarge""": 5_1_2,
    """microsoft/deberta-base-mnli""": 5_1_2,
    """microsoft/deberta-large-mnli""": 5_1_2,
    """microsoft/deberta-xlarge-mnli""": 5_1_2,
}

A_				:					Tuple							=      {
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}




def        snake_case_   (	)->						Optional[int]:





       '''simple docstring'''


       _UpperCAmelCase    :		List[str]     	=					(
           list(range(ord("""!"""						)			,  ord("""~"""						) + 1						)						) + list(range(ord("""¡"""						)			,  ord("""¬"""						) + 1						)						) + list(range(ord("""®"""						)			,  ord("""ÿ"""						) + 1						)						)
       )
       _UpperCAmelCase    :		str     	=					bs[:]
       _UpperCAmelCase    :		Optional[int]     	=					0
       for b in range(2**8						):
              if b not in bs:
                     bs.append(lowerCAmelCase_						)
                     cs.append(2**8 + n						)
                     n += 1
       _UpperCAmelCase    :		Optional[int]     	=					[chr(lowerCAmelCase_						) for n in cs]
       return dict(zip(lowerCAmelCase_			,  lowerCAmelCase_						)						)




def        snake_case_   (	lowerCAmelCase_						)->						Optional[Any]:





       '''simple docstring'''


       _UpperCAmelCase    :		str     	=					set()
       _UpperCAmelCase    :		Any     	=					word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)						)
              _UpperCAmelCase    :		List[Any]     	=					char
       return pairs



class 				lowercase			(    _lowerCamelCase							):


 """simple docstring"""


 UpperCAmelCase		      =    VOCAB_FILES_NAMES
 UpperCAmelCase		      =    PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase		      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase		      =    ["""input_ids""", """attention_mask""", """token_type_ids"""]







 def __init__(				self    ,a_    ,a_    ,a_="replace"    ,a_="[CLS]"    ,a_="[SEP]"    ,a_="[SEP]"    ,a_="[CLS]"    ,a_="[UNK]"    ,a_="[PAD]"    ,a_="[MASK]"    ,a_=False    ,a_=False    ,**a_    ,)						->  List[Any]:
        _UpperCAmelCase    :		Dict     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else bos_token
        _UpperCAmelCase    :		Optional[int]     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else eos_token
        _UpperCAmelCase    :		List[str]     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else sep_token
        _UpperCAmelCase    :		Tuple     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else cls_token
        _UpperCAmelCase    :		List[str]     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else unk_token
        _UpperCAmelCase    :		Dict     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else pad_token

        # Mask token behave like a normal word, i.e. include the space before it
        _UpperCAmelCase    :		List[str]     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else mask_token

        super().__init__(
            errors=a_    ,bos_token=a_    ,eos_token=a_    ,unk_token=a_    ,sep_token=a_    ,cls_token=a_    ,pad_token=a_    ,mask_token=a_    ,add_prefix_space=a_    ,add_bos_token=a_    ,**a_    ,)
        _UpperCAmelCase    :		Any     	=					add_bos_token

        with open(a_    ,encoding="""utf-8"""				) as vocab_handle:
               _UpperCAmelCase    :		Optional[int]     	=					json.load(a_				)
        _UpperCAmelCase    :		Any     	=					{v: k for k, v in self.encoder.items()}
        _UpperCAmelCase    :		str     	=					errors  # how to handle errors in decoding
        _UpperCAmelCase    :		Any     	=					bytes_to_unicode()
        _UpperCAmelCase    :		List[Any]     	=					{v: k for k, v in self.byte_encoder.items()}
        with open(a_    ,encoding="""utf-8"""				) as merges_handle:
               _UpperCAmelCase    :		int     	=					merges_handle.read().split("""\n"""				)[1:-1]
        _UpperCAmelCase    :		str     	=					[tuple(merge.split()				) for merge in bpe_merges]
        _UpperCAmelCase    :		str     	=					dict(zip(a_    ,range(len(a_				)				)				)				)
        _UpperCAmelCase    :		str     	=					{}
        _UpperCAmelCase    :		List[str]     	=					add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        _UpperCAmelCase    :		List[Any]     	=					re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""				)







 @property
 # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
 def _snake_case					(				self				)						->  Tuple:
        return len(self.encoder				)







 def _snake_case					(				self				)						->  Optional[Any]:
        return dict(self.encoder    ,**self.added_tokens_encoder				)







 def _snake_case					(				self    ,a_				)						->  Tuple:
        if token in self.cache:
               return self.cache[token]
        _UpperCAmelCase    :		Any     	=					tuple(a_				)
        _UpperCAmelCase    :		Tuple     	=					get_pairs(a_				)

        if not pairs:
               return token

        while True:
               _UpperCAmelCase    :		List[Any]     	=					min(a_    ,key=lambda a_				: self.bpe_ranks.get(a_    ,float("""inf"""				)				)				)
               if bigram not in self.bpe_ranks:
                      break
               _UpperCAmelCase	,_UpperCAmelCase    :		List[str]     	=					bigram
               _UpperCAmelCase    :		str     	=					[]
               _UpperCAmelCase    :		int     	=					0
               while i < len(a_				):
                      try:
                             _UpperCAmelCase    :		Tuple     	=					word.index(a_    ,a_				)
                      except ValueError:
                             new_word.extend(word[i:]				)
                             break
                      else:
                             new_word.extend(word[i:j]				)
                             _UpperCAmelCase    :		Tuple     	=					j

                      if word[i] == first and i < len(a_				) - 1 and word[i + 1] == second:
                             new_word.append(first + second				)
                             i += 2
                      else:
                             new_word.append(word[i]				)
                             i += 1
               _UpperCAmelCase    :		Optional[int]     	=					tuple(a_				)
               _UpperCAmelCase    :		Any     	=					new_word
               if len(a_				) == 1:
                      break
               else:
                      _UpperCAmelCase    :		Any     	=					get_pairs(a_				)
        _UpperCAmelCase    :		List[str]     	=					""" """.join(a_				)
        _UpperCAmelCase    :		Dict     	=					word
        return word







 def _snake_case					(				self    ,a_    ,a_ = None				)						->  List[int]:
        if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        _UpperCAmelCase    :		Optional[int]     	=					[self.cls_token_id]
        _UpperCAmelCase    :		str     	=					[self.sep_token_id]
        return cls + token_ids_a + sep + token_ids_a + sep







 def _snake_case					(				self    ,a_    ,a_ = None    ,a_ = False				)						->  List[int]:
        if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=a_    ,token_ids_a=a_    ,already_has_special_tokens=a_				)

        if token_ids_a is None:
               return [1] + ([0] * len(a_				)) + [1]
        return [1] + ([0] * len(a_				)) + [1] + ([0] * len(a_				)) + [1]







 def _snake_case					(				self    ,a_    ,a_ = None				)						->  List[int]:
        _UpperCAmelCase    :		List[Any]     	=					[self.sep_token_id]
        _UpperCAmelCase    :		Tuple     	=					[self.cls_token_id]

        if token_ids_a is None:
               return len(cls + token_ids_a + sep				) * [0]
        return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







 def _snake_case					(				self    ,a_				)						->  List[str]:
        _UpperCAmelCase    :		Optional[Any]     	=					[]
        for token in re.findall(self.pat    ,a_				):
               _UpperCAmelCase    :		List[Any]     	=					"""""".join(
                   self.byte_encoder[b] for b in token.encode("""utf-8"""				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a_				).split(""" """				)				)
        return bpe_tokens







 def _snake_case					(				self    ,a_				)						->  Union[str, Any]:
        return self.encoder.get(a_    ,self.encoder.get(self.unk_token				)				)







 def _snake_case					(				self    ,a_				)						->  Dict:
        return self.decoder.get(a_				)







 def _snake_case					(				self    ,a_				)						->  Optional[int]:
        _UpperCAmelCase    :		Optional[Any]     	=					"""""".join(a_				)
        _UpperCAmelCase    :		Union[str, Any]     	=					bytearray([self.byte_decoder[c] for c in text]				).decode("""utf-8"""    ,errors=self.errors				)
        return text







 def _snake_case					(				self    ,a_    ,a_ = None				)						->  Tuple[str]:
        if not os.path.isdir(a_				):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''				)
               return
        _UpperCAmelCase    :		Union[str, Any]     	=					os.path.join(
            a_    ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
        _UpperCAmelCase    :		Tuple     	=					os.path.join(
            a_    ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]				)

        with open(a_    ,"""w"""    ,encoding="""utf-8"""				) as f:
               f.write(json.dumps(self.encoder    ,indent=2    ,sort_keys=a_    ,ensure_ascii=a_				) + """\n"""				)

        _UpperCAmelCase    :		List[str]     	=					0
        with open(a_    ,"""w"""    ,encoding="""utf-8"""				) as writer:
               writer.write("""#version: 0.2\n"""				)
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,key=lambda a_				: kv[1]				):
                      if index != token_index:
                             logger.warning(
                                 f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                 """ Please check that the tokenizer is not corrupted!"""				)
                             _UpperCAmelCase    :		List[Any]     	=					token_index
                      writer.write(""" """.join(a_				) + """\n"""				)
                      index += 1

        return vocab_file, merge_file







 def _snake_case					(				self    ,a_    ,a_=False    ,**a_				)						->  List[str]:
        _UpperCAmelCase    :		int     	=					kwargs.pop("""add_prefix_space"""    ,self.add_prefix_space				)
        if (is_split_into_words or add_prefix_space) and (len(a_				) > 0 and not text[0].isspace()):
               _UpperCAmelCase    :		Any     	=					""" """ + text
        return (text, kwargs)







 def _snake_case					(				self    ,a_				)						->  List[int]:
        _UpperCAmelCase    :		str     	=					[]
        for is_user, text in conversation.iter_texts():
               input_ids.extend(self.encode(a_    ,add_special_tokens=a_				) + [self.eos_token_id]				)
        if len(a_				) > self.model_max_length:
               _UpperCAmelCase    :		List[Any]     	=					input_ids[-self.model_max_length :]
        return input_ids









"""simple docstring"""





_lowercase						   =     [0.4_85, 0.4_56, 0.4_06]
_lowercase						   =     [0.2_29, 0.2_24, 0.2_25]
_lowercase						   =     [0.5, 0.5, 0.5]
_lowercase						   =     [0.5, 0.5, 0.5]
_lowercase						   =     [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73]
_lowercase						   =     [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11]


