






"""simple docstring"""





import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation

UpperCamelCase_						:					int      				=			logging.get_logger(__name__)

UpperCamelCase_						:					Any      				=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

UpperCamelCase_						:					int      				=			{
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

UpperCamelCase_						:					List[str]      				=			{
    """microsoft/deberta-base""": 512,
    """microsoft/deberta-large""": 512,
    """microsoft/deberta-xlarge""": 512,
    """microsoft/deberta-base-mnli""": 512,
    """microsoft/deberta-large-mnli""": 512,
    """microsoft/deberta-xlarge-mnli""": 512,
}

UpperCamelCase_						:					Tuple      				=			{
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}



def      __lowercase       (     )       ->       List[str]:
      __snake_case   :     Any	      =(
          list(range(ord('''!'''			)							,		ord('''~'''			) + 1			)			) + list(range(ord('''¡'''			)							,		ord('''¬'''			) + 1			)			) + list(range(ord('''®'''			)							,		ord('''ÿ'''			) + 1			)			)
      )
      __snake_case   :     int	      =bs[:]
      __snake_case   :     Tuple	      =0
      for b in range(2**8			):
            if b not in bs:
                  bs.append(a			)
                  cs.append(2**8 + n			)
                  n += 1
      __snake_case   :     Optional[Any]	      =[chr(a			) for n in cs]
      return dict(zip(a							,		a			)			)



def      __lowercase       (     a			:					List[Any]			)       ->       List[str]:
      __snake_case   :     str	      =set()
      __snake_case   :     Optional[Any]	      =word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)			)
            __snake_case   :     str	      =char
      return pairs


class   _lowercase			(  lowerCAmelCase    ):
      _a :  Union[str, Any]										= VOCAB_FILES_NAMES
      _a :  str										= PRETRAINED_VOCAB_FILES_MAP
      _a :  Optional[int]										= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _a :  Optional[int]										= ['''input_ids''', '''attention_mask''', '''token_type_ids''']


      def __init__(     self					:						Optional[int]				, a					:						str				, a					:						Optional[Any]				, a					:						Optional[Any]="replace"				, a					:						Any="[CLS]"				, a					:						Union[str, Any]="[SEP]"				, a					:						Union[str, Any]="[SEP]"				, a					:						int="[CLS]"				, a					:						Dict="[UNK]"				, a					:						List[Any]="[PAD]"				, a					:						str="[MASK]"				, a					:						Union[str, Any]=False				, a					:						Tuple=False				, **a					:						str				, ):





            """simple docstring"""







            __snake_case   :     Tuple	      =AddedToken(a				, lstrip=a				, rstrip=a    ) if isinstance(a				, a    ) else bos_token
            __snake_case   :     int	      =AddedToken(a				, lstrip=a				, rstrip=a    ) if isinstance(a				, a    ) else eos_token
            __snake_case   :     Tuple	      =AddedToken(a				, lstrip=a				, rstrip=a    ) if isinstance(a				, a    ) else sep_token
            __snake_case   :     Tuple	      =AddedToken(a				, lstrip=a				, rstrip=a    ) if isinstance(a				, a    ) else cls_token
            __snake_case   :     Any	      =AddedToken(a				, lstrip=a				, rstrip=a    ) if isinstance(a				, a    ) else unk_token
            __snake_case   :     Union[str, Any]	      =AddedToken(a				, lstrip=a				, rstrip=a    ) if isinstance(a				, a    ) else pad_token

            # Mask token behave like a normal word, i.e. include the space before it
            __snake_case   :     Union[str, Any]	      =AddedToken(a				, lstrip=a				, rstrip=a    ) if isinstance(a				, a    ) else mask_token

            super().__init__(
                errors=a				, bos_token=a				, eos_token=a				, unk_token=a				, sep_token=a				, cls_token=a				, pad_token=a				, mask_token=a				, add_prefix_space=a				, add_bos_token=a				, **a				, )
            __snake_case   :     Optional[Any]	      =add_bos_token

            with open(a				, encoding='''utf-8'''    ) as vocab_handle:
                  __snake_case   :     Tuple	      =json.load(a    )
            __snake_case   :     int	      ={v: k for k, v in self.encoder.items()}
            __snake_case   :     Optional[Any]	      =errors  # how to handle errors in decoding
            __snake_case   :     List[Any]	      =bytes_to_unicode()
            __snake_case   :     List[str]	      ={v: k for k, v in self.byte_encoder.items()}
            with open(a				, encoding='''utf-8'''    ) as merges_handle:
                  __snake_case   :     Any	      =merges_handle.read().split('''\n'''    )[1:-1]
            __snake_case   :     List[str]	      =[tuple(merge.split()    ) for merge in bpe_merges]
            __snake_case   :     Dict	      =dict(zip(a				, range(len(a    )    )    )    )
            __snake_case   :     Union[str, Any]	      ={}
            __snake_case   :     List[str]	      =add_prefix_space

            # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
            __snake_case   :     Any	      =re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''    )


      @property
      # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
      def _UpperCamelCase    (     self					:						Any    ):





            """simple docstring"""







            return len(self.encoder    )


      def _UpperCamelCase    (     self					:						List[str]    ):





            """simple docstring"""







            return dict(self.encoder				, **self.added_tokens_encoder    )


      def _UpperCamelCase    (     self					:						Tuple				, a					:						Optional[int]    ):





            """simple docstring"""







            if token in self.cache:
                  return self.cache[token]
            __snake_case   :     Dict	      =tuple(a    )
            __snake_case   :     Optional[int]	      =get_pairs(a    )

            if not pairs:
                  return token

            while True:
                  __snake_case   :     Union[str, Any]	      =min(a				, key=lambda a    : self.bpe_ranks.get(a				, float('''inf'''    )    )    )
                  if bigram not in self.bpe_ranks:
                        break
                  __snake_case      ,					__snake_case   :     Dict	      =bigram
                  __snake_case   :     Optional[int]	      =[]
                  __snake_case   :     Tuple	      =0
                  while i < len(a    ):
                        try:
                              __snake_case   :     Union[str, Any]	      =word.index(a				, a    )
                        except ValueError:
                              new_word.extend(word[i:]    )
                              break
                        else:
                              new_word.extend(word[i:j]    )
                              __snake_case   :     int	      =j

                        if word[i] == first and i < len(a    ) - 1 and word[i + 1] == second:
                              new_word.append(first + second    )
                              i += 2
                        else:
                              new_word.append(word[i]    )
                              i += 1
                  __snake_case   :     int	      =tuple(a    )
                  __snake_case   :     List[str]	      =new_word
                  if len(a    ) == 1:
                        break
                  else:
                        __snake_case   :     Union[str, Any]	      =get_pairs(a    )
            __snake_case   :     List[Any]	      =''' '''.join(a    )
            __snake_case   :     Any	      =word
            return word


      def _UpperCamelCase    (     self					:						List[Any]				, a					:						List[int]				, a					:						Optional[List[int]] = None    ):





            """simple docstring"""







            if token_ids_a is None:
                  return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
            __snake_case   :     List[Any]	      =[self.cls_token_id]
            __snake_case   :     int	      =[self.sep_token_id]
            return cls + token_ids_a + sep + token_ids_a + sep


      def _UpperCamelCase    (     self					:						Any				, a					:						List[int]				, a					:						Optional[List[int]] = None				, a					:						bool = False    ):





            """simple docstring"""







            if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=a				, token_ids_a=a				, already_has_special_tokens=a    )

            if token_ids_a is None:
                  return [1] + ([0] * len(a    )) + [1]
            return [1] + ([0] * len(a    )) + [1] + ([0] * len(a    )) + [1]


      def _UpperCamelCase    (     self					:						Dict				, a					:						List[int]				, a					:						Optional[List[int]] = None    ):





            """simple docstring"""







            __snake_case   :     Union[str, Any]	      =[self.sep_token_id]
            __snake_case   :     int	      =[self.cls_token_id]

            if token_ids_a is None:
                  return len(cls + token_ids_a + sep    ) * [0]
            return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]


      def _UpperCamelCase    (     self					:						List[str]				, a					:						Tuple    ):





            """simple docstring"""







            __snake_case   :     List[Any]	      =[]
            for token in re.findall(self.pat				, a    ):
                  __snake_case   :     Optional[Any]	      =''''''.join(
                      self.byte_encoder[b] for b in token.encode('''utf-8'''    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                  bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a    ).split(''' '''    )    )
            return bpe_tokens


      def _UpperCamelCase    (     self					:						List[Any]				, a					:						Dict    ):





            """simple docstring"""







            return self.encoder.get(a				, self.encoder.get(self.unk_token    )    )


      def _UpperCamelCase    (     self					:						Union[str, Any]				, a					:						Any    ):





            """simple docstring"""







            return self.decoder.get(a    )


      def _UpperCamelCase    (     self					:						List[Any]				, a					:						str    ):





            """simple docstring"""







            __snake_case   :     Dict	      =''''''.join(a    )
            __snake_case   :     Any	      =bytearray([self.byte_decoder[c] for c in text]    ).decode('''utf-8'''				, errors=self.errors    )
            return text


      def _UpperCamelCase    (     self					:						Any				, a					:						str				, a					:						Optional[str] = None    ):





            """simple docstring"""







            if not os.path.isdir(a    ):
                  logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
                  return
            __snake_case   :     List[str]	      =os.path.join(
                a				, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
            __snake_case   :     int	      =os.path.join(
                a				, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']    )

            with open(a				, '''w'''				, encoding='''utf-8'''    ) as f:
                  f.write(json.dumps(self.encoder				, indent=2				, sort_keys=a				, ensure_ascii=a    ) + '''\n'''    )

            __snake_case   :     int	      =0
            with open(a				, '''w'''				, encoding='''utf-8'''    ) as writer:
                  writer.write('''#version: 0.2\n'''    )
                  for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				, key=lambda a    : kv[1]    ):
                        if index != token_index:
                              logger.warning(
                                  f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                  ''' Please check that the tokenizer is not corrupted!'''    )
                              __snake_case   :     List[str]	      =token_index
                        writer.write(''' '''.join(a    ) + '''\n'''    )
                        index += 1

            return vocab_file, merge_file


      def _UpperCamelCase    (     self					:						Union[str, Any]				, a					:						Dict				, a					:						Any=False				, **a					:						int    ):





            """simple docstring"""







            __snake_case   :     Tuple	      =kwargs.pop('''add_prefix_space'''				, self.add_prefix_space    )
            if (is_split_into_words or add_prefix_space) and (len(a    ) > 0 and not text[0].isspace()):
                  __snake_case   :     Optional[Any]	      =''' ''' + text
            return (text, kwargs)







      def _UpperCamelCase    (     self					:						int				, a					:						"Conversation"    ):





            """simple docstring"""







            __snake_case   :     List[Any]	      =[]
            for is_user, text in conversation.iter_texts():
                  input_ids.extend(self.encode(a				, add_special_tokens=a    ) + [self.eos_token_id]    )
            if len(a    ) > self.model_max_length:
                  __snake_case   :     Dict	      =input_ids[-self.model_max_length :]
            return input_ids










"""simple docstring"""





class   _lowercase			:


      def __init__(     self					:						Optional[Any]				, a					:						Union[str, Any]    ):





            """simple docstring"""







            __snake_case   :     Union[str, Any]	      =data
            __snake_case   :     Optional[Any]	      =None
            __snake_case   :     Any	      =None







      def __str__(     self					:						Any    ):





            """simple docstring"""







            return f'''{self.data}'''



class   _lowercase			:


      def __init__(     self					:						Optional[int]    ):





            """simple docstring"""







            __snake_case   :     Any	      =None
            __snake_case   :     int	      =None


      def __iter__(     self					:						int    ):





            """simple docstring"""







            __snake_case   :     List[str]	      =self.head
            while node:
                  yield node.data
                  __snake_case   :     str	      =node.next


      def __str__(     self					:						Optional[int]    ):





            """simple docstring"""







            return "->".join([str(a    ) for item in self]    )


      def __len__(     self					:						str    ):





            """simple docstring"""







            return sum(1 for _ in self    )


      def _UpperCamelCase    (     self					:						int				, a					:						List[Any]    ):





            """simple docstring"""







            self.insert_at_nth(0				, a    )


      def _UpperCamelCase    (     self					:						str				, a					:						Optional[int]    ):





            """simple docstring"""







            self.insert_at_nth(len(self    )				, a    )


      def _UpperCamelCase    (     self					:						Optional[Any]				, a					:						int				, a					:						Tuple    ):





            """simple docstring"""







            __snake_case   :     List[str]	      =len(self    )

            if not 0 <= index <= length:
                  raise IndexError('''list index out of range'''    )
            __snake_case   :     List[Any]	      =Node(a    )
            if self.head is None:
                  __snake_case   :     Dict	      =new_node
            elif index == 0:
                  __snake_case   :     Union[str, Any]	      =new_node
                  __snake_case   :     Any	      =self.head
                  __snake_case   :     List[Any]	      =new_node
            elif index == length:
                  __snake_case   :     int	      =new_node
                  __snake_case   :     Optional[Any]	      =self.tail
                  __snake_case   :     Optional[Any]	      =new_node
            else:
                  __snake_case   :     Any	      =self.head
                  for _ in range(0				, a    ):
                        __snake_case   :     Optional[Any]	      =temp.next
                  __snake_case   :     Optional[Any]	      =new_node
                  __snake_case   :     Union[str, Any]	      =temp.previous
                  __snake_case   :     List[Any]	      =temp
                  __snake_case   :     str	      =new_node


      def _UpperCamelCase    (     self					:						List[str]    ):





            """simple docstring"""







            return self.delete_at_nth(0    )


      def _UpperCamelCase    (     self					:						List[Any]    ):





            """simple docstring"""







            return self.delete_at_nth(len(self    ) - 1    )


      def _UpperCamelCase    (     self					:						Optional[int]				, a					:						int    ):





            """simple docstring"""







            __snake_case   :     Tuple	      =len(self    )

            if not 0 <= index <= length - 1:
                  raise IndexError('''list index out of range'''    )
            __snake_case   :     str	      =self.head  # default first node
            if length == 1:
                  __snake_case   :     List[Any]	      =None
            elif index == 0:
                  __snake_case   :     List[Any]	      =self.head.next
                  __snake_case   :     Optional[int]	      =None
            elif index == length - 1:
                  __snake_case   :     str	      =self.tail
                  __snake_case   :     Union[str, Any]	      =self.tail.previous
                  __snake_case   :     Optional[Any]	      =None
            else:
                  __snake_case   :     Optional[int]	      =self.head
                  for _ in range(0				, a    ):
                        __snake_case   :     List[Any]	      =temp.next
                  __snake_case   :     Union[str, Any]	      =temp
                  __snake_case   :     Optional[int]	      =temp.previous
                  __snake_case   :     int	      =temp.next
            return delete_node.data


      def _UpperCamelCase    (     self					:						List[Any]				, a					:						Any    ):





            """simple docstring"""







            __snake_case   :     Optional[int]	      =self.head

            while current.data != data:  # Find the position to delete
                  if current.next:
                        __snake_case   :     Dict	      =current.next
                  else:  # We have reached the end an no value matches
                        raise ValueError('''No data matching given value'''    )

            if current == self.head:
                  self.delete_head()

            elif current == self.tail:
                  self.delete_tail()

            else:  # Before: 1 <--> 2(current) <--> 3
                  __snake_case   :     Tuple	      =current.next  # 1 --> 3
                  __snake_case   :     Optional[Any]	      =current.previous  # 1 <--> 3
            return data







      def _UpperCamelCase    (     self					:						int    ):





            """simple docstring"""







            return len(self    ) == 0



def      __lowercase       (     )       ->       None:
      __snake_case   :     List[str]	      =DoublyLinkedList()
      assert linked_list.is_empty() is True
      assert str(a			) == ""

      try:
            linked_list.delete_head()
            raise AssertionError  # This should not happen.
      except IndexError:
            assert True  # This should happen.

      try:
            linked_list.delete_tail()
            raise AssertionError  # This should not happen.
      except IndexError:
            assert True  # This should happen.

      for i in range(10			):
            assert len(a			) == i
            linked_list.insert_at_nth(a							,		i + 1			)
      assert str(a			) == "->".join(str(a			) for i in range(1							,		11			)			)

      linked_list.insert_at_head(0			)
      linked_list.insert_at_tail(11			)
      assert str(a			) == "->".join(str(a			) for i in range(0							,		12			)			)

      assert linked_list.delete_head() == 0
      assert linked_list.delete_at_nth(9			) == 10
      assert linked_list.delete_tail() == 11
      assert len(a			) == 9
      assert str(a			) == "->".join(str(a			) for i in range(1							,		10			)			)


if __name__ == "__main__":
    from doctest import testmod

    testmod()


