

import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation

_lowerCamelCase							=logging.get_logger(__name__)

_lowerCamelCase							={
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_lowerCamelCase							={
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
}

_lowerCamelCase							={
    'gpt2': 10_24,
    'gpt2-medium': 10_24,
    'gpt2-large': 10_24,
    'gpt2-xl': 10_24,
    'distilgpt2': 10_24,
}







@lru_cache()
def      snake_case__							(  ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE 		=(
         list(range(ord('!'     ),       ord('~'     ) + 1     )     ) + list(range(ord('¡'     ),       ord('¬'     ) + 1     )     ) + list(range(ord('®'     ),       ord('ÿ'     ) + 1     )     )
     )
     SCREAMING_SNAKE_CASE 		=bs[:]
     SCREAMING_SNAKE_CASE 		=0
     for b in range(2**8     ):
          if b not in bs:
               bs.append(a_     )
               cs.append(2**8 + n     )
               n += 1
     SCREAMING_SNAKE_CASE 		=[chr(a_     ) for n in cs]
     return dict(zip(a_,       a_     )     )







def      snake_case__							(  lowerCAmelCase_     ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE 		=set()
     SCREAMING_SNAKE_CASE 		=word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)     )
          SCREAMING_SNAKE_CASE 		=char
     return pairs



class 						a_			(				__A	):



      """simple docstring"""







      __UpperCAmelCase     						= VOCAB_FILES_NAMES
      __UpperCAmelCase     						= PRETRAINED_VOCAB_FILES_MAP
      __UpperCAmelCase     						= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCAmelCase     						= ['input_ids', 'attention_mask']
      def __init__( self	:				Tuple     ,snake_case	:				str     ,snake_case	:				Union[str, Any]     ,snake_case	:				int="replace"     ,snake_case	:				Optional[int]="<|endoftext|>"     ,snake_case	:				Optional[int]="<|endoftext|>"     ,snake_case	:				Tuple="<|endoftext|>"     ,snake_case	:				Optional[int]=None     ,snake_case	:				List[str]=False     ,snake_case	:				List[str]=False     ,**snake_case	:				str     ,):
           SCREAMING_SNAKE_CASE 		=AddedToken(_lowerCamelCase     ,lstrip=_lowerCamelCase     ,rstrip=_lowerCamelCase							) if isinstance(_lowerCamelCase     ,_lowerCamelCase							) else bos_token
           SCREAMING_SNAKE_CASE 		=AddedToken(_lowerCamelCase     ,lstrip=_lowerCamelCase     ,rstrip=_lowerCamelCase							) if isinstance(_lowerCamelCase     ,_lowerCamelCase							) else eos_token
           SCREAMING_SNAKE_CASE 		=AddedToken(_lowerCamelCase     ,lstrip=_lowerCamelCase     ,rstrip=_lowerCamelCase							) if isinstance(_lowerCamelCase     ,_lowerCamelCase							) else unk_token
           SCREAMING_SNAKE_CASE 		=AddedToken(_lowerCamelCase     ,lstrip=_lowerCamelCase     ,rstrip=_lowerCamelCase							) if isinstance(_lowerCamelCase     ,_lowerCamelCase							) else pad_token
           super().__init__(
               errors=_lowerCamelCase     ,unk_token=_lowerCamelCase     ,bos_token=_lowerCamelCase     ,eos_token=_lowerCamelCase     ,pad_token=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase     ,add_bos_token=_lowerCamelCase     ,**_lowerCamelCase     ,)
           SCREAMING_SNAKE_CASE 		=add_bos_token

           with open(_lowerCamelCase     ,encoding='utf-8'							) as vocab_handle:
                SCREAMING_SNAKE_CASE 		=json.load(_lowerCamelCase							)
           SCREAMING_SNAKE_CASE 		={v: k for k, v in self.encoder.items()}
           SCREAMING_SNAKE_CASE 		=errors  # how to handle errors in decoding
           SCREAMING_SNAKE_CASE 		=bytes_to_unicode()
           SCREAMING_SNAKE_CASE 		={v: k for k, v in self.byte_encoder.items()}
           with open(_lowerCamelCase     ,encoding='utf-8'							) as merges_handle:
                SCREAMING_SNAKE_CASE 		=merges_handle.read().split('\n'							)[1:-1]
           SCREAMING_SNAKE_CASE 		=[tuple(merge.split()							) for merge in bpe_merges]
           SCREAMING_SNAKE_CASE 		=dict(zip(_lowerCamelCase     ,range(len(_lowerCamelCase							)							)							)							)
           SCREAMING_SNAKE_CASE 		={}
           SCREAMING_SNAKE_CASE 		=add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           SCREAMING_SNAKE_CASE 		=re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'							)
      @property
      def 			_lowerCAmelCase	( self	:				Tuple							):
           return len(self.encoder							)
      def 			_lowerCAmelCase	( self	:				int							):
           return dict(self.encoder     ,**self.added_tokens_encoder							)
      def 			_lowerCAmelCase	( self	:				Optional[int]     ,snake_case	:				Optional[Any]							):
           if token in self.cache:
                return self.cache[token]
           SCREAMING_SNAKE_CASE 		=tuple(_lowerCamelCase							)
           SCREAMING_SNAKE_CASE 		=get_pairs(_lowerCamelCase							)

           if not pairs:
                return token

           while True:
                SCREAMING_SNAKE_CASE 		=min(_lowerCamelCase     ,key=lambda snake_case							: self.bpe_ranks.get(_lowerCamelCase     ,float('inf'							)							)							)
                if bigram not in self.bpe_ranks:
                     break
                SCREAMING_SNAKE_CASE 		=bigram
                SCREAMING_SNAKE_CASE 		=[]
                SCREAMING_SNAKE_CASE 		=0
                while i < len(_lowerCamelCase							):
                     try:
                          SCREAMING_SNAKE_CASE 		=word.index(_lowerCamelCase     ,_lowerCamelCase							)
                     except ValueError:
                          new_word.extend(word[i:]							)
                          break
                     else:
                          new_word.extend(word[i:j]							)
                          SCREAMING_SNAKE_CASE 		=j

                     if word[i] == first and i < len(_lowerCamelCase							) - 1 and word[i + 1] == second:
                          new_word.append(first + second							)
                          i += 2
                     else:
                          new_word.append(word[i]							)
                          i += 1
                SCREAMING_SNAKE_CASE 		=tuple(_lowerCamelCase							)
                SCREAMING_SNAKE_CASE 		=new_word
                if len(_lowerCamelCase							) == 1:
                     break
                else:
                     SCREAMING_SNAKE_CASE 		=get_pairs(_lowerCamelCase							)
           SCREAMING_SNAKE_CASE 		=""" """.join(_lowerCamelCase							)
           SCREAMING_SNAKE_CASE 		=word
           return word
      def 			_lowerCAmelCase	( self	:				List[Any]     ,snake_case	:				int     ,snake_case	:				Any=None							):
           if self.add_bos_token:
                SCREAMING_SNAKE_CASE 		=[self.bos_token_id]
           else:
                SCREAMING_SNAKE_CASE 		=[]

           SCREAMING_SNAKE_CASE 		=bos_token_ids + token_ids_a

           if token_ids_a is None:
                return output

           return output + bos_token_ids + token_ids_a
      def 			_lowerCAmelCase	( self	:				Dict     ,snake_case	:				Any     ,snake_case	:				Tuple = None     ,snake_case	:				Any = False							):
           if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=_lowerCamelCase     ,token_ids_a=_lowerCamelCase     ,already_has_special_tokens=_lowerCamelCase							)

           if not self.add_bos_token:
                return super().get_special_tokens_mask(
                    token_ids_a=_lowerCamelCase     ,token_ids_a=_lowerCamelCase     ,already_has_special_tokens=_lowerCamelCase							)

           if token_ids_a is None:
                return [1] + ([0] * len(_lowerCamelCase							))
           return [1] + ([0] * len(_lowerCamelCase							)) + [1] + ([0] * len(_lowerCamelCase							))
      def 			_lowerCAmelCase	( self	:				List[str]     ,snake_case	:				Tuple							):
           SCREAMING_SNAKE_CASE 		=[]
           for token in re.findall(self.pat     ,_lowerCamelCase							):
                SCREAMING_SNAKE_CASE 		="""""".join(
                    self.byte_encoder[b] for b in token.encode('utf-8'							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase							).split(' '							)							)
           return bpe_tokens
      def 			_lowerCAmelCase	( self	:				Dict     ,snake_case	:				List[str]							):
           return self.encoder.get(_lowerCamelCase     ,self.encoder.get(self.unk_token							)							)
      def 			_lowerCAmelCase	( self	:				Any     ,snake_case	:				Optional[Any]							):
           return self.decoder.get(_lowerCamelCase							)
      def 			_lowerCAmelCase	( self	:				Dict     ,snake_case	:				str							):
           SCREAMING_SNAKE_CASE 		="""""".join(_lowerCamelCase							)
           SCREAMING_SNAKE_CASE 		=bytearray([self.byte_decoder[c] for c in text]							).decode('utf-8'     ,errors=self.errors							)
           return text
      def 			_lowerCAmelCase	( self	:				Optional[Any]     ,snake_case	:				str     ,snake_case	:				Dict = None							):
           if not os.path.isdir(_lowerCamelCase							):
                logger.error(f'Vocabulary path ({save_directory}) should be a directory'							)
                return
           SCREAMING_SNAKE_CASE 		=os.path.join(
               _lowerCamelCase     ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)
           SCREAMING_SNAKE_CASE 		=os.path.join(
               _lowerCamelCase     ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']							)

           with open(_lowerCamelCase     ,'w'     ,encoding='utf-8'							) as f:
                f.write(json.dumps(self.encoder     ,indent=2     ,sort_keys=_lowerCamelCase     ,ensure_ascii=_lowerCamelCase							) + '\n'							)

           SCREAMING_SNAKE_CASE 		=0
           with open(_lowerCamelCase     ,'w'     ,encoding='utf-8'							) as writer:
                writer.write('#version: 0.2\n'							)
                for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,key=lambda snake_case							: kv[1]							):
                     if index != token_index:
                          logger.warning(
                              f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                              ' Please check that the tokenizer is not corrupted!'							)
                          SCREAMING_SNAKE_CASE 		=token_index
                     writer.write(' '.join(_lowerCamelCase							) + '\n'							)
                     index += 1

           return vocab_file, merge_file
      def 			_lowerCAmelCase	( self	:				Optional[Any]     ,snake_case	:				List[str]     ,snake_case	:				int=False     ,**snake_case	:				Optional[int]							):
           SCREAMING_SNAKE_CASE 		=kwargs.pop('add_prefix_space'     ,self.add_prefix_space							)
           if is_split_into_words or add_prefix_space:
                SCREAMING_SNAKE_CASE 		=""" """ + text
           return (text, kwargs)


      def 			_lowerCAmelCase	( self	:				int     ,snake_case	:				Tuple							):
           SCREAMING_SNAKE_CASE 		=[]
           for is_user, text in conversation.iter_texts():
                input_ids.extend(self.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase							) + [self.eos_token_id]							)
           if len(_lowerCamelCase							) > self.model_max_length:
                SCREAMING_SNAKE_CASE 		=input_ids[-self.model_max_length :]
           return input_ids






'''simple docstring'''





from ..utils import DummyObject, requires_backends


class 							_lowerCAmelCase	(  metaclass=__A							):


 """simple docstring"""

 lowerCamelCase									=     ['''torch''', '''librosa''']







 def __init__(      self					,    *_lowerCamelCase					,    **_lowerCamelCase  )    ->    str:
    requires_backends(self					,    ["""torch""", """librosa"""]  )







 @classmethod
 def       UpperCAmelCase_       (      cls					,    *_lowerCamelCase					,    **_lowerCamelCase  )    ->    Dict:
    requires_backends(cls					,    ["""torch""", """librosa"""]  )







 @classmethod
 def       UpperCAmelCase_       (      cls					,    *_lowerCamelCase					,    **_lowerCamelCase  )    ->    Optional[int]:
    requires_backends(cls					,    ["""torch""", """librosa"""]  )


class 							_lowerCAmelCase	(  metaclass=__A							):


 """simple docstring"""

 lowerCamelCase									=     ['''torch''', '''librosa''']







 def __init__(      self					,    *_lowerCamelCase					,    **_lowerCamelCase  )    ->    Dict:
    requires_backends(self					,    ["""torch""", """librosa"""]  )







 @classmethod
 def       UpperCAmelCase_       (      cls					,    *_lowerCamelCase					,    **_lowerCamelCase  )    ->    str:
    requires_backends(cls					,    ["""torch""", """librosa"""]  )







 @classmethod
 def       UpperCAmelCase_       (      cls					,    *_lowerCamelCase					,    **_lowerCamelCase  )    ->    Dict:
    requires_backends(cls					,    ["""torch""", """librosa"""]  )
