
"""simple docstring"""




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation

__lowerCAmelCase							:  List[str] 				=		logging.get_logger(__name__)

__lowerCAmelCase							:  Union[str, Any] 				=		{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__lowerCAmelCase							:  List[Any] 				=		{
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
}

__lowerCAmelCase							:  Any 				=		{
    '''gpt2''': 1024,
    '''gpt2-medium''': 1024,
    '''gpt2-large''': 1024,
    '''gpt2-xl''': 1024,
    '''distilgpt2''': 1024,
}

@lru_cache()
def __snake_case (      )       ->    int:


		"""simple docstring"""





		a__   		=     (
		    list(range(ord('''!'''    )						,					ord('''~'''    ) + 1    )    ) + list(range(ord('''¡'''    )						,					ord('''¬'''    ) + 1    )    ) + list(range(ord('''®'''    )						,					ord('''ÿ'''    ) + 1    )    )
		)
		a__   		=     bs[:]
		a__   		=     0
		for b in range(2**8    ):
				if b not in bs:
						bs.append(UpperCamelCase    )
						cs.append(2**8 + n    )
						n += 1
		a__   		=     [chr(UpperCamelCase    ) for n in cs]
		return dict(zip(UpperCamelCase						,					UpperCamelCase    )    )

def __snake_case (      UpperCamelCase    )       ->    str:


		"""simple docstring"""





		a__   		=     set()
		a__   		=     word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)    )
				a__   		=     char
		return pairs




class SCREAMING_SNAKE_CASE       (      lowerCAmelCase					):






	'''simple docstring'''







	snake_case__							:		Union[str, Any]         =		VOCAB_FILES_NAMES
	snake_case__							:		Optional[Any]         =		PRETRAINED_VOCAB_FILES_MAP
	snake_case__							:		List[str]         =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__							:		int         =		['input_ids', 'attention_mask']


	def __init__(  self :List[str]       ,    __magic_name__ :Optional[int]       ,    __magic_name__ :Tuple       ,    __magic_name__ :Union[str, Any]="replace"       ,    __magic_name__ :Any="<|endoftext|>"       ,    __magic_name__ :str="<|endoftext|>"       ,    __magic_name__ :Dict="<|endoftext|>"       ,    __magic_name__ :Optional[Any]=None       ,    __magic_name__ :Optional[int]=False       ,    __magic_name__ :Optional[Any]=False       ,    **__magic_name__ :Optional[Any]       ,    )			->   Union[str, Any]:

			'''simple docstring'''






			a__   		=     AddedToken(__magic_name__       ,    lstrip=__magic_name__       ,    rstrip=__magic_name__			) if isinstance(__magic_name__       ,    __magic_name__			) else bos_token
			a__   		=     AddedToken(__magic_name__       ,    lstrip=__magic_name__       ,    rstrip=__magic_name__			) if isinstance(__magic_name__       ,    __magic_name__			) else eos_token
			a__   		=     AddedToken(__magic_name__       ,    lstrip=__magic_name__       ,    rstrip=__magic_name__			) if isinstance(__magic_name__       ,    __magic_name__			) else unk_token
			a__   		=     AddedToken(__magic_name__       ,    lstrip=__magic_name__       ,    rstrip=__magic_name__			) if isinstance(__magic_name__       ,    __magic_name__			) else pad_token
			super().__init__(
			    errors=__magic_name__       ,    unk_token=__magic_name__       ,    bos_token=__magic_name__       ,    eos_token=__magic_name__       ,    pad_token=__magic_name__       ,    add_prefix_space=__magic_name__       ,    add_bos_token=__magic_name__       ,    **__magic_name__       ,    )
			a__   		=     add_bos_token

			with open(__magic_name__       ,    encoding='''utf-8'''			) as vocab_handle:
					a__   		=     json.load(__magic_name__			)
			a__   		=     {v: k for k, v in self.encoder.items()}
			a__   		=     errors  # how to handle errors in decoding
			a__   		=     bytes_to_unicode()
			a__   		=     {v: k for k, v in self.byte_encoder.items()}
			with open(__magic_name__       ,    encoding='''utf-8'''			) as merges_handle:
					a__   		=     merges_handle.read().split('''\n'''			)[1:-1]
			a__   		=     [tuple(merge.split()			) for merge in bpe_merges]
			a__   		=     dict(zip(__magic_name__       ,    range(len(__magic_name__			)			)			)			)
			a__   		=     {}
			a__   		=     add_prefix_space

			# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
			a__   		=     re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''			)


	@property
	def    _UpperCamelCase							(  self :Dict			)			->   Optional[int]:

			'''simple docstring'''






			return len(self.encoder			)


	def    _UpperCamelCase							(  self :Any			)			->   Optional[Any]:

			'''simple docstring'''






			return dict(self.encoder       ,    **self.added_tokens_encoder			)


	def    _UpperCamelCase							(  self :Dict       ,    __magic_name__ :Any			)			->   Union[str, Any]:

			'''simple docstring'''






			if token in self.cache:
					return self.cache[token]
			a__   		=     tuple(__magic_name__			)
			a__   		=     get_pairs(__magic_name__			)

			if not pairs:
					return token

			while True:
					a__   		=     min(__magic_name__       ,    key=lambda __magic_name__			: self.bpe_ranks.get(__magic_name__       ,    float('''inf'''			)			)			)
					if bigram not in self.bpe_ranks:
							break
					a__		,	a__   		=     bigram
					a__   		=     []
					a__   		=     0
					while i < len(__magic_name__			):
							try:
									a__   		=     word.index(__magic_name__       ,    __magic_name__			)
							except ValueError:
									new_word.extend(word[i:]			)
									break
							else:
									new_word.extend(word[i:j]			)
									a__   		=     j

							if word[i] == first and i < len(__magic_name__			) - 1 and word[i + 1] == second:
									new_word.append(first + second			)
									i += 2
							else:
									new_word.append(word[i]			)
									i += 1
					a__   		=     tuple(__magic_name__			)
					a__   		=     new_word
					if len(__magic_name__			) == 1:
							break
					else:
							a__   		=     get_pairs(__magic_name__			)
			a__   		=     ''' '''.join(__magic_name__			)
			a__   		=     word
			return word


	def    _UpperCamelCase							(  self :Optional[int]       ,    __magic_name__ :Union[str, Any]       ,    __magic_name__ :int=None			)			->   Dict:

			'''simple docstring'''






			if self.add_bos_token:
					a__   		=     [self.bos_token_id]
			else:
					a__   		=     []

			a__   		=     bos_token_ids + token_ids_a

			if token_ids_a is None:
					return output

			return output + bos_token_ids + token_ids_a


	def    _UpperCamelCase							(  self :int       ,    __magic_name__ :List[int]       ,    __magic_name__ :Optional[List[int]] = None       ,    __magic_name__ :bool = False			)			->   List[int]:

			'''simple docstring'''






			if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=__magic_name__       ,    token_ids_a=__magic_name__       ,    already_has_special_tokens=__magic_name__			)

			if not self.add_bos_token:
					return super().get_special_tokens_mask(
					    token_ids_a=__magic_name__       ,    token_ids_a=__magic_name__       ,    already_has_special_tokens=__magic_name__			)

			if token_ids_a is None:
					return [1] + ([0] * len(__magic_name__			))
			return [1] + ([0] * len(__magic_name__			)) + [1] + ([0] * len(__magic_name__			))


	def    _UpperCamelCase							(  self :List[Any]       ,    __magic_name__ :List[str]			)			->   List[Any]:

			'''simple docstring'''






			a__   		=     []
			for token in re.findall(self.pat       ,    __magic_name__			):
					a__   		=     ''''''.join(
					    self.byte_encoder[b] for b in token.encode('''utf-8'''			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
					bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__magic_name__			).split(''' '''			)			)
			return bpe_tokens


	def    _UpperCamelCase							(  self :List[Any]       ,    __magic_name__ :Dict			)			->   List[Any]:

			'''simple docstring'''






			return self.encoder.get(__magic_name__       ,    self.encoder.get(self.unk_token			)			)


	def    _UpperCamelCase							(  self :Tuple       ,    __magic_name__ :int			)			->   List[str]:

			'''simple docstring'''






			return self.decoder.get(__magic_name__			)


	def    _UpperCamelCase							(  self :Union[str, Any]       ,    __magic_name__ :Optional[Any]			)			->   List[Any]:

			'''simple docstring'''






			a__   		=     ''''''.join(__magic_name__			)
			a__   		=     bytearray([self.byte_decoder[c] for c in text]			).decode('''utf-8'''       ,    errors=self.errors			)
			return text


	def    _UpperCamelCase							(  self :Optional[Any]       ,    __magic_name__ :str       ,    __magic_name__ :Optional[str] = None			)			->   Tuple[str]:

			'''simple docstring'''






			if not os.path.isdir(__magic_name__			):
					logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
					return
			a__   		=     os.path.join(
			    __magic_name__       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)
			a__   		=     os.path.join(
			    __magic_name__       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']			)

			with open(__magic_name__       ,    '''w'''       ,    encoding='''utf-8'''			) as f:
					f.write(json.dumps(self.encoder       ,    indent=2       ,    sort_keys=__magic_name__       ,    ensure_ascii=__magic_name__			) + '''\n'''			)

			a__   		=     0
			with open(__magic_name__       ,    '''w'''       ,    encoding='''utf-8'''			) as writer:
					writer.write('''#version: 0.2\n'''			)
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,    key=lambda __magic_name__			: kv[1]			):
							if index != token_index:
									logger.warning(
									    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
									    ''' Please check that the tokenizer is not corrupted!'''			)
									a__   		=     token_index
							writer.write(''' '''.join(__magic_name__			) + '''\n'''			)
							index += 1

			return vocab_file, merge_file


	def    _UpperCamelCase							(  self :Dict       ,    __magic_name__ :Any       ,    __magic_name__ :Union[str, Any]=False       ,    **__magic_name__ :Tuple			)			->   str:

			'''simple docstring'''






			a__   		=     kwargs.pop('''add_prefix_space'''       ,    self.add_prefix_space			)
			if is_split_into_words or add_prefix_space:
					a__   		=     ''' ''' + text
			return (text, kwargs)






	def    _UpperCamelCase							(  self :Optional[Any]       ,    __magic_name__ :"Conversation"			)			->   List[int]:

			'''simple docstring'''






			a__   		=     []
			for is_user, text in conversation.iter_texts():
					input_ids.extend(self.encode(__magic_name__       ,    add_special_tokens=__magic_name__			) + [self.eos_token_id]			)
			if len(__magic_name__			) > self.model_max_length:
					a__   		=     input_ids[-self.model_max_length :]
			return input_ids






"""simple docstring"""




import safs
class SCREAMING_SNAKE_CASE       (      safs.SaFileSystem					):






	'''simple docstring'''




