



from typing import Any





class    __lowerCAmelCase					:




				def __init__(				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       []
									_lowerCAmelCase         =       0




				def __str__(				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       """<""" + str(self.stack						)[1:-1] + """>"""
									return printed




				def 				snake_case  (				self		,							_snake_case						):



									"""simple docstring"""





									self.stack.append(_snake_case						)
									_lowerCAmelCase         =       self.length + 1




				def 				snake_case  (				self						):



									"""simple docstring"""





									self.rotate(1						)
									_lowerCAmelCase         =       self.stack[self.length - 1]
									_lowerCAmelCase         =       self.stack[:-1]
									self.rotate(self.length - 1						)
									_lowerCAmelCase         =       self.length - 1
									return dequeued




				def 				snake_case  (				self		,							_snake_case						):



									"""simple docstring"""





									for _ in range(_snake_case						):
														_lowerCAmelCase         =       self.stack[0]
														_lowerCAmelCase         =       self.stack[1:]
														self.put(_snake_case						)
														_lowerCAmelCase         =       self.length - 1




				def 				snake_case  (				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       self.get()
									self.put(_snake_case						)
									self.rotate(self.length - 1						)
									return front




				def 				snake_case  (				self						):



									"""simple docstring"""





									return self.length









'''simple docstring'''

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a__         =     logging.get_logger(__name__)

a__         =     {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

a__         =     {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

a__         =     {
    '''openai-gpt''': 512,
}




def    snake_case__     (  a	)			->       Tuple:





			'''simple docstring'''






			snake_case__          =						text.strip()
			if not text:
						return []
			snake_case__          =						text.split()
			return tokens





class __magic_name__(						__lowerCAmelCase      ):






						def __init__(   self						:						Any					,   __UpperCamelCase						:						str=True					,   __UpperCamelCase						:						Dict=None					,   __UpperCamelCase						:						Union[str, Any]=True					,   __UpperCamelCase						:						List[str]=None					,   __UpperCamelCase						:						Optional[int]=True					,   ):



									'''simple docstring'''






									if never_split is None:
												snake_case__          =						[]
									snake_case__          =						do_lower_case
									snake_case__          =						set(__UpperCamelCase       )
									snake_case__          =						tokenize_chinese_chars
									snake_case__          =						strip_accents
									snake_case__          =						do_split_on_punc






						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						Any=None       ):



									'''simple docstring'''






									snake_case__          =						self.never_split.union(set(__UpperCamelCase       )       ) if never_split else self.never_split
									snake_case__          =						self._clean_text(__UpperCamelCase       )

									# This was added on November 1st, 2018 for the multilingual and Chinese
									# models. This is also applied to the English models now, but it doesn't
									# matter since the English models were not trained on any Chinese data
									# and generally don't have any Chinese data in them (there are Chinese
									# characters in the vocabulary because Wikipedia does have some Chinese
									# words in the English Wikipedia.).
									if self.tokenize_chinese_chars:
												snake_case__          =						self._tokenize_chinese_chars(__UpperCamelCase       )
									# prevents treating the same character with different unicode codepoints as different characters
									snake_case__          =						unicodedata.normalize("""NFC"""					,   __UpperCamelCase       )
									snake_case__          =						whitespace_tokenize(__UpperCamelCase       )
									snake_case__          =						[]
									for token in orig_tokens:
												if token not in never_split:
															if self.do_lower_case:
																		snake_case__          =						token.lower()
																		if self.strip_accents is not False:
																					snake_case__          =						self._run_strip_accents(__UpperCamelCase       )
															elif self.strip_accents:
																		snake_case__          =						self._run_strip_accents(__UpperCamelCase       )
												split_tokens.extend(self._run_split_on_punc(__UpperCamelCase					,   __UpperCamelCase       )       )

									snake_case__          =						whitespace_tokenize(""" """.join(__UpperCamelCase       )       )
									return output_tokens






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						Tuple       ):



									'''simple docstring'''






									snake_case__          =						unicodedata.normalize("""NFD"""					,   __UpperCamelCase       )
									snake_case__          =						[]
									for char in text:
												snake_case__          =						unicodedata.category(__UpperCamelCase       )
												if cat == "Mn":
															continue
												output.append(__UpperCamelCase       )
									return "".join(__UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						int=None       ):



									'''simple docstring'''






									if not self.do_split_on_punc or (never_split is not None and text in never_split):
												return [text]
									snake_case__          =						list(__UpperCamelCase       )
									snake_case__          =						0
									snake_case__          =						True
									snake_case__          =						[]
									while i < len(__UpperCamelCase       ):
												snake_case__          =						chars[i]
												if _is_punctuation(__UpperCamelCase       ):
															output.append([char]       )
															snake_case__          =						True
												else:
															if start_new_word:
																		output.append([]       )
															snake_case__          =						False
															output[-1].append(__UpperCamelCase       )
												i += 1

									return ["".join(__UpperCamelCase       ) for x in output]






						def       __lowerCAmelCase(   self						:						Tuple					,   __UpperCamelCase						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						[]
									for char in text:
												snake_case__          =						ord(__UpperCamelCase       )
												if self._is_chinese_char(__UpperCamelCase       ):
															output.append(""" """       )
															output.append(__UpperCamelCase       )
															output.append(""" """       )
												else:
															output.append(__UpperCamelCase       )
									return "".join(__UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Any					,   __UpperCamelCase						:						str       ):



									'''simple docstring'''






									if (
									    (cp >= 0X4_e00 and cp <= 0X9_fff)
									    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
									    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
									    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
									    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
									    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
									    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
									    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
									):  #
												return True

									return False







						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						Any       ):



									'''simple docstring'''






									snake_case__          =						[]
									for char in text:
												snake_case__          =						ord(__UpperCamelCase       )
												if cp == 0 or cp == 0Xf_ffd or _is_control(__UpperCamelCase       ):
															continue
												if _is_whitespace(__UpperCamelCase       ):
															output.append(""" """       )
												else:
															output.append(__UpperCamelCase       )
									return "".join(__UpperCamelCase       )







def    snake_case__     (  a	)			->       int:





			'''simple docstring'''






			snake_case__          =						set()
			snake_case__          =						word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)	)
						snake_case__          =						char
			return pairs




def    snake_case__     (  a	)			->       Union[str, Any]:





			'''simple docstring'''






			snake_case__          =						text.replace("""—"""       ,      """-"""	)
			snake_case__          =						text.replace("""–"""       ,      """-"""	)
			snake_case__          =						text.replace("""―"""       ,      """-"""	)
			snake_case__          =						text.replace("""…"""       ,      """..."""	)
			snake_case__          =						text.replace("""´"""       ,      """'"""	)
			snake_case__          =						re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""       ,      R""" \1 """       ,      a	)
			snake_case__          =						re.sub(R"""\s*\n\s*"""       ,      """ \n """       ,      a	)
			snake_case__          =						re.sub(R"""[^\S\n]+"""       ,      """ """       ,      a	)
			return text.strip()





class __magic_name__(						__lowerCAmelCase      ):
						UpperCAmelCase_		:					str    =  VOCAB_FILES_NAMES
						UpperCAmelCase_		:					Dict    =  PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase_		:					Optional[Any]    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase_		:					List[str]    =  ["""input_ids""", """attention_mask"""]






						def __init__(   self						:						int					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Dict="<unk>"					,   **__UpperCamelCase						:						Optional[Any]       ):



									'''simple docstring'''






									super().__init__(unk_token=__UpperCamelCase					,   **__UpperCamelCase       )

									try:
												import ftfy
												from spacy.lang.en import English

												snake_case__          =						English()
												snake_case__          =						_nlp.tokenizer
												snake_case__          =						ftfy.fix_text
									except ImportError:
												logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""       )
												snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase       )
												snake_case__          =						None

									with open(__UpperCamelCase					,   encoding="""utf-8"""       ) as vocab_handle:
												snake_case__          =						json.load(__UpperCamelCase       )
									snake_case__          =						{v: k for k, v in self.encoder.items()}
									with open(__UpperCamelCase					,   encoding="""utf-8"""       ) as merges_handle:
												snake_case__          =						merges_handle.read().split("""\n"""       )[1:-1]
									snake_case__          =						[tuple(merge.split()       ) for merge in merges]
									snake_case__          =						dict(zip(__UpperCamelCase					,   range(len(__UpperCamelCase       )       )       )       )
									snake_case__          =						{}






						@property
						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									return True






						@property
						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									return len(self.encoder       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									return dict(self.encoder					,   **self.added_tokens_encoder       )






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						tuple(token[:-1]       ) + (token[-1] + """</w>""",)
									if token in self.cache:
												return self.cache[token]
									snake_case__          =						get_pairs(__UpperCamelCase       )

									if not pairs:
												return token + "</w>"

									while True:
												snake_case__          =						min(__UpperCamelCase					,   key=lambda __UpperCamelCase       : self.bpe_ranks.get(__UpperCamelCase					,   float("""inf"""       )       )       )
												if bigram not in self.bpe_ranks:
															break
												snake_case__    ,			snake_case__          =						bigram
												snake_case__          =						[]
												snake_case__          =						0
												while i < len(__UpperCamelCase       ):
															try:
																		snake_case__          =						word.index(__UpperCamelCase					,   __UpperCamelCase       )
															except ValueError:
																		new_word.extend(word[i:]       )
																		break
															else:
																		new_word.extend(word[i:j]       )
																		snake_case__          =						j

															if word[i] == first and i < len(__UpperCamelCase       ) - 1 and word[i + 1] == second:
																		new_word.append(first + second       )
																		i += 2
															else:
																		new_word.append(word[i]       )
																		i += 1
												snake_case__          =						tuple(__UpperCamelCase       )
												snake_case__          =						new_word
												if len(__UpperCamelCase       ) == 1:
															break
												else:
															snake_case__          =						get_pairs(__UpperCamelCase       )
									snake_case__          =						""" """.join(__UpperCamelCase       )
									if word == "\n  </w>":
												snake_case__          =						"""\n</w>"""
									snake_case__          =						word
									return word






						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						[]
									if self.fix_text is None:
												# Using BERT's BasicTokenizer
												snake_case__          =						self.nlp.tokenize(__UpperCamelCase       )
												for token in text:
															split_tokens.extend(list(self.bpe(__UpperCamelCase       ).split(""" """       )       )       )
									else:
												# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
												snake_case__          =						self.nlp(text_standardize(self.fix_text(__UpperCamelCase       )       )       )
												for token in text:
															split_tokens.extend(list(self.bpe(token.text.lower()       ).split(""" """       )       )       )
									return split_tokens






						def       __lowerCAmelCase(   self						:						List[Any]					,   __UpperCamelCase						:						str       ):



									'''simple docstring'''






									return self.encoder.get(__UpperCamelCase					,   self.encoder.get(self.unk_token       )       )






						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						str       ):



									'''simple docstring'''






									return self.decoder.get(__UpperCamelCase					,   self.unk_token       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]					,   __UpperCamelCase						:						int       ):



									'''simple docstring'''






									snake_case__          =						"""""".join(__UpperCamelCase       ).replace("""</w>"""					,   """ """       ).strip()
									return out_string







						def       __lowerCAmelCase(   self						:						Any					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Optional[str] = None       ):



									'''simple docstring'''






									if not os.path.isdir(__UpperCamelCase       ):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
												return
									snake_case__          =						os.path.join(
									    __UpperCamelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
									snake_case__          =						os.path.join(
									    __UpperCamelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

									with open(__UpperCamelCase					,   """w"""					,   encoding="""utf-8"""       ) as f:
												f.write(json.dumps(self.encoder					,   indent=2					,   sort_keys=__UpperCamelCase					,   ensure_ascii=__UpperCamelCase       ) + """\n"""       )

									snake_case__          =						0
									with open(__UpperCamelCase					,   """w"""					,   encoding="""utf-8"""       ) as writer:
												writer.write("""#version: 0.2\n"""       )
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,   key=lambda __UpperCamelCase       : kv[1]       ):
															if index != token_index:
																		logger.warning(
																		    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																		    """ Please check that the tokenizer is not corrupted!"""       )
																		snake_case__          =						token_index
															writer.write(""" """.join(__UpperCamelCase       ) + """\n"""       )
															index += 1

									return vocab_file, merge_file