"""simple docstring"""




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__lowerCamelCase									=	logging.get_logger(__name__)

__lowerCamelCase									=	{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__lowerCamelCase									=	{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

__lowerCamelCase									=	{
    "openai-gpt": 5_12,
}






def 	UpperCAmelCase       (						UpperCamelCase__					):






							"""simple docstring"""





							A__						  =				text.strip()
							if not text:
														return []
							A__						  =				text.split()
							return tokens
class      UpperCamelCase__(      __A			):





						def __init__(							self		,__UpperCAmelCase=True		,__UpperCAmelCase=None		,__UpperCAmelCase=True		,__UpperCAmelCase=None		,__UpperCAmelCase=True		,)       ->					Dict:
													if never_split is None:
																				A__						  =				[]
													A__						  =				do_lower_case
													A__						  =				set(__UpperCAmelCase  )
													A__						  =				tokenize_chinese_chars
													A__						  =				strip_accents
													A__						  =				do_split_on_punc





						def 					snake_case__						(							self		,__UpperCAmelCase		,__UpperCAmelCase=None  )       ->					str:
													A__						  =				self.never_split.union(set(__UpperCAmelCase  )  ) if never_split else self.never_split
													A__						  =				self._clean_text(__UpperCAmelCase  )

													# This was added on November 1st, 2018 for the multilingual and Chinese
													# models. This is also applied to the English models now, but it doesn't
													# matter since the English models were not trained on any Chinese data
													# and generally don't have any Chinese data in them (there are Chinese
													# characters in the vocabulary because Wikipedia does have some Chinese
													# words in the English Wikipedia.).
													if self.tokenize_chinese_chars:
																				A__						  =				self._tokenize_chinese_chars(__UpperCAmelCase  )
													# prevents treating the same character with different unicode codepoints as different characters
													A__						  =				unicodedata.normalize('NFC'		,__UpperCAmelCase  )
													A__						  =				whitespace_tokenize(__UpperCAmelCase  )
													A__						  =				[]
													for token in orig_tokens:
																				if token not in never_split:
																											if self.do_lower_case:
																																		A__						  =				token.lower()
																																		if self.strip_accents is not False:
																																									A__						  =				self._run_strip_accents(__UpperCAmelCase  )
																											elif self.strip_accents:
																																		A__						  =				self._run_strip_accents(__UpperCAmelCase  )
																				split_tokens.extend(self._run_split_on_punc(__UpperCAmelCase		,__UpperCAmelCase  )  )

													A__						  =				whitespace_tokenize(' '.join(__UpperCAmelCase  )  )
													return output_tokens





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					Optional[int]:
													A__						  =				unicodedata.normalize('NFD'		,__UpperCAmelCase  )
													A__						  =				[]
													for char in text:
																				A__						  =				unicodedata.category(__UpperCAmelCase  )
																				if cat == "Mn":
																											continue
																				output.append(__UpperCAmelCase  )
													return "".join(__UpperCAmelCase  )





						def 					snake_case__						(							self		,__UpperCAmelCase		,__UpperCAmelCase=None  )       ->					Optional[Any]:
													if not self.do_split_on_punc or (never_split is not None and text in never_split):
																				return [text]
													A__						  =				list(__UpperCAmelCase  )
													A__						  =				0
													A__						  =				True
													A__						  =				[]
													while i < len(__UpperCAmelCase  ):
																				A__						  =				chars[i]
																				if _is_punctuation(__UpperCAmelCase  ):
																											output.append([char]  )
																											A__						  =				True
																				else:
																											if start_new_word:
																																		output.append([]  )
																											A__						  =				False
																											output[-1].append(__UpperCAmelCase  )
																				i += 1

													return ["".join(__UpperCAmelCase  ) for x in output]





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					str:
													A__						  =				[]
													for char in text:
																				A__						  =				ord(__UpperCAmelCase  )
																				if self._is_chinese_char(__UpperCAmelCase  ):
																											output.append(' '  )
																											output.append(__UpperCAmelCase  )
																											output.append(' '  )
																				else:
																											output.append(__UpperCAmelCase  )
													return "".join(__UpperCAmelCase  )





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					Any:
													# This defines a "chinese character" as anything in the CJK Unicode block:
													#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
													#
													# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
													# despite its name. The modern Korean Hangul alphabet is a different block,
													# as is Japanese Hiragana and Katakana. Those alphabets are used to write
													# space-separated words, so they are not treated specially and handled
													# like the all of the other languages.
													if (
													    (cp >= 0X4_E00 and cp <= 0X9_FFF)
													    or (cp >= 0X3_400 and cp <= 0X4_DBF)  #
													    or (cp >= 0X20_000 and cp <= 0X2A_6DF)  #
													    or (cp >= 0X2A_700 and cp <= 0X2B_73F)  #
													    or (cp >= 0X2B_740 and cp <= 0X2B_81F)  #
													    or (cp >= 0X2B_820 and cp <= 0X2C_EAF)  #
													    or (cp >= 0XF_900 and cp <= 0XF_AFF)
													    or (cp >= 0X2F_800 and cp <= 0X2F_A1F)  #
													):  #
																				return True

													return False





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					Optional[int]:
													A__						  =				[]
													for char in text:
																				A__						  =				ord(__UpperCAmelCase  )
																				if cp == 0 or cp == 0XF_FFD or _is_control(__UpperCAmelCase  ):
																											continue
																				if _is_whitespace(__UpperCAmelCase  ):
																											output.append(' '  )
																				else:
																											output.append(__UpperCAmelCase  )
													return "".join(__UpperCAmelCase  )






def 	UpperCAmelCase       (						UpperCamelCase__					):






							"""simple docstring"""





							A__						  =				set()
							A__						  =				word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)					)
														A__						  =				char
							return pairs






def 	UpperCAmelCase       (						UpperCamelCase__					):






							"""simple docstring"""





							A__						  =				text.replace('—'							,     '-'					)
							A__						  =				text.replace('–'							,     '-'					)
							A__						  =				text.replace('―'							,     '-'					)
							A__						  =				text.replace('…'							,     '...'					)
							A__						  =				text.replace('´'							,     '\''					)
							A__						  =				re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'							,     r' \1 '							,     UpperCamelCase__					)
							A__						  =				re.sub(r'\s*\n\s*'							,     ' \n '							,     UpperCamelCase__					)
							A__						  =				re.sub(r'[^\S\n]+'							,     ' '							,     UpperCamelCase__					)
							return text.strip()


class      UpperCamelCase__(      __A			):
						lowerCAmelCase__	:    Optional[Any]    				=		VOCAB_FILES_NAMES
						lowerCAmelCase__	:    Union[str, Any]    				=		PRETRAINED_VOCAB_FILES_MAP
						lowerCAmelCase__	:    Optional[int]    				=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCAmelCase__	:    Dict    				=		['input_ids', 'attention_mask']





						def __init__(							self		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase="<unk>"		,**__UpperCAmelCase  )       ->					Tuple:
													super().__init__(unk_token=__UpperCAmelCase		,**__UpperCAmelCase  )

													try:
																				import ftfy
																				from spacy.lang.en import English

																				A__						  =				English()
																				A__						  =				_nlp.tokenizer
																				A__						  =				ftfy.fix_text
													except ImportError:
																				logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'  )
																				A__						  =				BasicTokenizer(do_lower_case=__UpperCAmelCase  )
																				A__						  =				None

													with open(__UpperCAmelCase		,encoding='utf-8'  ) as vocab_handle:
																				A__						  =				json.load(__UpperCAmelCase  )
													A__						  =				{v: k for k, v in self.encoder.items()}
													with open(__UpperCAmelCase		,encoding='utf-8'  ) as merges_handle:
																				A__						  =				merges_handle.read().split('\n'  )[1:-1]
													A__						  =				[tuple(merge.split()  ) for merge in merges]
													A__						  =				dict(zip(__UpperCAmelCase		,range(len(__UpperCAmelCase  )  )  )  )
													A__						  =				{}





						@property
						def 					snake_case__						(							self  )       ->					List[Any]:
													return True





						@property
						def 					snake_case__						(							self  )       ->					List[str]:
													return len(self.encoder  )





						def 					snake_case__						(							self  )       ->					Tuple:
													return dict(self.encoder		,**self.added_tokens_encoder  )





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					int:
													A__						  =				tuple(token[:-1]  ) + (token[-1] + '</w>',)
													if token in self.cache:
																				return self.cache[token]
													A__						  =				get_pairs(__UpperCAmelCase  )

													if not pairs:
																				return token + "</w>"

													while True:
																				A__						  =				min(__UpperCAmelCase		,key=lambda __UpperCAmelCase  : self.bpe_ranks.get(__UpperCAmelCase		,float('inf'  )  )  )
																				if bigram not in self.bpe_ranks:
																											break
																				A__  ,	A__						  =				bigram
																				A__						  =				[]
																				A__						  =				0
																				while i < len(__UpperCAmelCase  ):
																											try:
																																		A__						  =				word.index(__UpperCAmelCase		,__UpperCAmelCase  )
																											except ValueError:
																																		new_word.extend(word[i:]  )
																																		break
																											else:
																																		new_word.extend(word[i:j]  )
																																		A__						  =				j

																											if word[i] == first and i < len(__UpperCAmelCase  ) - 1 and word[i + 1] == second:
																																		new_word.append(first + second  )
																																		i += 2
																											else:
																																		new_word.append(word[i]  )
																																		i += 1
																				A__						  =				tuple(__UpperCAmelCase  )
																				A__						  =				new_word
																				if len(__UpperCAmelCase  ) == 1:
																											break
																				else:
																											A__						  =				get_pairs(__UpperCAmelCase  )
													A__						  =				' '.join(__UpperCAmelCase  )
													if word == "\n  </w>":
																				A__						  =				'\n</w>'
													A__						  =				word
													return word





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					int:
													A__						  =				[]
													if self.fix_text is None:
																				# Using BERT's BasicTokenizer
																				A__						  =				self.nlp.tokenize(__UpperCAmelCase  )
																				for token in text:
																											split_tokens.extend(list(self.bpe(__UpperCAmelCase  ).split(' '  )  )  )
													else:
																				# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
																				A__						  =				self.nlp(text_standardize(self.fix_text(__UpperCAmelCase  )  )  )
																				for token in text:
																											split_tokens.extend(list(self.bpe(token.text.lower()  ).split(' '  )  )  )
													return split_tokens





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					Optional[int]:
													return self.encoder.get(__UpperCAmelCase		,self.encoder.get(self.unk_token  )  )





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					List[Any]:
													return self.decoder.get(__UpperCAmelCase		,self.unk_token  )





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					Any:
													A__						  =				''.join(__UpperCAmelCase  ).replace('</w>'		,' '  ).strip()
													return out_string





						def 					snake_case__						(							self		,__UpperCAmelCase		,__UpperCAmelCase = None  )       ->					Tuple[str]:
													if not os.path.isdir(__UpperCAmelCase  ):
																				logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
																				return
													A__						  =				os.path.join(
													    __UpperCAmelCase		,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
													A__						  =				os.path.join(
													    __UpperCAmelCase		,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']  )

													with open(__UpperCAmelCase		,'w'		,encoding='utf-8'  ) as f:
																				f.write(json.dumps(self.encoder		,indent=2		,sort_keys=__UpperCAmelCase		,ensure_ascii=__UpperCAmelCase  ) + '\n'  )

													A__						  =				0
													with open(__UpperCAmelCase		,'w'		,encoding='utf-8'  ) as writer:
																				writer.write('#version: 0.2\n'  )
																				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,key=lambda __UpperCAmelCase  : kv[1]  ):
																											if index != token_index:
																																		logger.warning(
																																		    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																																		    ' Please check that the tokenizer is not corrupted!'  )
																																		A__						  =				token_index
																											writer.write(' '.join(__UpperCAmelCase  ) + '\n'  )
																											index += 1

													return vocab_file, merge_file






"""simple docstring"""




def 	UpperCAmelCase       (						UpperCamelCase__ = 100							,     UpperCamelCase__ = 100					):






							"""simple docstring"""





							return max(
							    sum(int(UpperCamelCase__					) for x in str(base**power					)					)
							    for base in range(UpperCamelCase__					)
							    for power in range(UpperCamelCase__					)					)


# Tests
if __name__ == "__main__":
				import doctest

				doctest.testmod()





