




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__A    :				Tuple    						=	logging.get_logger(__name__)

__A    :				Tuple    						=	{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

__A    :				List[Any]    						=	{
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

__A    :				Tuple    						=	{
    """openai-gpt""": 5_1_2,
}
def        lowerCamelCase_     (     SCREAMING_SNAKE_CASE ):
			'''simple docstring'''
			SCREAMING_SNAKE_CASE     =       text.strip()
			if not text:
						return []
			SCREAMING_SNAKE_CASE     =       text.split()
			return tokens




class       UpperCAmelCase_   ( A							):




							'''simple docstring'''



							def __init__(       self							:						List[Any]							,  a							:						List[str]=True							,  a							:						List[str]=None							,  a							:						Any=True							,  a							:						str=None							,  a							:						List[str]=True							,  )      ->       Optional[int]:
										if never_split is None:
													SCREAMING_SNAKE_CASE     =       []
										SCREAMING_SNAKE_CASE     =       do_lower_case
										SCREAMING_SNAKE_CASE     =       set(a      )
										SCREAMING_SNAKE_CASE     =       tokenize_chinese_chars
										SCREAMING_SNAKE_CASE     =       strip_accents
										SCREAMING_SNAKE_CASE     =       do_split_on_punc





							def  _UpperCAmelCase  (       self							:						Optional[int]							,  a							:						int							,  a							:						Union[str, Any]=None      )      ->       int:
										SCREAMING_SNAKE_CASE     =       self.never_split.union(set(a      )      ) if never_split else self.never_split
										SCREAMING_SNAKE_CASE     =       self._clean_text(a      )

										# This was added on November 1st, 2018 for the multilingual and Chinese
										# models. This is also applied to the English models now, but it doesn't
										# matter since the English models were not trained on any Chinese data
										# and generally don't have any Chinese data in them (there are Chinese
										# characters in the vocabulary because Wikipedia does have some Chinese
										# words in the English Wikipedia.).
										if self.tokenize_chinese_chars:
													SCREAMING_SNAKE_CASE     =       self._tokenize_chinese_chars(a      )
										# prevents treating the same character with different unicode codepoints as different characters
										SCREAMING_SNAKE_CASE     =       unicodedata.normalize("""NFC"""							,  a      )
										SCREAMING_SNAKE_CASE     =       whitespace_tokenize(a      )
										SCREAMING_SNAKE_CASE     =       []
										for token in orig_tokens:
													if token not in never_split:
																if self.do_lower_case:
																			SCREAMING_SNAKE_CASE     =       token.lower()
																			if self.strip_accents is not False:
																						SCREAMING_SNAKE_CASE     =       self._run_strip_accents(a      )
																elif self.strip_accents:
																			SCREAMING_SNAKE_CASE     =       self._run_strip_accents(a      )
													split_tokens.extend(self._run_split_on_punc(a							,  a      )      )

										SCREAMING_SNAKE_CASE     =       whitespace_tokenize(""" """.join(a      )      )
										return output_tokens





							def  _UpperCAmelCase  (       self							:						Tuple							,  a							:						int      )      ->       int:
										SCREAMING_SNAKE_CASE     =       unicodedata.normalize("""NFD"""							,  a      )
										SCREAMING_SNAKE_CASE     =       []
										for char in text:
													SCREAMING_SNAKE_CASE     =       unicodedata.category(a      )
													if cat == "Mn":
																continue
													output.append(a      )
										return "".join(a      )





							def  _UpperCAmelCase  (       self							:						List[Any]							,  a							:						Optional[int]							,  a							:						Union[str, Any]=None      )      ->       Any:
										if not self.do_split_on_punc or (never_split is not None and text in never_split):
													return [text]
										SCREAMING_SNAKE_CASE     =       list(a      )
										SCREAMING_SNAKE_CASE     =       0
										SCREAMING_SNAKE_CASE     =       True
										SCREAMING_SNAKE_CASE     =       []
										while i < len(a      ):
													SCREAMING_SNAKE_CASE     =       chars[i]
													if _is_punctuation(a      ):
																output.append([char]      )
																SCREAMING_SNAKE_CASE     =       True
													else:
																if start_new_word:
																			output.append([]      )
																SCREAMING_SNAKE_CASE     =       False
																output[-1].append(a      )
													i += 1

										return ["".join(a      ) for x in output]





							def  _UpperCAmelCase  (       self							:						Optional[int]							,  a							:						List[str]      )      ->       List[Any]:
										SCREAMING_SNAKE_CASE     =       []
										for char in text:
													SCREAMING_SNAKE_CASE     =       ord(a      )
													if self._is_chinese_char(a      ):
																output.append(""" """      )
																output.append(a      )
																output.append(""" """      )
													else:
																output.append(a      )
										return "".join(a      )





							def  _UpperCAmelCase  (       self							:						List[str]							,  a							:						Optional[int]      )      ->       Dict:
										# This defines a "chinese character" as anything in the CJK Unicode block:
										#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
										#
										# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
										# despite its name. The modern Korean Hangul alphabet is a different block,
										# as is Japanese Hiragana and Katakana. Those alphabets are used to write
										# space-separated words, so they are not treated specially and handled
										# like the all of the other languages.
										if (
										    (cp >= 0x4_E00 and cp <= 0x9_FFF)
										    or (cp >= 0x3_400 and cp <= 0x4_DBF)  #
										    or (cp >= 0x20_000 and cp <= 0x2A_6DF)  #
										    or (cp >= 0x2A_700 and cp <= 0x2B_73F)  #
										    or (cp >= 0x2B_740 and cp <= 0x2B_81F)  #
										    or (cp >= 0x2B_820 and cp <= 0x2C_EAF)  #
										    or (cp >= 0xF_900 and cp <= 0xF_AFF)
										    or (cp >= 0x2F_800 and cp <= 0x2F_A1F)  #
										):  #
													return True

										return False







							def  _UpperCAmelCase  (       self							:						str							,  a							:						List[Any]      )      ->       int:
										SCREAMING_SNAKE_CASE     =       []
										for char in text:
													SCREAMING_SNAKE_CASE     =       ord(a      )
													if cp == 0 or cp == 0xF_FFD or _is_control(a      ):
																continue
													if _is_whitespace(a      ):
																output.append(""" """      )
													else:
																output.append(a      )
										return "".join(a      )





def        lowerCamelCase_     (     SCREAMING_SNAKE_CASE ):
			'''simple docstring'''
			SCREAMING_SNAKE_CASE     =       set()
			SCREAMING_SNAKE_CASE     =       word[0]
			for char in word[1:]:
						pairs.add((prev_char, char) )
						SCREAMING_SNAKE_CASE     =       char
			return pairs
def        lowerCamelCase_     (     SCREAMING_SNAKE_CASE ):
			'''simple docstring'''
			SCREAMING_SNAKE_CASE     =       text.replace("""—"""       ,						"""-""" )
			SCREAMING_SNAKE_CASE     =       text.replace("""–"""       ,						"""-""" )
			SCREAMING_SNAKE_CASE     =       text.replace("""―"""       ,						"""-""" )
			SCREAMING_SNAKE_CASE     =       text.replace("""…"""       ,						"""...""" )
			SCREAMING_SNAKE_CASE     =       text.replace("""´"""       ,						"""'""" )
			SCREAMING_SNAKE_CASE     =       re.sub(r"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""       ,						r""" \1 """       ,						SCREAMING_SNAKE_CASE )
			SCREAMING_SNAKE_CASE     =       re.sub(r"""\s*\n\s*"""       ,						""" \n """       ,						SCREAMING_SNAKE_CASE )
			SCREAMING_SNAKE_CASE     =       re.sub(r"""[^\S\n]+"""       ,						""" """       ,						SCREAMING_SNAKE_CASE )
			return text.strip()




class       UpperCAmelCase_   ( A							):




							'''simple docstring'''



							a__    	=  VOCAB_FILES_NAMES
							a__    	=  PRETRAINED_VOCAB_FILES_MAP
							a__    	=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							a__    	=  ['''input_ids''', '''attention_mask''']





							def __init__(       self							:						Tuple							,  a							:						Union[str, Any]							,  a							:						List[Any]							,  a							:						Optional[Any]="<unk>"							,  **a							:						Optional[Any]      )      ->       int:
										super().__init__(unk_token=a							,  **a      )

										try:
													import ftfy
													from spacy.lang.en import English

													SCREAMING_SNAKE_CASE     =       English()
													SCREAMING_SNAKE_CASE     =       _nlp.tokenizer
													SCREAMING_SNAKE_CASE     =       ftfy.fix_text
										except ImportError:
													logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""      )
													SCREAMING_SNAKE_CASE     =       BasicTokenizer(do_lower_case=a      )
													SCREAMING_SNAKE_CASE     =       None

										with open(a							,  encoding="""utf-8"""      ) as vocab_handle:
													SCREAMING_SNAKE_CASE     =       json.load(a      )
										SCREAMING_SNAKE_CASE     =       {v: k for k, v in self.encoder.items()}
										with open(a							,  encoding="""utf-8"""      ) as merges_handle:
													SCREAMING_SNAKE_CASE     =       merges_handle.read().split("""\n"""      )[1:-1]
										SCREAMING_SNAKE_CASE     =       [tuple(merge.split()      ) for merge in merges]
										SCREAMING_SNAKE_CASE     =       dict(zip(a							,  range(len(a      )      )      )      )
										SCREAMING_SNAKE_CASE     =       {}





							@property
							def  _UpperCAmelCase  (       self							:						str      )      ->       Optional[int]:
										return True





							@property
							def  _UpperCAmelCase  (       self							:						Optional[int]      )      ->       Dict:
										return len(self.encoder      )





							def  _UpperCAmelCase  (       self							:						Optional[Any]      )      ->       Dict:
										return dict(self.encoder							,  **self.added_tokens_encoder      )





							def  _UpperCAmelCase  (       self							:						int							,  a							:						Optional[int]      )      ->       int:
										SCREAMING_SNAKE_CASE     =       tuple(token[:-1]      ) + (token[-1] + """</w>""",)
										if token in self.cache:
													return self.cache[token]
										SCREAMING_SNAKE_CASE     =       get_pairs(a      )

										if not pairs:
													return token + "</w>"

										while True:
													SCREAMING_SNAKE_CASE     =       min(a							,  key=lambda a      : self.bpe_ranks.get(a							,  float("""inf"""      )      )      )
													if bigram not in self.bpe_ranks:
																break
													SCREAMING_SNAKE_CASE  ,	SCREAMING_SNAKE_CASE     =       bigram
													SCREAMING_SNAKE_CASE     =       []
													SCREAMING_SNAKE_CASE     =       0
													while i < len(a      ):
																try:
																			SCREAMING_SNAKE_CASE     =       word.index(a							,  a      )
																except ValueError:
																			new_word.extend(word[i:]      )
																			break
																else:
																			new_word.extend(word[i:j]      )
																			SCREAMING_SNAKE_CASE     =       j

																if word[i] == first and i < len(a      ) - 1 and word[i + 1] == second:
																			new_word.append(first + second      )
																			i += 2
																else:
																			new_word.append(word[i]      )
																			i += 1
													SCREAMING_SNAKE_CASE     =       tuple(a      )
													SCREAMING_SNAKE_CASE     =       new_word
													if len(a      ) == 1:
																break
													else:
																SCREAMING_SNAKE_CASE     =       get_pairs(a      )
										SCREAMING_SNAKE_CASE     =       """ """.join(a      )
										if word == "\n  </w>":
													SCREAMING_SNAKE_CASE     =       """\n</w>"""
										SCREAMING_SNAKE_CASE     =       word
										return word





							def  _UpperCAmelCase  (       self							:						Tuple							,  a							:						Any      )      ->       Optional[Any]:
										SCREAMING_SNAKE_CASE     =       []
										if self.fix_text is None:
													# Using BERT's BasicTokenizer
													SCREAMING_SNAKE_CASE     =       self.nlp.tokenize(a      )
													for token in text:
																split_tokens.extend(list(self.bpe(a      ).split(""" """      )      )      )
										else:
													# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
													SCREAMING_SNAKE_CASE     =       self.nlp(text_standardize(self.fix_text(a      )      )      )
													for token in text:
																split_tokens.extend(list(self.bpe(token.text.lower()      ).split(""" """      )      )      )
										return split_tokens





							def  _UpperCAmelCase  (       self							:						Optional[Any]							,  a							:						Optional[int]      )      ->       int:
										return self.encoder.get(a							,  self.encoder.get(self.unk_token      )      )





							def  _UpperCAmelCase  (       self							:						Union[str, Any]							,  a							:						Dict      )      ->       Union[str, Any]:
										return self.decoder.get(a							,  self.unk_token      )





							def  _UpperCAmelCase  (       self							:						Union[str, Any]							,  a							:						List[str]      )      ->       str:
										SCREAMING_SNAKE_CASE     =       """""".join(a      ).replace("""</w>"""							,  """ """      ).strip()
										return out_string







							def  _UpperCAmelCase  (       self							:						str							,  a							:						str							,  a							:						Optional[str] = None      )      ->       Tuple[str]:
										if not os.path.isdir(a      ):
													logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""      )
													return
										SCREAMING_SNAKE_CASE     =       os.path.join(
										    a							,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )
										SCREAMING_SNAKE_CASE     =       os.path.join(
										    a							,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]      )

										with open(a							,  """w"""							,  encoding="""utf-8"""      ) as f:
													f.write(json.dumps(self.encoder							,  indent=2							,  sort_keys=a							,  ensure_ascii=a      ) + """\n"""      )

										SCREAMING_SNAKE_CASE     =       0
										with open(a							,  """w"""							,  encoding="""utf-8"""      ) as writer:
													writer.write("""#version: 0.2\n"""      )
													for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,  key=lambda a      : kv[1]      ):
																if index != token_index:
																			logger.warning(
																			    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																			    """ Please check that the tokenizer is not corrupted!"""      )
																			SCREAMING_SNAKE_CASE     =       token_index
																writer.write(""" """.join(a      ) + """\n"""      )
																index += 1

										return vocab_file, merge_file












from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A    :				List[str]    						=	logging.get_logger(__name__)

__A    :				List[Any]    						=	{
    """mnaylor/mega-base-wikitext""": """https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json""",
}




class       UpperCAmelCase_   ( A							):




							'''simple docstring'''



							a__    	=  '''mega'''





							def __init__(       self							:						Any							,  a							:						List[str]=30_522							,  a							:						Dict=128							,  a							:						Dict=4							,  a							:						Union[str, Any]=256							,  a							:						Tuple=16							,  a							:						Any=True							,  a							:						Dict=64							,  a							:						Union[str, Any]=False							,  a							:						Tuple=-1							,  a							:						str=None							,  a							:						int=True							,  a							:						Optional[int]="scalenorm"							,  a							:						Optional[int]=True							,  a							:						Dict="silu"							,  a							:						Optional[Any]="softmax"							,  a							:						Any=0.1							,  a							:						List[Any]=0.1							,  a							:						Tuple=0.1							,  a							:						Optional[int]=False							,  a							:						List[str]=True							,  a							:						Optional[int]=256							,  a							:						List[str]=True							,  a							:						int=0.1							,  a							:						str=2_048							,  a							:						Tuple=False							,  a							:						Optional[int]=2							,  a							:						Optional[int]=0.02							,  a							:						Optional[int]=0.2							,  a							:						Union[str, Any]=0.02							,  a							:						str=1.0							,  a							:						Dict=1							,  a							:						Union[str, Any]=0							,  a							:						Dict=2							,  a							:						List[str]="rotary"							,  a							:						Union[str, Any]=None							,  a							:						List[Any]=True							,  a							:						Tuple=True							,  **a							:						Optional[int]							,  )      ->       Dict:
										super().__init__(pad_token_id=a							,  bos_token_id=a							,  eos_token_id=a							,  **a      )

										SCREAMING_SNAKE_CASE     =       vocab_size
										SCREAMING_SNAKE_CASE     =       hidden_size
										SCREAMING_SNAKE_CASE     =       num_hidden_layers
										SCREAMING_SNAKE_CASE     =       activation
										SCREAMING_SNAKE_CASE     =       attention_activation
										SCREAMING_SNAKE_CASE     =       intermediate_size
										SCREAMING_SNAKE_CASE     =       ema_projection_size
										SCREAMING_SNAKE_CASE     =       bidirectional
										SCREAMING_SNAKE_CASE     =       shared_representation_size
										SCREAMING_SNAKE_CASE     =       use_chunking
										SCREAMING_SNAKE_CASE     =       chunk_size
										SCREAMING_SNAKE_CASE     =       truncation
										SCREAMING_SNAKE_CASE     =       normalize_before_mega
										SCREAMING_SNAKE_CASE     =       normalization_type
										SCREAMING_SNAKE_CASE     =       norm_affine
										SCREAMING_SNAKE_CASE     =       dropout_prob
										SCREAMING_SNAKE_CASE     =       hidden_dropout_prob
										SCREAMING_SNAKE_CASE     =       attention_probs_dropout_prob
										SCREAMING_SNAKE_CASE     =       use_feature_dropout
										SCREAMING_SNAKE_CASE     =       use_normalized_ffn
										SCREAMING_SNAKE_CASE     =       nffn_hidden_size
										SCREAMING_SNAKE_CASE     =       normalize_before_ffn
										SCREAMING_SNAKE_CASE     =       nffn_activation_dropout_prob
										SCREAMING_SNAKE_CASE     =       max_positions
										SCREAMING_SNAKE_CASE     =       add_token_type_embeddings
										SCREAMING_SNAKE_CASE     =       type_vocab_size
										SCREAMING_SNAKE_CASE     =       initializer_range
										SCREAMING_SNAKE_CASE     =       ema_delta_alpha_range
										SCREAMING_SNAKE_CASE     =       ema_beta_range
										SCREAMING_SNAKE_CASE     =       ema_gamma_omega_range
										SCREAMING_SNAKE_CASE     =       relative_positional_bias
										SCREAMING_SNAKE_CASE     =       use_cache
										SCREAMING_SNAKE_CASE     =       classifier_dropout
										SCREAMING_SNAKE_CASE     =       add_lm_hidden_dense_layer
										SCREAMING_SNAKE_CASE     =       1  # not used but required by Hugging Face





class       UpperCAmelCase_   ( A							):




							'''simple docstring'''



							@property
							def  _UpperCAmelCase  (       self							:						Union[str, Any]      )      ->       Mapping[str, Mapping[int, str]]:
										if self.task == "multiple-choice":
													SCREAMING_SNAKE_CASE     =       {0: """batch""", 1: """choice""", 2: """sequence"""}
										else:
													SCREAMING_SNAKE_CASE     =       {0: """batch""", 1: """sequence"""}
										return OrderedDict(
										    [
										        ("""input_ids""", dynamic_axis),
										        ("""attention_mask""", dynamic_axis),
										    ]      )






