



import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__UpperCAmelCase    	=  logging.get_logger(__name__)

__UpperCAmelCase    	=  {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__UpperCAmelCase    	=  {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

__UpperCAmelCase    	=  {
    '''openai-gpt''': 5_12,
}





def     __lowerCamelCase							(   __magic_name__						:	str    ):
							a__:							Union[str, Any]									=text.strip()
							if not text:
														return []
							a__:							Any									=text.split()
							return tokens


class 						lowerCamelCase__			(		_UpperCAmelCase ):


						def __init__(			self    :	Any   ,							_a    :	List[Any]=True   ,							_a    :	Optional[int]=None   ,							_a    :	int=True   ,							_a    :	List[Any]=None   ,							_a    :	Optional[int]=True   ,							):





													if never_split is None:
																				a__:							Optional[Any]									=[]
													a__:							Optional[Any]									=do_lower_case
													a__:							List[str]									=set(_lowerCAmelCase      )
													a__:							Any									=tokenize_chinese_chars
													a__:							List[Any]									=strip_accents
													a__:							int									=do_split_on_punc


						def 	_lowerCamelCase      (			self    :	Optional[Any]   ,							_a    :	Optional[int]   ,							_a    :	List[str]=None      ):
													a__:							Tuple									=self.never_split.union(set(_lowerCAmelCase      )      ) if never_split else self.never_split
													a__:							str									=self._clean_text(_lowerCAmelCase      )

													# This was added on November 1st, 2018 for the multilingual and Chinese
													# models. This is also applied to the English models now, but it doesn't
													# matter since the English models were not trained on any Chinese data
													# and generally don't have any Chinese data in them (there are Chinese
													# characters in the vocabulary because Wikipedia does have some Chinese
													# words in the English Wikipedia.).
													if self.tokenize_chinese_chars:
																				a__:							Tuple									=self._tokenize_chinese_chars(_lowerCAmelCase      )
													# prevents treating the same character with different unicode codepoints as different characters
													a__:							int									=unicodedata.normalize("NFC"   ,							_lowerCAmelCase      )
													a__:							Optional[int]									=whitespace_tokenize(_lowerCAmelCase      )
													a__:							List[Any]									=[]
													for token in orig_tokens:
																				if token not in never_split:
																											if self.do_lower_case:
																																		a__:							int									=token.lower()
																																		if self.strip_accents is not False:
																																									a__:							Optional[Any]									=self._run_strip_accents(_lowerCAmelCase      )
																											elif self.strip_accents:
																																		a__:							Optional[Any]									=self._run_strip_accents(_lowerCAmelCase      )
																				split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase   ,							_lowerCAmelCase      )      )

													a__:							List[Any]									=whitespace_tokenize(" ".join(_lowerCAmelCase      )      )
													return output_tokens


						def 	_lowerCamelCase      (			self    :	Optional[int]   ,							_a    :	Optional[int]      ):
													a__:							Union[str, Any]									=unicodedata.normalize("NFD"   ,							_lowerCAmelCase      )
													a__:							Dict									=[]
													for char in text:
																				a__:							Any									=unicodedata.category(_lowerCAmelCase      )
																				if cat == "Mn":
																											continue
																				output.append(_lowerCAmelCase      )
													return "".join(_lowerCAmelCase      )


						def 	_lowerCamelCase      (			self    :	Optional[Any]   ,							_a    :	Optional[int]   ,							_a    :	Optional[int]=None      ):





													if not self.do_split_on_punc or (never_split is not None and text in never_split):
																				return [text]
													a__:							List[str]									=list(_lowerCAmelCase      )
													a__:							str									=0
													a__:							Tuple									=True
													a__:							Tuple									=[]
													while i < len(_lowerCAmelCase      ):
																				a__:							Optional[int]									=chars[i]
																				if _is_punctuation(_lowerCAmelCase      ):
																											output.append([char]      )
																											a__:							List[str]									=True
																				else:
																											if start_new_word:
																																		output.append([]      )
																											a__:							Dict									=False
																											output[-1].append(_lowerCAmelCase      )
																				i += 1

													return ["".join(_lowerCAmelCase      ) for x in output]


						def 	_lowerCamelCase      (			self    :	str   ,							_a    :	Dict      ):
													a__:							Any									=[]
													for char in text:
																				a__:							Any									=ord(_lowerCAmelCase      )
																				if self._is_chinese_char(_lowerCAmelCase      ):
																											output.append(" "      )
																											output.append(_lowerCAmelCase      )
																											output.append(" "      )
																				else:
																											output.append(_lowerCAmelCase      )
													return "".join(_lowerCAmelCase      )


						def 	_lowerCamelCase      (			self    :	Any   ,							_a    :	List[str]      ):





													if (
													    (cp >= 0x4_e00 and cp <= 0x9_fff)
													    or (cp >= 0x3_400 and cp <= 0x4_dbf)  #
													    or (cp >= 0x20_000 and cp <= 0x2a_6df)  #
													    or (cp >= 0x2a_700 and cp <= 0x2b_73f)  #
													    or (cp >= 0x2b_740 and cp <= 0x2b_81f)  #
													    or (cp >= 0x2b_820 and cp <= 0x2c_eaf)  #
													    or (cp >= 0xf_900 and cp <= 0xf_aff)
													    or (cp >= 0x2f_800 and cp <= 0x2f_a1f)  #
													):  #
																				return True

													return False







						def 	_lowerCamelCase      (			self    :	Union[str, Any]   ,							_a    :	Optional[int]      ):
													a__:							Any									=[]
													for char in text:
																				a__:							str									=ord(_lowerCAmelCase      )
																				if cp == 0 or cp == 0xf_ffd or _is_control(_lowerCAmelCase      ):
																											continue
																				if _is_whitespace(_lowerCAmelCase      ):
																											output.append(" "      )
																				else:
																											output.append(_lowerCAmelCase      )
													return "".join(_lowerCAmelCase      )





def     __lowerCamelCase							(   __magic_name__						:	Dict    ):
							a__:							str									=set()
							a__:							List[Any]									=word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)    )
														a__:							int									=char
							return pairs





def     __lowerCamelCase							(   __magic_name__						:	List[Any]    ):
							a__:							Optional[Any]									=text.replace("—"  ,			"-"    )
							a__:							Optional[int]									=text.replace("–"  ,			"-"    )
							a__:							Union[str, Any]									=text.replace("―"  ,			"-"    )
							a__:							Optional[Any]									=text.replace("…"  ,			"..."    )
							a__:							Optional[int]									=text.replace("´"  ,			"\'"    )
							a__:							List[Any]									=re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"  ,			R" \1 "  ,			__snake_case    )
							a__:							Dict									=re.sub(R"\s*\n\s*"  ,			" \n "  ,			__snake_case    )
							a__:							str									=re.sub(R"[^\S\n]+"  ,			" "  ,			__snake_case    )
							return text.strip()







class 						lowerCamelCase__			(		_UpperCAmelCase ):
						_lowerCAmelCase             =			VOCAB_FILES_NAMES
						_lowerCAmelCase             =			PRETRAINED_VOCAB_FILES_MAP
						_lowerCAmelCase             =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_lowerCAmelCase             =			["input_ids", "attention_mask"]


						def __init__(			self    :	Tuple   ,							_a    :	str   ,							_a    :	List[str]   ,							_a    :	List[Any]="<unk>"   ,							**_a    :	Optional[Any]      ):
													super().__init__(unk_token=_lowerCAmelCase   ,							**_lowerCAmelCase      )

													try:
																				import ftfy
																				from spacy.lang.en import English

																				a__:							List[Any]									=English()
																				a__:							Tuple									=_nlp.tokenizer
																				a__:							Optional[int]									=ftfy.fix_text
													except ImportError:
																				logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."      )
																				a__:							Any									=BasicTokenizer(do_lower_case=_lowerCAmelCase      )
																				a__:							Tuple									=None

													with open(_lowerCAmelCase   ,							encoding="utf-8"      ) as vocab_handle:
																				a__:							List[str]									=json.load(_lowerCAmelCase      )
													a__:							List[str]									={v: k for k, v in self.encoder.items()}
													with open(_lowerCAmelCase   ,							encoding="utf-8"      ) as merges_handle:
																				a__:							List[Any]									=merges_handle.read().split("\n"      )[1:-1]
													a__:							str									=[tuple(merge.split()      ) for merge in merges]
													a__:							Tuple									=dict(zip(_lowerCAmelCase   ,							range(len(_lowerCAmelCase      )      )      )      )
													a__:							Dict									={}


						@property
						def 	_lowerCamelCase      (			self    :	Tuple      ):
													return True


						@property
						def 	_lowerCamelCase      (			self    :	int      ):
													return len(self.encoder      )


						def 	_lowerCamelCase      (			self    :	List[str]      ):
													return dict(self.encoder   ,							**self.added_tokens_encoder      )


						def 	_lowerCamelCase      (			self    :	int   ,							_a    :	Tuple      ):
													a__:							Optional[int]									=tuple(token[:-1]      ) + (token[-1] + "</w>",)
													if token in self.cache:
																				return self.cache[token]
													a__:							str									=get_pairs(_lowerCAmelCase      )

													if not pairs:
																				return token + "</w>"

													while True:
																				a__:							int									=min(_lowerCAmelCase   ,							key=lambda _a      : self.bpe_ranks.get(_lowerCAmelCase   ,							float("inf"      )      )      )
																				if bigram not in self.bpe_ranks:
																											break
																				a__				,      a__:							List[Any]									=bigram
																				a__:							int									=[]
																				a__:							List[Any]									=0
																				while i < len(_lowerCAmelCase      ):
																											try:
																																		a__:							Tuple									=word.index(_lowerCAmelCase   ,							_lowerCAmelCase      )
																											except ValueError:
																																		new_word.extend(word[i:]      )
																																		break
																											else:
																																		new_word.extend(word[i:j]      )
																																		a__:							List[str]									=j

																											if word[i] == first and i < len(_lowerCAmelCase      ) - 1 and word[i + 1] == second:
																																		new_word.append(first + second      )
																																		i += 2
																											else:
																																		new_word.append(word[i]      )
																																		i += 1
																				a__:							Any									=tuple(_lowerCAmelCase      )
																				a__:							List[Any]									=new_word
																				if len(_lowerCAmelCase      ) == 1:
																											break
																				else:
																											a__:							Optional[Any]									=get_pairs(_lowerCAmelCase      )
													a__:							Optional[Any]									=" ".join(_lowerCAmelCase      )
													if word == "\n  </w>":
																				a__:							Union[str, Any]									="\n</w>"
													a__:							Tuple									=word
													return word


						def 	_lowerCamelCase      (			self    :	Optional[Any]   ,							_a    :	List[Any]      ):
													a__:							Optional[Any]									=[]
													if self.fix_text is None:
																				# Using BERT's BasicTokenizer
																				a__:							Dict									=self.nlp.tokenize(_lowerCAmelCase      )
																				for token in text:
																											split_tokens.extend(list(self.bpe(_lowerCAmelCase      ).split(" "      )      )      )
													else:
																				# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
																				a__:							Optional[int]									=self.nlp(text_standardize(self.fix_text(_lowerCAmelCase      )      )      )
																				for token in text:
																											split_tokens.extend(list(self.bpe(token.text.lower()      ).split(" "      )      )      )
													return split_tokens


						def 	_lowerCamelCase      (			self    :	int   ,							_a    :	Dict      ):
													return self.encoder.get(_lowerCAmelCase   ,							self.encoder.get(self.unk_token      )      )


						def 	_lowerCamelCase      (			self    :	Optional[int]   ,							_a    :	List[str]      ):
													return self.decoder.get(_lowerCAmelCase   ,							self.unk_token      )


						def 	_lowerCamelCase      (			self    :	Optional[Any]   ,							_a    :	Any      ):
													a__:							List[Any]									="".join(_lowerCAmelCase      ).replace("</w>"   ,							" "      ).strip()
													return out_string







						def 	_lowerCamelCase      (			self    :	List[Any]   ,							_a    :	str   ,							_a    :	Optional[str] = None      ):





													if not os.path.isdir(_lowerCAmelCase      ):
																				logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
																				return
													a__:							List[str]									=os.path.join(
													    _lowerCAmelCase   ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )
													a__:							Union[str, Any]									=os.path.join(
													    _lowerCAmelCase   ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]      )

													with open(_lowerCAmelCase   ,							"w"   ,							encoding="utf-8"      ) as f:
																				f.write(json.dumps(self.encoder   ,							indent=2   ,							sort_keys=_lowerCAmelCase   ,							ensure_ascii=_lowerCAmelCase      ) + "\n"      )

													a__:							Tuple									=0
													with open(_lowerCAmelCase   ,							"w"   ,							encoding="utf-8"      ) as writer:
																				writer.write("#version: 0.2\n"      )
																				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,							key=lambda _a      : kv[1]      ):
																											if index != token_index:
																																		logger.warning(
																																		    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																																		    " Please check that the tokenizer is not corrupted!"      )
																																		a__:							Optional[Any]									=token_index
																											writer.write(" ".join(_lowerCAmelCase      ) + "\n"      )
																											index += 1

													return vocab_file, merge_file









from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase			:					List[Any]         =							logging.get_logger(__name__)

UpperCamelCase			:					Tuple         =							{
    '''mnaylor/mega-base-wikitext''': '''https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json''',
}

class      __snake_case				(   _UpperCAmelCase):
					SCREAMING_SNAKE_CASE_		:       str								=			"mega"
					def __init__(						self       :						Optional[int]	,      _lowerCAmelCase       :						List[Any]=3_0_5_2_2	,      _lowerCAmelCase       :						List[str]=1_2_8	,      _lowerCAmelCase       :						Dict=4	,      _lowerCAmelCase       :						Optional[Any]=2_5_6	,      _lowerCAmelCase       :						Optional[Any]=1_6	,      _lowerCAmelCase       :						Dict=True	,      _lowerCAmelCase       :						Optional[Any]=6_4	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						Optional[int]=-1	,      _lowerCAmelCase       :						Any=None	,      _lowerCAmelCase       :						List[Any]=True	,      _lowerCAmelCase       :						List[Any]="scalenorm"	,      _lowerCAmelCase       :						str=True	,      _lowerCAmelCase       :						Union[str, Any]="silu"	,      _lowerCAmelCase       :						Optional[Any]="softmax"	,      _lowerCAmelCase       :						Optional[Any]=0.1	,      _lowerCAmelCase       :						Dict=0.1	,      _lowerCAmelCase       :						Any=0.1	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						Any=True	,      _lowerCAmelCase       :						int=2_5_6	,      _lowerCAmelCase       :						List[str]=True	,      _lowerCAmelCase       :						str=0.1	,      _lowerCAmelCase       :						List[str]=2_0_4_8	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Dict=0.02	,      _lowerCAmelCase       :						str=0.2	,      _lowerCAmelCase       :						List[str]=0.02	,      _lowerCAmelCase       :						int=1.0	,      _lowerCAmelCase       :						Optional[int]=1	,      _lowerCAmelCase       :						List[str]=0	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Optional[Any]="rotary"	,      _lowerCAmelCase       :						int=None	,      _lowerCAmelCase       :						int=True	,      _lowerCAmelCase       :						Optional[Any]=True	,      **_lowerCAmelCase       :						str	,      )							->					Any:



											'''simple docstring'''





											super().__init__(pad_token_id=_lowerCAmelCase	,      bos_token_id=_lowerCAmelCase	,      eos_token_id=_lowerCAmelCase	,      **_lowerCAmelCase  )

											__magic_name__					=      vocab_size
											__magic_name__					=      hidden_size
											__magic_name__					=      num_hidden_layers
											__magic_name__					=      activation
											__magic_name__					=      attention_activation
											__magic_name__					=      intermediate_size
											__magic_name__					=      ema_projection_size
											__magic_name__					=      bidirectional
											__magic_name__					=      shared_representation_size
											__magic_name__					=      use_chunking
											__magic_name__					=      chunk_size
											__magic_name__					=      truncation
											__magic_name__					=      normalize_before_mega
											__magic_name__					=      normalization_type
											__magic_name__					=      norm_affine
											__magic_name__					=      dropout_prob
											__magic_name__					=      hidden_dropout_prob
											__magic_name__					=      attention_probs_dropout_prob
											__magic_name__					=      use_feature_dropout
											__magic_name__					=      use_normalized_ffn
											__magic_name__					=      nffn_hidden_size
											__magic_name__					=      normalize_before_ffn
											__magic_name__					=      nffn_activation_dropout_prob
											__magic_name__					=      max_positions
											__magic_name__					=      add_token_type_embeddings
											__magic_name__					=      type_vocab_size
											__magic_name__					=      initializer_range
											__magic_name__					=      ema_delta_alpha_range
											__magic_name__					=      ema_beta_range
											__magic_name__					=      ema_gamma_omega_range
											__magic_name__					=      relative_positional_bias
											__magic_name__					=      use_cache
											__magic_name__					=      classifier_dropout
											__magic_name__					=      add_lm_hidden_dense_layer
											__magic_name__					=      1  # not used but required by Hugging Face







class      __snake_case				(   _UpperCAmelCase):
					@property
					def 	a			(						self       :						int  )							->					Mapping[str, Mapping[int, str]]:



											'''simple docstring'''





											if self.task == "multiple-choice":
																	__magic_name__					=      {0: 'batch', 1: 'choice', 2: 'sequence'}
											else:
																	__magic_name__					=      {0: 'batch', 1: 'sequence'}
											return OrderedDict(
											    [
											        ('input_ids', dynamic_axis),
											        ('attention_mask', dynamic_axis),
											    ]  )

