


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


UpperCamelCase			:					Union[str, Any]         =							logging.get_logger(__name__)

UpperCamelCase			:					List[Any]         =							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase			:					str         =							{
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

UpperCamelCase			:					Dict         =							{
    '''microsoft/mpnet-base''': 512,
}

UpperCamelCase			:					Dict         =							{
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}

class      __snake_case				(   _UpperCAmelCase):
					SCREAMING_SNAKE_CASE_		:       Any								=			VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE_		:       Optional[Any]								=			PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE_		:       str								=			PRETRAINED_INIT_CONFIGURATION
					SCREAMING_SNAKE_CASE_		:       Optional[Any]								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE_		:       Union[str, Any]								=			MPNetTokenizer
					SCREAMING_SNAKE_CASE_		:       Union[str, Any]								=			["input_ids", "attention_mask"]
					def __init__(						self       :						Tuple	,      _lowerCAmelCase       :						Tuple=None	,      _lowerCAmelCase       :						str=None	,      _lowerCAmelCase       :						str=True	,      _lowerCAmelCase       :						Optional[int]="<s>"	,      _lowerCAmelCase       :						List[Any]="</s>"	,      _lowerCAmelCase       :						Optional[Any]="</s>"	,      _lowerCAmelCase       :						Optional[Any]="<s>"	,      _lowerCAmelCase       :						str="[UNK]"	,      _lowerCAmelCase       :						List[Any]="<pad>"	,      _lowerCAmelCase       :						List[Any]="<mask>"	,      _lowerCAmelCase       :						Any=True	,      _lowerCAmelCase       :						Optional[Any]=None	,      **_lowerCAmelCase       :						Union[str, Any]	,      )							->					Optional[int]:



											'''simple docstring'''





											super().__init__(
											    _lowerCAmelCase	,      tokenizer_file=_lowerCAmelCase	,      do_lower_case=_lowerCAmelCase	,      bos_token=_lowerCAmelCase	,      eos_token=_lowerCAmelCase	,      sep_token=_lowerCAmelCase	,      cls_token=_lowerCAmelCase	,      unk_token=_lowerCAmelCase	,      pad_token=_lowerCAmelCase	,      mask_token=_lowerCAmelCase	,      tokenize_chinese_chars=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase	,      **_lowerCAmelCase	,      )

											__magic_name__					=      json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
											if (
											    pre_tok_state.get('lowercase'	,      _lowerCAmelCase  ) != do_lower_case
											    or pre_tok_state.get('strip_accents'	,      _lowerCAmelCase  ) != strip_accents
											):
																	__magic_name__					=      getattr(_lowerCAmelCase	,      pre_tok_state.pop('type'  )  )
																	__magic_name__					=      do_lower_case
																	__magic_name__					=      strip_accents
																	__magic_name__					=      pre_tok_class(**_lowerCAmelCase  )

											__magic_name__					=      do_lower_case
					@property
					def 	a			(						self       :						Dict  )							->					str:



											'''simple docstring'''





											if self._mask_token is None:
																	if self.verbose:
																							logger.error('Using mask_token, but it is not set yet.'  )
																	return None
											return str(self._mask_token  )
					@mask_token.setter
					def 	a			(						self       :						str	,      _lowerCAmelCase       :						List[str]  )							->					Optional[Any]:



											'''simple docstring'''





											__magic_name__					=      AddedToken(_lowerCAmelCase	,      lstrip=_lowerCAmelCase	,      rstrip=_lowerCAmelCase  ) if isinstance(_lowerCAmelCase	,      _lowerCAmelCase  ) else value
											__magic_name__					=      value
					def 	a			(						self       :						List[str]	,      _lowerCAmelCase       :						Tuple	,      _lowerCAmelCase       :						Dict=None  )							->					Union[str, Any]:



											'''simple docstring'''





											__magic_name__					=      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																	return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]
					def 	a			(						self       :						List[str]	,      _lowerCAmelCase       :						List[int]	,      _lowerCAmelCase       :						Optional[List[int]] = None  )							->					List[int]:



											'''simple docstring'''





											__magic_name__					=      [self.sep_token_id]
											__magic_name__					=      [self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep  ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]







					def 	a			(						self       :						Tuple	,      _lowerCAmelCase       :						str	,      _lowerCAmelCase       :						Optional[str] = None  )							->					Tuple[str]:



											'''simple docstring'''





											__magic_name__					=      self._tokenizer.model.save(_lowerCAmelCase	,      name=_lowerCAmelCase  )
											return tuple(_lowerCAmelCase  )





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase			:					List[Any]         =							logging.get_logger(__name__)

UpperCamelCase			:					Tuple         =							{
    '''mnaylor/mega-base-wikitext''': '''https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json''',
}

class      __snake_case				(   _UpperCAmelCase):
					SCREAMING_SNAKE_CASE_		:       str								=			"mega"
					def __init__(						self       :						Optional[int]	,      _lowerCAmelCase       :						List[Any]=3_0_5_2_2	,      _lowerCAmelCase       :						List[str]=1_2_8	,      _lowerCAmelCase       :						Dict=4	,      _lowerCAmelCase       :						Optional[Any]=2_5_6	,      _lowerCAmelCase       :						Optional[Any]=1_6	,      _lowerCAmelCase       :						Dict=True	,      _lowerCAmelCase       :						Optional[Any]=6_4	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						Optional[int]=-1	,      _lowerCAmelCase       :						Any=None	,      _lowerCAmelCase       :						List[Any]=True	,      _lowerCAmelCase       :						List[Any]="scalenorm"	,      _lowerCAmelCase       :						str=True	,      _lowerCAmelCase       :						Union[str, Any]="silu"	,      _lowerCAmelCase       :						Optional[Any]="softmax"	,      _lowerCAmelCase       :						Optional[Any]=0.1	,      _lowerCAmelCase       :						Dict=0.1	,      _lowerCAmelCase       :						Any=0.1	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						Any=True	,      _lowerCAmelCase       :						int=2_5_6	,      _lowerCAmelCase       :						List[str]=True	,      _lowerCAmelCase       :						str=0.1	,      _lowerCAmelCase       :						List[str]=2_0_4_8	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Dict=0.02	,      _lowerCAmelCase       :						str=0.2	,      _lowerCAmelCase       :						List[str]=0.02	,      _lowerCAmelCase       :						int=1.0	,      _lowerCAmelCase       :						Optional[int]=1	,      _lowerCAmelCase       :						List[str]=0	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Optional[Any]="rotary"	,      _lowerCAmelCase       :						int=None	,      _lowerCAmelCase       :						int=True	,      _lowerCAmelCase       :						Optional[Any]=True	,      **_lowerCAmelCase       :						str	,      )							->					Any:



											'''simple docstring'''





											super().__init__(pad_token_id=_lowerCAmelCase	,      bos_token_id=_lowerCAmelCase	,      eos_token_id=_lowerCAmelCase	,      **_lowerCAmelCase  )

											__magic_name__					=      vocab_size
											__magic_name__					=      hidden_size
											__magic_name__					=      num_hidden_layers
											__magic_name__					=      activation
											__magic_name__					=      attention_activation
											__magic_name__					=      intermediate_size
											__magic_name__					=      ema_projection_size
											__magic_name__					=      bidirectional
											__magic_name__					=      shared_representation_size
											__magic_name__					=      use_chunking
											__magic_name__					=      chunk_size
											__magic_name__					=      truncation
											__magic_name__					=      normalize_before_mega
											__magic_name__					=      normalization_type
											__magic_name__					=      norm_affine
											__magic_name__					=      dropout_prob
											__magic_name__					=      hidden_dropout_prob
											__magic_name__					=      attention_probs_dropout_prob
											__magic_name__					=      use_feature_dropout
											__magic_name__					=      use_normalized_ffn
											__magic_name__					=      nffn_hidden_size
											__magic_name__					=      normalize_before_ffn
											__magic_name__					=      nffn_activation_dropout_prob
											__magic_name__					=      max_positions
											__magic_name__					=      add_token_type_embeddings
											__magic_name__					=      type_vocab_size
											__magic_name__					=      initializer_range
											__magic_name__					=      ema_delta_alpha_range
											__magic_name__					=      ema_beta_range
											__magic_name__					=      ema_gamma_omega_range
											__magic_name__					=      relative_positional_bias
											__magic_name__					=      use_cache
											__magic_name__					=      classifier_dropout
											__magic_name__					=      add_lm_hidden_dense_layer
											__magic_name__					=      1  # not used but required by Hugging Face







class      __snake_case				(   _UpperCAmelCase):
					@property
					def 	a			(						self       :						int  )							->					Mapping[str, Mapping[int, str]]:



											'''simple docstring'''





											if self.task == "multiple-choice":
																	__magic_name__					=      {0: 'batch', 1: 'choice', 2: 'sequence'}
											else:
																	__magic_name__					=      {0: 'batch', 1: 'sequence'}
											return OrderedDict(
											    [
											        ('input_ids', dynamic_axis),
											        ('attention_mask', dynamic_axis),
											    ]  )

