



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 						lowerCamelCase__			(		_UpperCAmelCase					,       unittest.TestCase ):
						_lowerCAmelCase             =			ElectraTokenizer
						_lowerCAmelCase             =			ElectraTokenizerFast
						_lowerCAmelCase             =			True
						_lowerCAmelCase             =			True
						_lowerCAmelCase             =			filter_non_english


						def 	_lowerCamelCase      (			self    :	int      ):
													super().setUp()

													a__:							str									=[
													    "[UNK]",
													    "[CLS]",
													    "[SEP]",
													    "[PAD]",
													    "[MASK]",
													    "want",
													    "##want",
													    "##ed",
													    "wa",
													    "un",
													    "runn",
													    "##ing",
													    ",",
													    "low",
													    "lowest",
													]
													a__:							int									=os.path.join(self.tmpdirname   ,							VOCAB_FILES_NAMES["vocab_file"]      )
													with open(self.vocab_file   ,							"w"   ,							encoding="utf-8"      ) as vocab_writer:
																				vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )


						def 	_lowerCamelCase      (			self    :	Tuple   ,							_a    :	Dict      ):
													a__:							List[Any]									="UNwant\u00E9d,running"
													a__:							Tuple									="unwanted, running"
													return input_text, output_text


						def 	_lowerCamelCase      (			self    :	Union[str, Any]      ):
													a__:							List[Any]									=self.tokenizer_class(self.vocab_file      )

													a__:							Optional[Any]									=tokenizer.tokenize("UNwant\u00E9d,running"      )
													self.assertListEqual(_lowerCAmelCase   ,							["un", "##want", "##ed", ",", "runn", "##ing"]      )
													self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase      )   ,							[9, 6, 7, 1_2, 1_0, 1_1]      )


						def 	_lowerCamelCase      (			self    :	Optional[int]      ):





													if not self.test_rust_tokenizer:
																				return

													a__:							int									=self.get_tokenizer()
													a__:							Optional[Any]									=self.get_rust_tokenizer()

													a__:							Dict									="UNwant\u00E9d,running"

													a__:							Optional[Any]									=tokenizer.tokenize(_lowerCAmelCase      )
													a__:							List[str]									=rust_tokenizer.tokenize(_lowerCAmelCase      )
													self.assertListEqual(_lowerCAmelCase   ,							_lowerCAmelCase      )

													a__:							Tuple									=tokenizer.encode(_lowerCAmelCase   ,							add_special_tokens=_lowerCAmelCase      )
													a__:							Dict									=rust_tokenizer.encode(_lowerCAmelCase   ,							add_special_tokens=_lowerCAmelCase      )
													self.assertListEqual(_lowerCAmelCase   ,							_lowerCAmelCase      )

													a__:							str									=self.get_rust_tokenizer()
													a__:							List[str]									=tokenizer.encode(_lowerCAmelCase      )
													a__:							Any									=rust_tokenizer.encode(_lowerCAmelCase      )
													self.assertListEqual(_lowerCAmelCase   ,							_lowerCAmelCase      )

													# With lower casing
													a__:							Union[str, Any]									=self.get_tokenizer(do_lower_case=_lowerCAmelCase      )
													a__:							List[str]									=self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase      )

													a__:							Any									="UNwant\u00E9d,running"

													a__:							Tuple									=tokenizer.tokenize(_lowerCAmelCase      )
													a__:							Any									=rust_tokenizer.tokenize(_lowerCAmelCase      )
													self.assertListEqual(_lowerCAmelCase   ,							_lowerCAmelCase      )

													a__:							List[Any]									=tokenizer.encode(_lowerCAmelCase   ,							add_special_tokens=_lowerCAmelCase      )
													a__:							List[str]									=rust_tokenizer.encode(_lowerCAmelCase   ,							add_special_tokens=_lowerCAmelCase      )
													self.assertListEqual(_lowerCAmelCase   ,							_lowerCAmelCase      )

													a__:							str									=self.get_rust_tokenizer()
													a__:							str									=tokenizer.encode(_lowerCAmelCase      )
													a__:							Tuple									=rust_tokenizer.encode(_lowerCAmelCase      )
													self.assertListEqual(_lowerCAmelCase   ,							_lowerCAmelCase      )


						def 	_lowerCamelCase      (			self    :	str      ):
													a__:							int									=BasicTokenizer()

													self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"      )   ,							["ah", "\u535A", "\u63A8", "zz"]      )


						def 	_lowerCamelCase      (			self    :	Optional[int]      ):
													a__:							Tuple									=BasicTokenizer(do_lower_case=_lowerCAmelCase      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )   ,							["hello", "!", "how", "are", "you", "?"]      )
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )   ,							["hello"]      )


						def 	_lowerCamelCase      (			self    :	int      ):
													a__:							int									=BasicTokenizer(do_lower_case=_lowerCAmelCase   ,							strip_accents=_lowerCAmelCase      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )   ,							["hällo", "!", "how", "are", "you", "?"]      )
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )   ,							["h\u00E9llo"]      )


						def 	_lowerCamelCase      (			self    :	Any      ):
													a__:							List[str]									=BasicTokenizer(do_lower_case=_lowerCAmelCase   ,							strip_accents=_lowerCAmelCase      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )   ,							["hallo", "!", "how", "are", "you", "?"]      )
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )   ,							["hello"]      )


						def 	_lowerCamelCase      (			self    :	List[str]      ):
													a__:							Any									=BasicTokenizer(do_lower_case=_lowerCAmelCase      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )   ,							["hallo", "!", "how", "are", "you", "?"]      )
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )   ,							["hello"]      )


						def 	_lowerCamelCase      (			self    :	Union[str, Any]      ):
													a__:							Optional[int]									=BasicTokenizer(do_lower_case=_lowerCAmelCase      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )   ,							["HeLLo", "!", "how", "Are", "yoU", "?"]      )


						def 	_lowerCamelCase      (			self    :	int      ):
													a__:							str									=BasicTokenizer(do_lower_case=_lowerCAmelCase   ,							strip_accents=_lowerCAmelCase      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )   ,							["HäLLo", "!", "how", "Are", "yoU", "?"]      )


						def 	_lowerCamelCase      (			self    :	Dict      ):
													a__:							List[str]									=BasicTokenizer(do_lower_case=_lowerCAmelCase   ,							strip_accents=_lowerCAmelCase      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )   ,							["HaLLo", "!", "how", "Are", "yoU", "?"]      )


						def 	_lowerCamelCase      (			self    :	Tuple      ):
													a__:							str									=BasicTokenizer(do_lower_case=_lowerCAmelCase   ,							never_split=["[UNK]"]      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"      )   ,							["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]      )


						def 	_lowerCamelCase      (			self    :	Dict      ):
													a__:							Dict									=["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

													a__:							int									={}
													for i, token in enumerate(_lowerCAmelCase      ):
																				a__:							List[Any]									=i
													a__:							int									=WordpieceTokenizer(vocab=_lowerCAmelCase   ,							unk_token="[UNK]"      )

													self.assertListEqual(tokenizer.tokenize(""      )   ,							[]      )

													self.assertListEqual(tokenizer.tokenize("unwanted running"      )   ,							["un", "##want", "##ed", "runn", "##ing"]      )

													self.assertListEqual(tokenizer.tokenize("unwantedX running"      )   ,							["[UNK]", "runn", "##ing"]      )


						def 	_lowerCamelCase      (			self    :	List[str]      ):
													self.assertTrue(_is_whitespace(" "      )      )
													self.assertTrue(_is_whitespace("\t"      )      )
													self.assertTrue(_is_whitespace("\r"      )      )
													self.assertTrue(_is_whitespace("\n"      )      )
													self.assertTrue(_is_whitespace("\u00A0"      )      )

													self.assertFalse(_is_whitespace("A"      )      )
													self.assertFalse(_is_whitespace("-"      )      )


						def 	_lowerCamelCase      (			self    :	Optional[int]      ):
													self.assertTrue(_is_control("\u0005"      )      )

													self.assertFalse(_is_control("A"      )      )
													self.assertFalse(_is_control(" "      )      )
													self.assertFalse(_is_control("\t"      )      )
													self.assertFalse(_is_control("\r"      )      )


						def 	_lowerCamelCase      (			self    :	List[Any]      ):
													self.assertTrue(_is_punctuation("-"      )      )
													self.assertTrue(_is_punctuation("$"      )      )
													self.assertTrue(_is_punctuation("`"      )      )
													self.assertTrue(_is_punctuation("."      )      )

													self.assertFalse(_is_punctuation("A"      )      )
													self.assertFalse(_is_punctuation(" "      )      )


						def 	_lowerCamelCase      (			self    :	Optional[Any]      ):
													a__:							int									=self.get_tokenizer()
													a__:							str									=self.get_rust_tokenizer()

													# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
													self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase      ) for t in ["Test", "\xad", "test"]]   ,							[["[UNK]"], [], ["[UNK]"]]      )

													self.assertListEqual(
													    [rust_tokenizer.tokenize(_lowerCAmelCase      ) for t in ["Test", "\xad", "test"]]   ,							[["[UNK]"], [], ["[UNK]"]]      )


						@slow
						def 	_lowerCamelCase      (			self    :	str      ):
													a__:							List[str]									=self.tokenizer_class.from_pretrained("google/electra-base-discriminator"      )

													a__:							Dict									=tokenizer.encode("sequence builders"   ,							add_special_tokens=_lowerCAmelCase      )
													a__:							str									=tokenizer.encode("multi-sequence build"   ,							add_special_tokens=_lowerCAmelCase      )

													a__:							int									=tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase      )
													a__:							Optional[int]									=tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase   ,							_lowerCAmelCase      )

													assert encoded_sentence == [1_0_1] + text + [1_0_2]
													assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]


						def 	_lowerCamelCase      (			self    :	int      ):





													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
																											a__:							Dict									=self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase   ,							**_lowerCAmelCase      )

																											a__:							str									=F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																											a__:							Any									=tokenizer_r.encode_plus(
																											    _lowerCAmelCase   ,							return_attention_mask=_lowerCAmelCase   ,							return_token_type_ids=_lowerCAmelCase   ,							return_offsets_mapping=_lowerCAmelCase   ,							add_special_tokens=_lowerCAmelCase   ,							)

																											a__:							Optional[int]									=tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase   ,							"do_lower_case"      ) else False
																											a__:							str									=(
																											    [
																											        ((0, 0), tokenizer_r.cls_token),
																											        ((0, 1), "A"),
																											        ((1, 2), ","),
																											        ((3, 5), "na"),
																											        ((5, 6), "##ï"),
																											        ((6, 8), "##ve"),
																											        ((9, 1_5), tokenizer_r.mask_token),
																											        ((1_6, 2_1), "Allen"),
																											        ((2_1, 2_3), "##NL"),
																											        ((2_3, 2_4), "##P"),
																											        ((2_5, 3_3), "sentence"),
																											        ((3_3, 3_4), "."),
																											        ((0, 0), tokenizer_r.sep_token),
																											    ]
																											    if not do_lower_case
																											    else [
																											        ((0, 0), tokenizer_r.cls_token),
																											        ((0, 1), "a"),
																											        ((1, 2), ","),
																											        ((3, 8), "naive"),
																											        ((9, 1_5), tokenizer_r.mask_token),
																											        ((1_6, 2_1), "allen"),
																											        ((2_1, 2_3), "##nl"),
																											        ((2_3, 2_4), "##p"),
																											        ((2_5, 3_3), "sentence"),
																											        ((3_3, 3_4), "."),
																											        ((0, 0), tokenizer_r.sep_token),
																											    ]
																											)

																											self.assertEqual(
																											    [e[1] for e in expected_results]   ,							tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]      )      )
																											self.assertEqual([e[0] for e in expected_results]   ,							tokens["offset_mapping"]      )







						def 	_lowerCamelCase      (			self    :	Any      ):
													a__:							str									=["的", "人", "有"]
													a__:							Union[str, Any]									="".join(_lowerCAmelCase      )
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
																											a__:							str									=True
																											a__:							Optional[Any]									=self.tokenizer_class.from_pretrained(_lowerCAmelCase   ,							**_lowerCAmelCase      )
																											a__:							Tuple									=self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase   ,							**_lowerCAmelCase      )

																											a__:							Dict									=tokenizer_p.encode(_lowerCAmelCase   ,							add_special_tokens=_lowerCAmelCase      )
																											a__:							str									=tokenizer_r.encode(_lowerCAmelCase   ,							add_special_tokens=_lowerCAmelCase      )

																											a__:							int									=tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase      )
																											a__:							int									=tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase      )

																											# it is expected that each Chinese character is not preceded by "##"
																											self.assertListEqual(_lowerCAmelCase   ,							_lowerCAmelCase      )
																											self.assertListEqual(_lowerCAmelCase   ,							_lowerCAmelCase      )

																											a__:							str									=False
																											a__:							int									=self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase   ,							**_lowerCAmelCase      )
																											a__:							Union[str, Any]									=self.tokenizer_class.from_pretrained(_lowerCAmelCase   ,							**_lowerCAmelCase      )

																											a__:							Dict									=tokenizer_r.encode(_lowerCAmelCase   ,							add_special_tokens=_lowerCAmelCase      )
																											a__:							Tuple									=tokenizer_p.encode(_lowerCAmelCase   ,							add_special_tokens=_lowerCAmelCase      )

																											a__:							List[Any]									=tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase      )
																											a__:							Optional[Any]									=tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase      )

																											# it is expected that only the first Chinese character is not preceded by "##".
																											a__:							Union[str, Any]									=[
																											    F"##{token}" if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase      )
																											]
																											self.assertListEqual(_lowerCAmelCase   ,							_lowerCAmelCase      )
																											self.assertListEqual(_lowerCAmelCase   ,							_lowerCAmelCase      )









from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase			:					List[Any]         =							logging.get_logger(__name__)

UpperCamelCase			:					Tuple         =							{
    '''mnaylor/mega-base-wikitext''': '''https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json''',
}

class      __snake_case				(   _UpperCAmelCase):
					SCREAMING_SNAKE_CASE_		:       str								=			"mega"
					def __init__(						self       :						Optional[int]	,      _lowerCAmelCase       :						List[Any]=3_0_5_2_2	,      _lowerCAmelCase       :						List[str]=1_2_8	,      _lowerCAmelCase       :						Dict=4	,      _lowerCAmelCase       :						Optional[Any]=2_5_6	,      _lowerCAmelCase       :						Optional[Any]=1_6	,      _lowerCAmelCase       :						Dict=True	,      _lowerCAmelCase       :						Optional[Any]=6_4	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						Optional[int]=-1	,      _lowerCAmelCase       :						Any=None	,      _lowerCAmelCase       :						List[Any]=True	,      _lowerCAmelCase       :						List[Any]="scalenorm"	,      _lowerCAmelCase       :						str=True	,      _lowerCAmelCase       :						Union[str, Any]="silu"	,      _lowerCAmelCase       :						Optional[Any]="softmax"	,      _lowerCAmelCase       :						Optional[Any]=0.1	,      _lowerCAmelCase       :						Dict=0.1	,      _lowerCAmelCase       :						Any=0.1	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						Any=True	,      _lowerCAmelCase       :						int=2_5_6	,      _lowerCAmelCase       :						List[str]=True	,      _lowerCAmelCase       :						str=0.1	,      _lowerCAmelCase       :						List[str]=2_0_4_8	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Dict=0.02	,      _lowerCAmelCase       :						str=0.2	,      _lowerCAmelCase       :						List[str]=0.02	,      _lowerCAmelCase       :						int=1.0	,      _lowerCAmelCase       :						Optional[int]=1	,      _lowerCAmelCase       :						List[str]=0	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Optional[Any]="rotary"	,      _lowerCAmelCase       :						int=None	,      _lowerCAmelCase       :						int=True	,      _lowerCAmelCase       :						Optional[Any]=True	,      **_lowerCAmelCase       :						str	,      )							->					Any:



											'''simple docstring'''





											super().__init__(pad_token_id=_lowerCAmelCase	,      bos_token_id=_lowerCAmelCase	,      eos_token_id=_lowerCAmelCase	,      **_lowerCAmelCase  )

											__magic_name__					=      vocab_size
											__magic_name__					=      hidden_size
											__magic_name__					=      num_hidden_layers
											__magic_name__					=      activation
											__magic_name__					=      attention_activation
											__magic_name__					=      intermediate_size
											__magic_name__					=      ema_projection_size
											__magic_name__					=      bidirectional
											__magic_name__					=      shared_representation_size
											__magic_name__					=      use_chunking
											__magic_name__					=      chunk_size
											__magic_name__					=      truncation
											__magic_name__					=      normalize_before_mega
											__magic_name__					=      normalization_type
											__magic_name__					=      norm_affine
											__magic_name__					=      dropout_prob
											__magic_name__					=      hidden_dropout_prob
											__magic_name__					=      attention_probs_dropout_prob
											__magic_name__					=      use_feature_dropout
											__magic_name__					=      use_normalized_ffn
											__magic_name__					=      nffn_hidden_size
											__magic_name__					=      normalize_before_ffn
											__magic_name__					=      nffn_activation_dropout_prob
											__magic_name__					=      max_positions
											__magic_name__					=      add_token_type_embeddings
											__magic_name__					=      type_vocab_size
											__magic_name__					=      initializer_range
											__magic_name__					=      ema_delta_alpha_range
											__magic_name__					=      ema_beta_range
											__magic_name__					=      ema_gamma_omega_range
											__magic_name__					=      relative_positional_bias
											__magic_name__					=      use_cache
											__magic_name__					=      classifier_dropout
											__magic_name__					=      add_lm_hidden_dense_layer
											__magic_name__					=      1  # not used but required by Hugging Face







class      __snake_case				(   _UpperCAmelCase):
					@property
					def 	a			(						self       :						int  )							->					Mapping[str, Mapping[int, str]]:



											'''simple docstring'''





											if self.task == "multiple-choice":
																	__magic_name__					=      {0: 'batch', 1: 'choice', 2: 'sequence'}
											else:
																	__magic_name__					=      {0: 'batch', 1: 'sequence'}
											return OrderedDict(
											    [
											        ('input_ids', dynamic_axis),
											        ('attention_mask', dynamic_axis),
											    ]  )

