

'''simple docstring'''







import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 					__a   (		_snake_case							,unittest.TestCase  ):
						__SCREAMING_SNAKE_CASE     :					Optional[int]  =      ElectraTokenizer
						__SCREAMING_SNAKE_CASE     :					Union[str, Any]  =      ElectraTokenizerFast
						__SCREAMING_SNAKE_CASE     :					str  =      True
						__SCREAMING_SNAKE_CASE     :					List[str]  =      True
						__SCREAMING_SNAKE_CASE     :					Tuple  =      filter_non_english

						def 	_UpperCAmelCase							(		self	:						Union[str, Any])						->Any:

							"""simple docstring"""



							super().setUp()

							_lowercase							=     [
							    """[UNK]""",
							    """[CLS]""",
							    """[SEP]""",
							    """[PAD]""",
							    """[MASK]""",
							    """want""",
							    """##want""",
							    """##ed""",
							    """wa""",
							    """un""",
							    """runn""",
							    """##ing""",
							    """,""",
							    """low""",
							    """lowest""",
							]
							_lowercase							=     os.path.join(self.tmpdirname    ,			VOCAB_FILES_NAMES["""vocab_file"""])
							with open(self.vocab_file    ,			"""w"""    ,			encoding="""utf-8""") as vocab_writer:
								vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))

						def 	_UpperCAmelCase							(		self	:						int    ,			lowercase__	:						int)						->int:

							"""simple docstring"""



							_lowercase							=     """UNwant\u00E9d,running"""
							_lowercase							=     """unwanted, running"""
							return input_text, output_text

						def 	_UpperCAmelCase							(		self	:						int)						->Optional[int]:

							"""simple docstring"""



							_lowercase							=     self.tokenizer_class(self.vocab_file)

							_lowercase							=     tokenizer.tokenize("""UNwant\u00E9d,running""")
							self.assertListEqual(lowercase__    ,			["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""])
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__)    ,			[9, 6, 7, 12, 10, 11])

						def 	_UpperCAmelCase							(		self	:						List[str])						->List[str]:

							"""simple docstring"""



							if not self.test_rust_tokenizer:
								return

							_lowercase							=     self.get_tokenizer()
							_lowercase							=     self.get_rust_tokenizer()

							_lowercase							=     """UNwant\u00E9d,running"""

							_lowercase							=     tokenizer.tokenize(lowercase__)
							_lowercase							=     rust_tokenizer.tokenize(lowercase__)
							self.assertListEqual(lowercase__    ,			lowercase__)

							_lowercase							=     tokenizer.encode(lowercase__    ,			add_special_tokens=lowercase__)
							_lowercase							=     rust_tokenizer.encode(lowercase__    ,			add_special_tokens=lowercase__)
							self.assertListEqual(lowercase__    ,			lowercase__)

							_lowercase							=     self.get_rust_tokenizer()
							_lowercase							=     tokenizer.encode(lowercase__)
							_lowercase							=     rust_tokenizer.encode(lowercase__)
							self.assertListEqual(lowercase__    ,			lowercase__)

							# With lower casing
							_lowercase							=     self.get_tokenizer(do_lower_case=lowercase__)
							_lowercase							=     self.get_rust_tokenizer(do_lower_case=lowercase__)

							_lowercase							=     """UNwant\u00E9d,running"""

							_lowercase							=     tokenizer.tokenize(lowercase__)
							_lowercase							=     rust_tokenizer.tokenize(lowercase__)
							self.assertListEqual(lowercase__    ,			lowercase__)

							_lowercase							=     tokenizer.encode(lowercase__    ,			add_special_tokens=lowercase__)
							_lowercase							=     rust_tokenizer.encode(lowercase__    ,			add_special_tokens=lowercase__)
							self.assertListEqual(lowercase__    ,			lowercase__)

							_lowercase							=     self.get_rust_tokenizer()
							_lowercase							=     tokenizer.encode(lowercase__)
							_lowercase							=     rust_tokenizer.encode(lowercase__)
							self.assertListEqual(lowercase__    ,			lowercase__)

						def 	_UpperCAmelCase							(		self	:						Tuple)						->Dict:

							"""simple docstring"""



							_lowercase							=     BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""")    ,			["""ah""", """\u535A""", """\u63A8""", """zz"""])

						def 	_UpperCAmelCase							(		self	:						Any)						->Union[str, Any]:

							"""simple docstring"""



							_lowercase							=     BasicTokenizer(do_lower_case=lowercase__)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)    ,			["""hello""", """!""", """how""", """are""", """you""", """?"""])
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")    ,			["""hello"""])

						def 	_UpperCAmelCase							(		self	:						Optional[int])						->Tuple:

							"""simple docstring"""



							_lowercase							=     BasicTokenizer(do_lower_case=lowercase__    ,			strip_accents=lowercase__)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)    ,			["""hällo""", """!""", """how""", """are""", """you""", """?"""])
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")    ,			["""h\u00E9llo"""])

						def 	_UpperCAmelCase							(		self	:						List[str])						->Optional[Any]:

							"""simple docstring"""



							_lowercase							=     BasicTokenizer(do_lower_case=lowercase__    ,			strip_accents=lowercase__)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)    ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""])
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")    ,			["""hello"""])

						def 	_UpperCAmelCase							(		self	:						Tuple)						->Optional[int]:

							"""simple docstring"""



							_lowercase							=     BasicTokenizer(do_lower_case=lowercase__)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)    ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""])
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")    ,			["""hello"""])

						def 	_UpperCAmelCase							(		self	:						str)						->Optional[int]:

							"""simple docstring"""



							_lowercase							=     BasicTokenizer(do_lower_case=lowercase__)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)    ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""])

						def 	_UpperCAmelCase							(		self	:						int)						->Union[str, Any]:

							"""simple docstring"""



							_lowercase							=     BasicTokenizer(do_lower_case=lowercase__    ,			strip_accents=lowercase__)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)    ,			["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""])

						def 	_UpperCAmelCase							(		self	:						Any)						->Any:

							"""simple docstring"""



							_lowercase							=     BasicTokenizer(do_lower_case=lowercase__    ,			strip_accents=lowercase__)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)    ,			["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""])

						def 	_UpperCAmelCase							(		self	:						Optional[Any])						->Tuple:

							"""simple docstring"""



							_lowercase							=     BasicTokenizer(do_lower_case=lowercase__    ,			never_split=["""[UNK]"""])

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""")    ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""])

						def 	_UpperCAmelCase							(		self	:						List[Any])						->Tuple:

							"""simple docstring"""



							_lowercase							=     ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

							_lowercase							=     {}
							for i, token in enumerate(lowercase__):
								_lowercase							=     i
							_lowercase							=     WordpieceTokenizer(vocab=lowercase__    ,			unk_token="""[UNK]""")

							self.assertListEqual(tokenizer.tokenize("""""")    ,			[])

							self.assertListEqual(tokenizer.tokenize("""unwanted running""")    ,			["""un""", """##want""", """##ed""", """runn""", """##ing"""])

							self.assertListEqual(tokenizer.tokenize("""unwantedX running""")    ,			["""[UNK]""", """runn""", """##ing"""])

						def 	_UpperCAmelCase							(		self	:						Dict)						->str:

							"""simple docstring"""



							self.assertTrue(_is_whitespace(""" """))
							self.assertTrue(_is_whitespace("""\t"""))
							self.assertTrue(_is_whitespace("""\r"""))
							self.assertTrue(_is_whitespace("""\n"""))
							self.assertTrue(_is_whitespace("""\u00A0"""))

							self.assertFalse(_is_whitespace("""A"""))
							self.assertFalse(_is_whitespace("""-"""))

						def 	_UpperCAmelCase							(		self	:						Optional[Any])						->Dict:

							"""simple docstring"""



							self.assertTrue(_is_control("""\u0005"""))

							self.assertFalse(_is_control("""A"""))
							self.assertFalse(_is_control(""" """))
							self.assertFalse(_is_control("""\t"""))
							self.assertFalse(_is_control("""\r"""))

						def 	_UpperCAmelCase							(		self	:						Optional[Any])						->Optional[Any]:

							"""simple docstring"""



							self.assertTrue(_is_punctuation("""-"""))
							self.assertTrue(_is_punctuation("""$"""))
							self.assertTrue(_is_punctuation("""`"""))
							self.assertTrue(_is_punctuation("""."""))

							self.assertFalse(_is_punctuation("""A"""))
							self.assertFalse(_is_punctuation(""" """))

						def 	_UpperCAmelCase							(		self	:						Tuple)						->Optional[Any]:

							"""simple docstring"""



							_lowercase							=     self.get_tokenizer()
							_lowercase							=     self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(lowercase__) for t in ["""Test""", """\xad""", """test"""]]    ,			[["""[UNK]"""], [], ["""[UNK]"""]])

							self.assertListEqual(
							    [rust_tokenizer.tokenize(lowercase__) for t in ["""Test""", """\xad""", """test"""]]    ,			[["""[UNK]"""], [], ["""[UNK]"""]])

						@slow
						def 	_UpperCAmelCase							(		self	:						List[str])						->Tuple:

							"""simple docstring"""



							_lowercase							=     self.tokenizer_class.from_pretrained("""google/electra-base-discriminator""")

							_lowercase							=     tokenizer.encode("""sequence builders"""    ,			add_special_tokens=lowercase__)
							_lowercase							=     tokenizer.encode("""multi-sequence build"""    ,			add_special_tokens=lowercase__)

							_lowercase							=     tokenizer.build_inputs_with_special_tokens(lowercase__)
							_lowercase							=     tokenizer.build_inputs_with_special_tokens(lowercase__    ,			lowercase__)

							assert encoded_sentence == [1_01] + text + [1_02]
							assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]

						def 	_UpperCAmelCase							(		self	:						int)						->Union[str, Any]:

							"""simple docstring"""



							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
									_lowercase							=     self.rust_tokenizer_class.from_pretrained(lowercase__    ,			**lowercase__)

									_lowercase							=     f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
									_lowercase							=     tokenizer_r.encode_plus(
									    lowercase__    ,			return_attention_mask=lowercase__    ,			return_token_type_ids=lowercase__    ,			return_offsets_mapping=lowercase__    ,			add_special_tokens=lowercase__    ,			)

									_lowercase							=     tokenizer_r.do_lower_case if hasattr(lowercase__    ,			"""do_lower_case""") else False
									_lowercase							=     (
									    [
									        ((0, 0), tokenizer_r.cls_token),
									        ((0, 1), """A"""),
									        ((1, 2), ""","""),
									        ((3, 5), """na"""),
									        ((5, 6), """##ï"""),
									        ((6, 8), """##ve"""),
									        ((9, 15), tokenizer_r.mask_token),
									        ((16, 21), """Allen"""),
									        ((21, 23), """##NL"""),
									        ((23, 24), """##P"""),
									        ((25, 33), """sentence"""),
									        ((33, 34), """."""),
									        ((0, 0), tokenizer_r.sep_token),
									    ]
									    if not do_lower_case
									    else [
									        ((0, 0), tokenizer_r.cls_token),
									        ((0, 1), """a"""),
									        ((1, 2), ""","""),
									        ((3, 8), """naive"""),
									        ((9, 15), tokenizer_r.mask_token),
									        ((16, 21), """allen"""),
									        ((21, 23), """##nl"""),
									        ((23, 24), """##p"""),
									        ((25, 33), """sentence"""),
									        ((33, 34), """."""),
									        ((0, 0), tokenizer_r.sep_token),
									    ]
									)

									self.assertEqual(
									    [e[1] for e in expected_results]    ,			tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]))
									self.assertEqual([e[0] for e in expected_results]    ,			tokens["""offset_mapping"""])


						def 	_UpperCAmelCase							(		self	:						Any)						->Tuple:

							"""simple docstring"""



							_lowercase							=     ["""的""", """人""", """有"""]
							_lowercase							=     """""".join(lowercase__)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
									_lowercase							=     True
									_lowercase							=     self.tokenizer_class.from_pretrained(lowercase__    ,			**lowercase__)
									_lowercase							=     self.rust_tokenizer_class.from_pretrained(lowercase__    ,			**lowercase__)

									_lowercase							=     tokenizer_p.encode(lowercase__    ,			add_special_tokens=lowercase__)
									_lowercase							=     tokenizer_r.encode(lowercase__    ,			add_special_tokens=lowercase__)

									_lowercase							=     tokenizer_r.convert_ids_to_tokens(lowercase__)
									_lowercase							=     tokenizer_p.convert_ids_to_tokens(lowercase__)

									# it is expected that each Chinese character is not preceded by "##"
									self.assertListEqual(lowercase__    ,			lowercase__)
									self.assertListEqual(lowercase__    ,			lowercase__)

									_lowercase							=     False
									_lowercase							=     self.rust_tokenizer_class.from_pretrained(lowercase__    ,			**lowercase__)
									_lowercase							=     self.tokenizer_class.from_pretrained(lowercase__    ,			**lowercase__)

									_lowercase							=     tokenizer_r.encode(lowercase__    ,			add_special_tokens=lowercase__)
									_lowercase							=     tokenizer_p.encode(lowercase__    ,			add_special_tokens=lowercase__)

									_lowercase							=     tokenizer_r.convert_ids_to_tokens(lowercase__)
									_lowercase							=     tokenizer_p.convert_ids_to_tokens(lowercase__)

									# it is expected that only the first Chinese character is not preceded by "##".
									_lowercase							=     [
									    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowercase__)
									]
									self.assertListEqual(lowercase__    ,			lowercase__)
									self.assertListEqual(lowercase__    ,			lowercase__)







'''simple docstring'''







from transformers import HfArgumentParser, PyTorchBenchmark, PyTorchBenchmarkArguments






def 					_SCREAMING_SNAKE_CASE		(						):
	_lowercase							=     HfArgumentParser(snake_case_	)
	try:
		_lowercase							=     parser.parse_args_into_dataclasses()[0]
	except ValueError as e:
		_lowercase							=     """Arg --no_{0} is no longer used, please use --no-{0} instead."""
		_lowercase							=     """ """.join(str(snake_case_	).split(""" """	)[:-1]	)
		_lowercase							=     """"""
		_lowercase							=     eval(str(snake_case_	).split(""" """	)[-1]	)
		_lowercase							=     []
		for arg in depreciated_args:
			# arg[2:] removes '--'
			if arg[2:] in PyTorchBenchmarkArguments.deprecated_args:
				# arg[5:] removes '--no_'
				full_error_msg += arg_error_msg.format(arg[5:]	)
			else:
				wrong_args.append(snake_case_	)
		if len(snake_case_	) > 0:
			_lowercase							=     full_error_msg + begin_error_msg + str(snake_case_	)
		raise ValueError(snake_case_	)

	_lowercase							=     PyTorchBenchmark(args=snake_case_	)
	benchmark.run()


if __name__ == "__main__":
							main()




