




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class       UpperCAmelCase_   ( __SCREAMING_SNAKE_CASE		,   unittest.TestCase							):




							'''simple docstring'''



							a__    	=  ElectraTokenizer
							a__    	=  ElectraTokenizerFast
							a__    	=  True
							a__    	=  True
							a__    	=  filter_non_english





							def  _UpperCAmelCase  (       self							:						str      )      ->       Optional[int]:
										super().setUp()

										SCREAMING_SNAKE_CASE     =       [
										    """[UNK]""",
										    """[CLS]""",
										    """[SEP]""",
										    """[PAD]""",
										    """[MASK]""",
										    """want""",
										    """##want""",
										    """##ed""",
										    """wa""",
										    """un""",
										    """runn""",
										    """##ing""",
										    """,""",
										    """low""",
										    """lowest""",
										]
										SCREAMING_SNAKE_CASE     =       os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES["""vocab_file"""]      )
										with open(self.vocab_file							,  """w"""							,  encoding="""utf-8"""      ) as vocab_writer:
													vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]      )      )





							def  _UpperCAmelCase  (       self							:						Union[str, Any]							,  a							:						Optional[Any]      )      ->       List[Any]:
										SCREAMING_SNAKE_CASE     =       """UNwant\u00E9d,running"""
										SCREAMING_SNAKE_CASE     =       """unwanted, running"""
										return input_text, output_text





							def  _UpperCAmelCase  (       self							:						Dict      )      ->       Optional[int]:
										SCREAMING_SNAKE_CASE     =       self.tokenizer_class(self.vocab_file      )

										SCREAMING_SNAKE_CASE     =       tokenizer.tokenize("""UNwant\u00E9d,running"""      )
										self.assertListEqual(lowerCamelCase_							,  ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]      )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_      )							,  [9, 6, 7, 12, 10, 11]      )





							def  _UpperCAmelCase  (       self							:						Optional[int]      )      ->       Optional[int]:
										if not self.test_rust_tokenizer:
													return

										SCREAMING_SNAKE_CASE     =       self.get_tokenizer()
										SCREAMING_SNAKE_CASE     =       self.get_rust_tokenizer()

										SCREAMING_SNAKE_CASE     =       """UNwant\u00E9d,running"""

										SCREAMING_SNAKE_CASE     =       tokenizer.tokenize(lowerCamelCase_      )
										SCREAMING_SNAKE_CASE     =       rust_tokenizer.tokenize(lowerCamelCase_      )
										self.assertListEqual(lowerCamelCase_							,  lowerCamelCase_      )

										SCREAMING_SNAKE_CASE     =       tokenizer.encode(lowerCamelCase_							,  add_special_tokens=lowerCamelCase_      )
										SCREAMING_SNAKE_CASE     =       rust_tokenizer.encode(lowerCamelCase_							,  add_special_tokens=lowerCamelCase_      )
										self.assertListEqual(lowerCamelCase_							,  lowerCamelCase_      )

										SCREAMING_SNAKE_CASE     =       self.get_rust_tokenizer()
										SCREAMING_SNAKE_CASE     =       tokenizer.encode(lowerCamelCase_      )
										SCREAMING_SNAKE_CASE     =       rust_tokenizer.encode(lowerCamelCase_      )
										self.assertListEqual(lowerCamelCase_							,  lowerCamelCase_      )

										# With lower casing
										SCREAMING_SNAKE_CASE     =       self.get_tokenizer(do_lower_case=lowerCamelCase_      )
										SCREAMING_SNAKE_CASE     =       self.get_rust_tokenizer(do_lower_case=lowerCamelCase_      )

										SCREAMING_SNAKE_CASE     =       """UNwant\u00E9d,running"""

										SCREAMING_SNAKE_CASE     =       tokenizer.tokenize(lowerCamelCase_      )
										SCREAMING_SNAKE_CASE     =       rust_tokenizer.tokenize(lowerCamelCase_      )
										self.assertListEqual(lowerCamelCase_							,  lowerCamelCase_      )

										SCREAMING_SNAKE_CASE     =       tokenizer.encode(lowerCamelCase_							,  add_special_tokens=lowerCamelCase_      )
										SCREAMING_SNAKE_CASE     =       rust_tokenizer.encode(lowerCamelCase_							,  add_special_tokens=lowerCamelCase_      )
										self.assertListEqual(lowerCamelCase_							,  lowerCamelCase_      )

										SCREAMING_SNAKE_CASE     =       self.get_rust_tokenizer()
										SCREAMING_SNAKE_CASE     =       tokenizer.encode(lowerCamelCase_      )
										SCREAMING_SNAKE_CASE     =       rust_tokenizer.encode(lowerCamelCase_      )
										self.assertListEqual(lowerCamelCase_							,  lowerCamelCase_      )





							def  _UpperCAmelCase  (       self							:						List[Any]      )      ->       Any:
										SCREAMING_SNAKE_CASE     =       BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""      )							,  ["""ah""", """\u535A""", """\u63A8""", """zz"""]      )





							def  _UpperCAmelCase  (       self							:						Dict      )      ->       int:
										SCREAMING_SNAKE_CASE     =       BasicTokenizer(do_lower_case=lowerCamelCase_      )

										self.assertListEqual(
										    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """      )							,  ["""hello""", """!""", """how""", """are""", """you""", """?"""]      )
										self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )							,  ["""hello"""]      )





							def  _UpperCAmelCase  (       self							:						List[Any]      )      ->       Union[str, Any]:
										SCREAMING_SNAKE_CASE     =       BasicTokenizer(do_lower_case=lowerCamelCase_							,  strip_accents=lowerCamelCase_      )

										self.assertListEqual(
										    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )							,  ["""hällo""", """!""", """how""", """are""", """you""", """?"""]      )
										self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )							,  ["""h\u00E9llo"""]      )





							def  _UpperCAmelCase  (       self							:						List[Any]      )      ->       int:
										SCREAMING_SNAKE_CASE     =       BasicTokenizer(do_lower_case=lowerCamelCase_							,  strip_accents=lowerCamelCase_      )

										self.assertListEqual(
										    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )							,  ["""hallo""", """!""", """how""", """are""", """you""", """?"""]      )
										self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )							,  ["""hello"""]      )





							def  _UpperCAmelCase  (       self							:						Any      )      ->       str:
										SCREAMING_SNAKE_CASE     =       BasicTokenizer(do_lower_case=lowerCamelCase_      )

										self.assertListEqual(
										    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )							,  ["""hallo""", """!""", """how""", """are""", """you""", """?"""]      )
										self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )							,  ["""hello"""]      )





							def  _UpperCAmelCase  (       self							:						int      )      ->       List[Any]:
										SCREAMING_SNAKE_CASE     =       BasicTokenizer(do_lower_case=lowerCamelCase_      )

										self.assertListEqual(
										    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """      )							,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]      )





							def  _UpperCAmelCase  (       self							:						int      )      ->       int:
										SCREAMING_SNAKE_CASE     =       BasicTokenizer(do_lower_case=lowerCamelCase_							,  strip_accents=lowerCamelCase_      )

										self.assertListEqual(
										    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )							,  ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]      )





							def  _UpperCAmelCase  (       self							:						str      )      ->       str:
										SCREAMING_SNAKE_CASE     =       BasicTokenizer(do_lower_case=lowerCamelCase_							,  strip_accents=lowerCamelCase_      )

										self.assertListEqual(
										    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )							,  ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]      )





							def  _UpperCAmelCase  (       self							:						Dict      )      ->       int:
										SCREAMING_SNAKE_CASE     =       BasicTokenizer(do_lower_case=lowerCamelCase_							,  never_split=["""[UNK]"""]      )

										self.assertListEqual(
										    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""      )							,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]      )





							def  _UpperCAmelCase  (       self							:						Optional[Any]      )      ->       List[Any]:
										SCREAMING_SNAKE_CASE     =       ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

										SCREAMING_SNAKE_CASE     =       {}
										for i, token in enumerate(lowerCamelCase_      ):
													SCREAMING_SNAKE_CASE     =       i
										SCREAMING_SNAKE_CASE     =       WordpieceTokenizer(vocab=lowerCamelCase_							,  unk_token="""[UNK]"""      )

										self.assertListEqual(tokenizer.tokenize(""""""      )							,  []      )

										self.assertListEqual(tokenizer.tokenize("""unwanted running"""      )							,  ["""un""", """##want""", """##ed""", """runn""", """##ing"""]      )

										self.assertListEqual(tokenizer.tokenize("""unwantedX running"""      )							,  ["""[UNK]""", """runn""", """##ing"""]      )





							def  _UpperCAmelCase  (       self							:						List[Any]      )      ->       str:
										self.assertTrue(_is_whitespace(""" """      )      )
										self.assertTrue(_is_whitespace("""\t"""      )      )
										self.assertTrue(_is_whitespace("""\r"""      )      )
										self.assertTrue(_is_whitespace("""\n"""      )      )
										self.assertTrue(_is_whitespace("""\u00A0"""      )      )

										self.assertFalse(_is_whitespace("""A"""      )      )
										self.assertFalse(_is_whitespace("""-"""      )      )





							def  _UpperCAmelCase  (       self							:						List[str]      )      ->       Optional[int]:
										self.assertTrue(_is_control("""\u0005"""      )      )

										self.assertFalse(_is_control("""A"""      )      )
										self.assertFalse(_is_control(""" """      )      )
										self.assertFalse(_is_control("""\t"""      )      )
										self.assertFalse(_is_control("""\r"""      )      )





							def  _UpperCAmelCase  (       self							:						Union[str, Any]      )      ->       Optional[Any]:
										self.assertTrue(_is_punctuation("""-"""      )      )
										self.assertTrue(_is_punctuation("""$"""      )      )
										self.assertTrue(_is_punctuation("""`"""      )      )
										self.assertTrue(_is_punctuation("""."""      )      )

										self.assertFalse(_is_punctuation("""A"""      )      )
										self.assertFalse(_is_punctuation(""" """      )      )





							def  _UpperCAmelCase  (       self							:						Any      )      ->       str:
										SCREAMING_SNAKE_CASE     =       self.get_tokenizer()
										SCREAMING_SNAKE_CASE     =       self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowerCamelCase_      ) for t in ["""Test""", """\xad""", """test"""]]							,  [["""[UNK]"""], [], ["""[UNK]"""]]      )

										self.assertListEqual(
										    [rust_tokenizer.tokenize(lowerCamelCase_      ) for t in ["""Test""", """\xad""", """test"""]]							,  [["""[UNK]"""], [], ["""[UNK]"""]]      )





							@slow
							def  _UpperCAmelCase  (       self							:						Any      )      ->       Any:
										SCREAMING_SNAKE_CASE     =       self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""      )

										SCREAMING_SNAKE_CASE     =       tokenizer.encode("""sequence builders"""							,  add_special_tokens=lowerCamelCase_      )
										SCREAMING_SNAKE_CASE     =       tokenizer.encode("""multi-sequence build"""							,  add_special_tokens=lowerCamelCase_      )

										SCREAMING_SNAKE_CASE     =       tokenizer.build_inputs_with_special_tokens(lowerCamelCase_      )
										SCREAMING_SNAKE_CASE     =       tokenizer.build_inputs_with_special_tokens(lowerCamelCase_							,  lowerCamelCase_      )

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]





							def  _UpperCAmelCase  (       self							:						Tuple      )      ->       Optional[int]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
																SCREAMING_SNAKE_CASE     =       self.rust_tokenizer_class.from_pretrained(lowerCamelCase_							,  **lowerCamelCase_      )

																SCREAMING_SNAKE_CASE     =       f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																SCREAMING_SNAKE_CASE     =       tokenizer_r.encode_plus(
																    lowerCamelCase_							,  return_attention_mask=lowerCamelCase_							,  return_token_type_ids=lowerCamelCase_							,  return_offsets_mapping=lowerCamelCase_							,  add_special_tokens=lowerCamelCase_							,  )

																SCREAMING_SNAKE_CASE     =       tokenizer_r.do_lower_case if hasattr(lowerCamelCase_							,  """do_lower_case"""      ) else False
																SCREAMING_SNAKE_CASE     =       (
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), """A"""),
																        ((1, 2), ""","""),
																        ((3, 5), """na"""),
																        ((5, 6), """##ï"""),
																        ((6, 8), """##ve"""),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), """Allen"""),
																        ((21, 23), """##NL"""),
																        ((23, 24), """##P"""),
																        ((25, 33), """sentence"""),
																        ((33, 34), """."""),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), """a"""),
																        ((1, 2), ""","""),
																        ((3, 8), """naive"""),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), """allen"""),
																        ((21, 23), """##nl"""),
																        ((23, 24), """##p"""),
																        ((25, 33), """sentence"""),
																        ((33, 34), """."""),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]							,  tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]      )      )
																self.assertEqual([e[0] for e in expected_results]							,  tokens["""offset_mapping"""]      )







							def  _UpperCAmelCase  (       self							:						List[Any]      )      ->       Dict:
										SCREAMING_SNAKE_CASE     =       ["""的""", """人""", """有"""]
										SCREAMING_SNAKE_CASE     =       """""".join(lowerCamelCase_      )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
																SCREAMING_SNAKE_CASE     =       True
																SCREAMING_SNAKE_CASE     =       self.tokenizer_class.from_pretrained(lowerCamelCase_							,  **lowerCamelCase_      )
																SCREAMING_SNAKE_CASE     =       self.rust_tokenizer_class.from_pretrained(lowerCamelCase_							,  **lowerCamelCase_      )

																SCREAMING_SNAKE_CASE     =       tokenizer_p.encode(lowerCamelCase_							,  add_special_tokens=lowerCamelCase_      )
																SCREAMING_SNAKE_CASE     =       tokenizer_r.encode(lowerCamelCase_							,  add_special_tokens=lowerCamelCase_      )

																SCREAMING_SNAKE_CASE     =       tokenizer_r.convert_ids_to_tokens(lowerCamelCase_      )
																SCREAMING_SNAKE_CASE     =       tokenizer_p.convert_ids_to_tokens(lowerCamelCase_      )

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(lowerCamelCase_							,  lowerCamelCase_      )
																self.assertListEqual(lowerCamelCase_							,  lowerCamelCase_      )

																SCREAMING_SNAKE_CASE     =       False
																SCREAMING_SNAKE_CASE     =       self.rust_tokenizer_class.from_pretrained(lowerCamelCase_							,  **lowerCamelCase_      )
																SCREAMING_SNAKE_CASE     =       self.tokenizer_class.from_pretrained(lowerCamelCase_							,  **lowerCamelCase_      )

																SCREAMING_SNAKE_CASE     =       tokenizer_r.encode(lowerCamelCase_							,  add_special_tokens=lowerCamelCase_      )
																SCREAMING_SNAKE_CASE     =       tokenizer_p.encode(lowerCamelCase_							,  add_special_tokens=lowerCamelCase_      )

																SCREAMING_SNAKE_CASE     =       tokenizer_r.convert_ids_to_tokens(lowerCamelCase_      )
																SCREAMING_SNAKE_CASE     =       tokenizer_p.convert_ids_to_tokens(lowerCamelCase_      )

																# it is expected that only the first Chinese character is not preceded by "##".
																SCREAMING_SNAKE_CASE     =       [
																    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase_      )
																]
																self.assertListEqual(lowerCamelCase_							,  lowerCamelCase_      )
																self.assertListEqual(lowerCamelCase_							,  lowerCamelCase_      )










"""simple docstring"""







from __future__ import annotations






def        a__			(							_UpperCamelCase			):





						"""simple docstring"""

						UpperCAmelCase__												=     sorted(_UpperCamelCase			)
						UpperCAmelCase__												=     len(_UpperCamelCase			)
						UpperCAmelCase__												=     length >> 1
						return (
						    (sorted_list[mid_index] + sorted_list[mid_index - 1]) / 2
						    if length % 2 == 0
						    else sorted_list[mid_index]
						)






def        a__			(							):





						"""simple docstring"""

						import doctest

						doctest.testmod()


if __name__ == "__main__":
						main()