import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class     snake_case_			(						snake_case_    ,unittest.TestCase			):
   __A      :						List[Any]   =							ElectraTokenizer
   __A      :						str   =							ElectraTokenizerFast
   __A      :						Optional[int]   =							True
   __A      :						Dict   =							True
   __A      :						str   =							filter_non_english






   def  __UpperCamelCase	( self							:	Dict    )   ->		int:
       super().setUp()

       lowercase__					:  Any          =							[
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       lowercase__					:  str          =							os.path.join(self.tmpdirname      ,		VOCAB_FILES_NAMES["vocab_file"]    )
       with open(self.vocab_file      ,		"w"      ,		encoding="utf-8"    ) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]    )    )






   def  __UpperCamelCase	( self							:	Optional[int]      ,		lowercase_							:	int    )   ->		str:
       lowercase__					:  Union[str, Any]          =							'''UNwant\u00E9d,running'''
       lowercase__					:  Dict          =							'''unwanted, running'''
       return input_text, output_text






   def  __UpperCamelCase	( self							:	Optional[int]    )   ->		Optional[Any]:
       lowercase__					:  Tuple          =							self.tokenizer_class(self.vocab_file    )

       lowercase__					:  Union[str, Any]          =							tokenizer.tokenize("UNwant\u00E9d,running"    )
       self.assertListEqual(UpperCAmelCase__      ,		["un", "##want", "##ed", ",", "runn", "##ing"]    )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__    )      ,		[9, 6, 7, 12, 10, 11]    )






   def  __UpperCamelCase	( self							:	List[str]    )   ->		Optional[int]:
       if not self.test_rust_tokenizer:
           return

       lowercase__					:  Tuple          =							self.get_tokenizer()
       lowercase__					:  Dict          =							self.get_rust_tokenizer()

       lowercase__					:  Dict          =							'''UNwant\u00E9d,running'''

       lowercase__					:  Optional[int]          =							tokenizer.tokenize(UpperCAmelCase__    )
       lowercase__					:  Optional[int]          =							rust_tokenizer.tokenize(UpperCAmelCase__    )
       self.assertListEqual(UpperCAmelCase__      ,		UpperCAmelCase__    )

       lowercase__					:  int          =							tokenizer.encode(UpperCAmelCase__      ,		add_special_tokens=UpperCAmelCase__    )
       lowercase__					:  Any          =							rust_tokenizer.encode(UpperCAmelCase__      ,		add_special_tokens=UpperCAmelCase__    )
       self.assertListEqual(UpperCAmelCase__      ,		UpperCAmelCase__    )

       lowercase__					:  Dict          =							self.get_rust_tokenizer()
       lowercase__					:  Optional[Any]          =							tokenizer.encode(UpperCAmelCase__    )
       lowercase__					:  int          =							rust_tokenizer.encode(UpperCAmelCase__    )
       self.assertListEqual(UpperCAmelCase__      ,		UpperCAmelCase__    )

       # With lower casing
       lowercase__					:  Dict          =							self.get_tokenizer(do_lower_case=UpperCAmelCase__    )
       lowercase__					:  List[Any]          =							self.get_rust_tokenizer(do_lower_case=UpperCAmelCase__    )

       lowercase__					:  Union[str, Any]          =							'''UNwant\u00E9d,running'''

       lowercase__					:  int          =							tokenizer.tokenize(UpperCAmelCase__    )
       lowercase__					:  List[Any]          =							rust_tokenizer.tokenize(UpperCAmelCase__    )
       self.assertListEqual(UpperCAmelCase__      ,		UpperCAmelCase__    )

       lowercase__					:  Any          =							tokenizer.encode(UpperCAmelCase__      ,		add_special_tokens=UpperCAmelCase__    )
       lowercase__					:  str          =							rust_tokenizer.encode(UpperCAmelCase__      ,		add_special_tokens=UpperCAmelCase__    )
       self.assertListEqual(UpperCAmelCase__      ,		UpperCAmelCase__    )

       lowercase__					:  Union[str, Any]          =							self.get_rust_tokenizer()
       lowercase__					:  Optional[int]          =							tokenizer.encode(UpperCAmelCase__    )
       lowercase__					:  int          =							rust_tokenizer.encode(UpperCAmelCase__    )
       self.assertListEqual(UpperCAmelCase__      ,		UpperCAmelCase__    )






   def  __UpperCamelCase	( self							:	Union[str, Any]    )   ->		Optional[int]:
       lowercase__					:  Tuple          =							BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"    )      ,		["ah", "\u535A", "\u63A8", "zz"]    )






   def  __UpperCamelCase	( self							:	str    )   ->		Tuple:
       lowercase__					:  List[str]          =							BasicTokenizer(do_lower_case=UpperCAmelCase__    )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "    )      ,		["hello", "!", "how", "are", "you", "?"]    )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )      ,		["hello"]    )






   def  __UpperCamelCase	( self							:	Optional[int]    )   ->		int:
       lowercase__					:  List[Any]          =							BasicTokenizer(do_lower_case=UpperCAmelCase__      ,		strip_accents=UpperCAmelCase__    )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )      ,		["hällo", "!", "how", "are", "you", "?"]    )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )      ,		["h\u00E9llo"]    )






   def  __UpperCamelCase	( self							:	List[Any]    )   ->		Union[str, Any]:
       lowercase__					:  int          =							BasicTokenizer(do_lower_case=UpperCAmelCase__      ,		strip_accents=UpperCAmelCase__    )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )      ,		["hallo", "!", "how", "are", "you", "?"]    )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )      ,		["hello"]    )






   def  __UpperCamelCase	( self							:	int    )   ->		Union[str, Any]:
       lowercase__					:  int          =							BasicTokenizer(do_lower_case=UpperCAmelCase__    )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )      ,		["hallo", "!", "how", "are", "you", "?"]    )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )      ,		["hello"]    )






   def  __UpperCamelCase	( self							:	Dict    )   ->		Dict:
       lowercase__					:  Dict          =							BasicTokenizer(do_lower_case=UpperCAmelCase__    )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "    )      ,		["HeLLo", "!", "how", "Are", "yoU", "?"]    )






   def  __UpperCamelCase	( self							:	int    )   ->		Any:
       lowercase__					:  Optional[Any]          =							BasicTokenizer(do_lower_case=UpperCAmelCase__      ,		strip_accents=UpperCAmelCase__    )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )      ,		["HäLLo", "!", "how", "Are", "yoU", "?"]    )






   def  __UpperCamelCase	( self							:	str    )   ->		Any:
       lowercase__					:  Dict          =							BasicTokenizer(do_lower_case=UpperCAmelCase__      ,		strip_accents=UpperCAmelCase__    )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )      ,		["HaLLo", "!", "how", "Are", "yoU", "?"]    )






   def  __UpperCamelCase	( self							:	Tuple    )   ->		Tuple:
       lowercase__					:  Dict          =							BasicTokenizer(do_lower_case=UpperCAmelCase__      ,		never_split=["[UNK]"]    )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"    )      ,		["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]    )






   def  __UpperCamelCase	( self							:	str    )   ->		List[Any]:
       lowercase__					:  str          =							['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       lowercase__					:  Tuple          =							{}
       for i, token in enumerate(UpperCAmelCase__    ):
           lowercase__					:  Tuple          =							i
       lowercase__					:  Optional[Any]          =							WordpieceTokenizer(vocab=UpperCAmelCase__      ,		unk_token="[UNK]"    )

       self.assertListEqual(tokenizer.tokenize(""    )      ,		[]    )

       self.assertListEqual(tokenizer.tokenize("unwanted running"    )      ,		["un", "##want", "##ed", "runn", "##ing"]    )

       self.assertListEqual(tokenizer.tokenize("unwantedX running"    )      ,		["[UNK]", "runn", "##ing"]    )






   def  __UpperCamelCase	( self							:	Union[str, Any]    )   ->		Optional[int]:
       self.assertTrue(_is_whitespace(" "    )    )
       self.assertTrue(_is_whitespace("\t"    )    )
       self.assertTrue(_is_whitespace("\r"    )    )
       self.assertTrue(_is_whitespace("\n"    )    )
       self.assertTrue(_is_whitespace("\u00A0"    )    )

       self.assertFalse(_is_whitespace("A"    )    )
       self.assertFalse(_is_whitespace("-"    )    )






   def  __UpperCamelCase	( self							:	Any    )   ->		Optional[int]:
       self.assertTrue(_is_control("\u0005"    )    )

       self.assertFalse(_is_control("A"    )    )
       self.assertFalse(_is_control(" "    )    )
       self.assertFalse(_is_control("\t"    )    )
       self.assertFalse(_is_control("\r"    )    )






   def  __UpperCamelCase	( self							:	Optional[int]    )   ->		Dict:
       self.assertTrue(_is_punctuation("-"    )    )
       self.assertTrue(_is_punctuation("$"    )    )
       self.assertTrue(_is_punctuation("`"    )    )
       self.assertTrue(_is_punctuation("."    )    )

       self.assertFalse(_is_punctuation("A"    )    )
       self.assertFalse(_is_punctuation(" "    )    )






   def  __UpperCamelCase	( self							:	List[Any]    )   ->		List[Any]:
       lowercase__					:  Tuple          =							self.get_tokenizer()
       lowercase__					:  Dict          =							self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(UpperCAmelCase__    ) for t in ["Test", "\xad", "test"]]      ,		[["[UNK]"], [], ["[UNK]"]]    )

       self.assertListEqual(
           [rust_tokenizer.tokenize(UpperCAmelCase__    ) for t in ["Test", "\xad", "test"]]      ,		[["[UNK]"], [], ["[UNK]"]]    )






   @slow
   def  __UpperCamelCase	( self							:	Dict    )   ->		Any:
       lowercase__					:  Dict          =							self.tokenizer_class.from_pretrained("google/electra-base-discriminator"    )

       lowercase__					:  Optional[int]          =							tokenizer.encode("sequence builders"      ,		add_special_tokens=UpperCAmelCase__    )
       lowercase__					:  Union[str, Any]          =							tokenizer.encode("multi-sequence build"      ,		add_special_tokens=UpperCAmelCase__    )

       lowercase__					:  Optional[Any]          =							tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__    )
       lowercase__					:  Dict          =							tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__      ,		UpperCAmelCase__    )

       assert encoded_sentence == [1_01] + text + [1_02]
       assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]






   def  __UpperCamelCase	( self							:	Union[str, Any]    )   ->		Dict:
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
               lowercase__					:  int          =							self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__      ,		**UpperCAmelCase__    )

               lowercase__					:  Optional[Any]          =							F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
               lowercase__					:  Union[str, Any]          =							tokenizer_r.encode_plus(
                   UpperCAmelCase__      ,		return_attention_mask=UpperCAmelCase__      ,		return_token_type_ids=UpperCAmelCase__      ,		return_offsets_mapping=UpperCAmelCase__      ,		add_special_tokens=UpperCAmelCase__      ,		)

               lowercase__					:  Tuple          =							tokenizer_r.do_lower_case if hasattr(UpperCAmelCase__      ,		"do_lower_case"    ) else False
               lowercase__					:  Union[str, Any]          =							(
                   [
                       ((0, 0), tokenizer_r.cls_token),
                       ((0, 1), '''A'''),
                       ((1, 2), ''','''),
                       ((3, 5), '''na'''),
                       ((5, 6), '''##ï'''),
                       ((6, 8), '''##ve'''),
                       ((9, 15), tokenizer_r.mask_token),
                       ((16, 21), '''Allen'''),
                       ((21, 23), '''##NL'''),
                       ((23, 24), '''##P'''),
                       ((25, 33), '''sentence'''),
                       ((33, 34), '''.'''),
                       ((0, 0), tokenizer_r.sep_token),
                   ]
                   if not do_lower_case
                   else [
                       ((0, 0), tokenizer_r.cls_token),
                       ((0, 1), '''a'''),
                       ((1, 2), ''','''),
                       ((3, 8), '''naive'''),
                       ((9, 15), tokenizer_r.mask_token),
                       ((16, 21), '''allen'''),
                       ((21, 23), '''##nl'''),
                       ((23, 24), '''##p'''),
                       ((25, 33), '''sentence'''),
                       ((33, 34), '''.'''),
                       ((0, 0), tokenizer_r.sep_token),
                   ]
               )

               self.assertEqual(
                   [e[1] for e in expected_results]      ,		tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]    )    )
               self.assertEqual([e[0] for e in expected_results]      ,		tokens["offset_mapping"]    )







   def  __UpperCamelCase	( self							:	Optional[int]    )   ->		int:
       lowercase__					:  str          =							['''的''', '''人''', '''有''']
       lowercase__					:  Any          =							''''''.join(UpperCAmelCase__    )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
               lowercase__					:  Dict          =							True
               lowercase__					:  str          =							self.tokenizer_class.from_pretrained(UpperCAmelCase__      ,		**UpperCAmelCase__    )
               lowercase__					:  int          =							self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__      ,		**UpperCAmelCase__    )

               lowercase__					:  Dict          =							tokenizer_p.encode(UpperCAmelCase__      ,		add_special_tokens=UpperCAmelCase__    )
               lowercase__					:  Optional[Any]          =							tokenizer_r.encode(UpperCAmelCase__      ,		add_special_tokens=UpperCAmelCase__    )

               lowercase__					:  Any          =							tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__    )
               lowercase__					:  Union[str, Any]          =							tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__    )

               # it is expected that each Chinese character is not preceded by "##"
               self.assertListEqual(UpperCAmelCase__      ,		UpperCAmelCase__    )
               self.assertListEqual(UpperCAmelCase__      ,		UpperCAmelCase__    )

               lowercase__					:  Tuple          =							False
               lowercase__					:  List[Any]          =							self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__      ,		**UpperCAmelCase__    )
               lowercase__					:  Optional[int]          =							self.tokenizer_class.from_pretrained(UpperCAmelCase__      ,		**UpperCAmelCase__    )

               lowercase__					:  Optional[int]          =							tokenizer_r.encode(UpperCAmelCase__      ,		add_special_tokens=UpperCAmelCase__    )
               lowercase__					:  Tuple          =							tokenizer_p.encode(UpperCAmelCase__      ,		add_special_tokens=UpperCAmelCase__    )

               lowercase__					:  Dict          =							tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__    )
               lowercase__					:  Tuple          =							tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__    )

               # it is expected that only the first Chinese character is not preceded by "##".
               lowercase__					:  Optional[Any]          =							[
                   F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCAmelCase__    )
               ]
               self.assertListEqual(UpperCAmelCase__      ,		UpperCAmelCase__    )
               self.assertListEqual(UpperCAmelCase__      ,		UpperCAmelCase__    )










'''simple docstring'''






from .hash_table import HashTable
from .number_theory.prime_numbers import is_prime, next_prime



class  lowerCamelCase__				(						snake_case_						):


	"""simple docstring"""





	def __init__(					self   ,     *UpperCAmelCase__   ,     **UpperCAmelCase__			)       ->				Optional[int]:
					super().__init__(*UpperCAmelCase__   ,     **UpperCAmelCase__			)




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__   ,     UpperCAmelCase__			)       ->				Optional[int]:
					_A :				Union[str, Any]				=  (
					    next_prime(value % self.size_table			)
					    if not is_prime(value % self.size_table			)
					    else value % self.size_table
					)  # gt = bigger than
					return next_prime_gt - (data % next_prime_gt)




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__   ,     UpperCAmelCase__   ,     UpperCAmelCase__			)       ->				Union[str, Any]:
					return (increment * self.__hash_function_a(UpperCAmelCase__   ,     UpperCAmelCase__			)) % self.size_table




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__   ,     UpperCAmelCase__=None			)       ->				List[Any]:
					_A :				Optional[int]				=  1
					_A :				Optional[int]				=  self.hash_function(UpperCAmelCase__			)

					while self.values[new_key] is not None and self.values[new_key] != key:
									_A :				int				=  (
									    self.__hash_double_function(UpperCAmelCase__   ,     UpperCAmelCase__   ,     UpperCAmelCase__			)
									    if self.balanced_factor() >= self.lim_charge
									    else None
									)
									if new_key is None:
													break
									else:
													i += 1

					return new_key
