"""simple docstring"""




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      UpperCamelCase__(      __A		,    unittest.TestCase			):
						lowerCAmelCase__	:    str    				=		ElectraTokenizer
						lowerCAmelCase__	:    Optional[int]    				=		ElectraTokenizerFast
						lowerCAmelCase__	:    Optional[Any]    				=		True
						lowerCAmelCase__	:    Union[str, Any]    				=		True
						lowerCAmelCase__	:    Tuple    				=		filter_non_english





						def 					snake_case__						(							self  )       ->					Optional[int]:
													super().setUp()

													A__						  =				[
													    '''[UNK]''',
													    '''[CLS]''',
													    '''[SEP]''',
													    '''[PAD]''',
													    '''[MASK]''',
													    '''want''',
													    '''##want''',
													    '''##ed''',
													    '''wa''',
													    '''un''',
													    '''runn''',
													    '''##ing''',
													    ''',''',
													    '''low''',
													    '''lowest''',
													]
													A__						  =				os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['vocab_file']  )
													with open(self.vocab_file		,'w'		,encoding='utf-8'  ) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					List[str]:
													A__						  =				'''UNwant\u00E9d,running'''
													A__						  =				'''unwanted, running'''
													return input_text, output_text





						def 					snake_case__						(							self  )       ->					List[Any]:
													A__						  =				self.tokenizer_class(self.vocab_file  )

													A__						  =				tokenizer.tokenize('UNwant\u00E9d,running'  )
													self.assertListEqual(a__		,['un', '##want', '##ed', ',', 'runn', '##ing']  )
													self.assertListEqual(tokenizer.convert_tokens_to_ids(a__  )		,[9, 6, 7, 12, 10, 11]  )





						def 					snake_case__						(							self  )       ->					List[str]:






													if not self.test_rust_tokenizer:
																				return

													A__						  =				self.get_tokenizer()
													A__						  =				self.get_rust_tokenizer()

													A__						  =				'''UNwant\u00E9d,running'''

													A__						  =				tokenizer.tokenize(a__  )
													A__						  =				rust_tokenizer.tokenize(a__  )
													self.assertListEqual(a__		,a__  )

													A__						  =				tokenizer.encode(a__		,add_special_tokens=a__  )
													A__						  =				rust_tokenizer.encode(a__		,add_special_tokens=a__  )
													self.assertListEqual(a__		,a__  )

													A__						  =				self.get_rust_tokenizer()
													A__						  =				tokenizer.encode(a__  )
													A__						  =				rust_tokenizer.encode(a__  )
													self.assertListEqual(a__		,a__  )

													# With lower casing
													A__						  =				self.get_tokenizer(do_lower_case=a__  )
													A__						  =				self.get_rust_tokenizer(do_lower_case=a__  )

													A__						  =				'''UNwant\u00E9d,running'''

													A__						  =				tokenizer.tokenize(a__  )
													A__						  =				rust_tokenizer.tokenize(a__  )
													self.assertListEqual(a__		,a__  )

													A__						  =				tokenizer.encode(a__		,add_special_tokens=a__  )
													A__						  =				rust_tokenizer.encode(a__		,add_special_tokens=a__  )
													self.assertListEqual(a__		,a__  )

													A__						  =				self.get_rust_tokenizer()
													A__						  =				tokenizer.encode(a__  )
													A__						  =				rust_tokenizer.encode(a__  )
													self.assertListEqual(a__		,a__  )





						def 					snake_case__						(							self  )       ->					List[str]:
													A__						  =				BasicTokenizer()

													self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )		,['ah', '\u535A', '\u63A8', 'zz']  )





						def 					snake_case__						(							self  )       ->					str:
													A__						  =				BasicTokenizer(do_lower_case=a__  )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )		,['hello', '!', 'how', 'are', 'you', '?']  )
													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )		,['hello']  )





						def 					snake_case__						(							self  )       ->					Dict:
													A__						  =				BasicTokenizer(do_lower_case=a__		,strip_accents=a__  )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )		,['hällo', '!', 'how', 'are', 'you', '?']  )
													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )		,['h\u00E9llo']  )





						def 					snake_case__						(							self  )       ->					Dict:
													A__						  =				BasicTokenizer(do_lower_case=a__		,strip_accents=a__  )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )		,['hallo', '!', 'how', 'are', 'you', '?']  )
													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )		,['hello']  )





						def 					snake_case__						(							self  )       ->					List[Any]:
													A__						  =				BasicTokenizer(do_lower_case=a__  )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )		,['hallo', '!', 'how', 'are', 'you', '?']  )
													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )		,['hello']  )





						def 					snake_case__						(							self  )       ->					str:
													A__						  =				BasicTokenizer(do_lower_case=a__  )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )		,['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )





						def 					snake_case__						(							self  )       ->					Any:
													A__						  =				BasicTokenizer(do_lower_case=a__		,strip_accents=a__  )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )		,['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )





						def 					snake_case__						(							self  )       ->					Optional[int]:
													A__						  =				BasicTokenizer(do_lower_case=a__		,strip_accents=a__  )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )		,['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )





						def 					snake_case__						(							self  )       ->					Tuple:
													A__						  =				BasicTokenizer(do_lower_case=a__		,never_split=['[UNK]']  )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )		,['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )





						def 					snake_case__						(							self  )       ->					Tuple:
													A__						  =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

													A__						  =				{}
													for i, token in enumerate(a__  ):
																				A__						  =				i
													A__						  =				WordpieceTokenizer(vocab=a__		,unk_token='[UNK]'  )

													self.assertListEqual(tokenizer.tokenize(''  )		,[]  )

													self.assertListEqual(tokenizer.tokenize('unwanted running'  )		,['un', '##want', '##ed', 'runn', '##ing']  )

													self.assertListEqual(tokenizer.tokenize('unwantedX running'  )		,['[UNK]', 'runn', '##ing']  )





						def 					snake_case__						(							self  )       ->					int:
													self.assertTrue(_is_whitespace(' '  )  )
													self.assertTrue(_is_whitespace('\t'  )  )
													self.assertTrue(_is_whitespace('\r'  )  )
													self.assertTrue(_is_whitespace('\n'  )  )
													self.assertTrue(_is_whitespace('\u00A0'  )  )

													self.assertFalse(_is_whitespace('A'  )  )
													self.assertFalse(_is_whitespace('-'  )  )





						def 					snake_case__						(							self  )       ->					Union[str, Any]:
													self.assertTrue(_is_control('\u0005'  )  )

													self.assertFalse(_is_control('A'  )  )
													self.assertFalse(_is_control(' '  )  )
													self.assertFalse(_is_control('\t'  )  )
													self.assertFalse(_is_control('\r'  )  )





						def 					snake_case__						(							self  )       ->					List[str]:
													self.assertTrue(_is_punctuation('-'  )  )
													self.assertTrue(_is_punctuation('$'  )  )
													self.assertTrue(_is_punctuation('`'  )  )
													self.assertTrue(_is_punctuation('.'  )  )

													self.assertFalse(_is_punctuation('A'  )  )
													self.assertFalse(_is_punctuation(' '  )  )





						def 					snake_case__						(							self  )       ->					int:
													A__						  =				self.get_tokenizer()
													A__						  =				self.get_rust_tokenizer()

													# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
													self.assertListEqual([tokenizer.tokenize(a__  ) for t in ['Test', '\xad', 'test']]		,[['[UNK]'], [], ['[UNK]']]  )

													self.assertListEqual(
													    [rust_tokenizer.tokenize(a__  ) for t in ['Test', '\xad', 'test']]		,[['[UNK]'], [], ['[UNK]']]  )





						@slow
						def 					snake_case__						(							self  )       ->					List[str]:
													A__						  =				self.tokenizer_class.from_pretrained('google/electra-base-discriminator'  )

													A__						  =				tokenizer.encode('sequence builders'		,add_special_tokens=a__  )
													A__						  =				tokenizer.encode('multi-sequence build'		,add_special_tokens=a__  )

													A__						  =				tokenizer.build_inputs_with_special_tokens(a__  )
													A__						  =				tokenizer.build_inputs_with_special_tokens(a__		,a__  )

													assert encoded_sentence == [1_01] + text + [1_02]
													assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]





						def 					snake_case__						(							self  )       ->					List[Any]:






													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																											A__						  =				self.rust_tokenizer_class.from_pretrained(a__		,**a__  )

																											A__						  =				f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																											A__						  =				tokenizer_r.encode_plus(
																											    a__		,return_attention_mask=a__		,return_token_type_ids=a__		,return_offsets_mapping=a__		,add_special_tokens=a__		,)

																											A__						  =				tokenizer_r.do_lower_case if hasattr(a__		,'do_lower_case'  ) else False
																											A__						  =				(
																											    [
																											        ((0, 0), tokenizer_r.cls_token),
																											        ((0, 1), '''A'''),
																											        ((1, 2), ''','''),
																											        ((3, 5), '''na'''),
																											        ((5, 6), '''##ï'''),
																											        ((6, 8), '''##ve'''),
																											        ((9, 15), tokenizer_r.mask_token),
																											        ((16, 21), '''Allen'''),
																											        ((21, 23), '''##NL'''),
																											        ((23, 24), '''##P'''),
																											        ((25, 33), '''sentence'''),
																											        ((33, 34), '''.'''),
																											        ((0, 0), tokenizer_r.sep_token),
																											    ]
																											    if not do_lower_case
																											    else [
																											        ((0, 0), tokenizer_r.cls_token),
																											        ((0, 1), '''a'''),
																											        ((1, 2), ''','''),
																											        ((3, 8), '''naive'''),
																											        ((9, 15), tokenizer_r.mask_token),
																											        ((16, 21), '''allen'''),
																											        ((21, 23), '''##nl'''),
																											        ((23, 24), '''##p'''),
																											        ((25, 33), '''sentence'''),
																											        ((33, 34), '''.'''),
																											        ((0, 0), tokenizer_r.sep_token),
																											    ]
																											)

																											self.assertEqual(
																											    [e[1] for e in expected_results]		,tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
																											self.assertEqual([e[0] for e in expected_results]		,tokens['offset_mapping']  )





						def 					snake_case__						(							self  )       ->					str:
													A__						  =				['''的''', '''人''', '''有''']
													A__						  =				''''''.join(a__  )
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																											A__						  =				True
																											A__						  =				self.tokenizer_class.from_pretrained(a__		,**a__  )
																											A__						  =				self.rust_tokenizer_class.from_pretrained(a__		,**a__  )

																											A__						  =				tokenizer_p.encode(a__		,add_special_tokens=a__  )
																											A__						  =				tokenizer_r.encode(a__		,add_special_tokens=a__  )

																											A__						  =				tokenizer_r.convert_ids_to_tokens(a__  )
																											A__						  =				tokenizer_p.convert_ids_to_tokens(a__  )

																											# it is expected that each Chinese character is not preceded by "##"
																											self.assertListEqual(a__		,a__  )
																											self.assertListEqual(a__		,a__  )

																											A__						  =				False
																											A__						  =				self.rust_tokenizer_class.from_pretrained(a__		,**a__  )
																											A__						  =				self.tokenizer_class.from_pretrained(a__		,**a__  )

																											A__						  =				tokenizer_r.encode(a__		,add_special_tokens=a__  )
																											A__						  =				tokenizer_p.encode(a__		,add_special_tokens=a__  )

																											A__						  =				tokenizer_r.convert_ids_to_tokens(a__  )
																											A__						  =				tokenizer_p.convert_ids_to_tokens(a__  )

																											# it is expected that only the first Chinese character is not preceded by "##".
																											A__						  =				[
																											    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(a__  )
																											]
																											self.assertListEqual(a__		,a__  )
																											self.assertListEqual(a__		,a__  )









a					:				Tuple       =      [0.4_85, 0.4_56, 0.4_06]
a					:				Optional[int]       =      [0.2_29, 0.2_24, 0.2_25]
a					:				List[Any]       =      [0.5, 0.5, 0.5]
a					:				Tuple       =      [0.5, 0.5, 0.5]
a					:				Optional[Any]       =      [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73]
a					:				int       =      [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11]






