


"""simple docstring"""







import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 				lowerCamelCase							(      __SCREAMING_SNAKE_CASE					,       unittest.TestCase			):


							"""simple docstring"""
							SCREAMING_SNAKE_CASE              =   ElectraTokenizer
							SCREAMING_SNAKE_CASE              =   ElectraTokenizerFast
							SCREAMING_SNAKE_CASE              =   True
							SCREAMING_SNAKE_CASE              =   True
							SCREAMING_SNAKE_CASE              =   filter_non_english






							def        lowerCAmelCase  (    self			:  str					):
													super().setUp()

													UpperCAmelCase__												=     [
													    '''[UNK]''',
													    '''[CLS]''',
													    '''[SEP]''',
													    '''[PAD]''',
													    '''[MASK]''',
													    '''want''',
													    '''##want''',
													    '''##ed''',
													    '''wa''',
													    '''un''',
													    '''runn''',
													    '''##ing''',
													    ''',''',
													    '''low''',
													    '''lowest''',
													]
													UpperCAmelCase__												=     os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES['''vocab_file''']					)
													with open(self.vocab_file		,			'''w'''		,			encoding='''utf-8'''					) as vocab_writer:
																			vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)






							def        lowerCAmelCase  (    self			:  Union[str, Any]		,			lowerCamelCase_			:  Optional[Any]					):
													UpperCAmelCase__												=     '''UNwant\u00E9d,running'''
													UpperCAmelCase__												=     '''unwanted, running'''
													return input_text, output_text






							def        lowerCAmelCase  (    self			:  Dict					):
													UpperCAmelCase__												=     self.tokenizer_class(self.vocab_file					)

													UpperCAmelCase__												=     tokenizer.tokenize('''UNwant\u00E9d,running'''					)
													self.assertListEqual(lowerCamelCase_		,			['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']					)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_					)		,			[9, 6, 7, 12, 10, 11]					)






							def        lowerCAmelCase  (    self			:  Optional[int]					):
													if not self.test_rust_tokenizer:
																			return

													UpperCAmelCase__												=     self.get_tokenizer()
													UpperCAmelCase__												=     self.get_rust_tokenizer()

													UpperCAmelCase__												=     '''UNwant\u00E9d,running'''

													UpperCAmelCase__												=     tokenizer.tokenize(lowerCamelCase_					)
													UpperCAmelCase__												=     rust_tokenizer.tokenize(lowerCamelCase_					)
													self.assertListEqual(lowerCamelCase_		,			lowerCamelCase_					)

													UpperCAmelCase__												=     tokenizer.encode(lowerCamelCase_		,			add_special_tokens=lowerCamelCase_					)
													UpperCAmelCase__												=     rust_tokenizer.encode(lowerCamelCase_		,			add_special_tokens=lowerCamelCase_					)
													self.assertListEqual(lowerCamelCase_		,			lowerCamelCase_					)

													UpperCAmelCase__												=     self.get_rust_tokenizer()
													UpperCAmelCase__												=     tokenizer.encode(lowerCamelCase_					)
													UpperCAmelCase__												=     rust_tokenizer.encode(lowerCamelCase_					)
													self.assertListEqual(lowerCamelCase_		,			lowerCamelCase_					)

													# With lower casing
													UpperCAmelCase__												=     self.get_tokenizer(do_lower_case=lowerCamelCase_					)
													UpperCAmelCase__												=     self.get_rust_tokenizer(do_lower_case=lowerCamelCase_					)

													UpperCAmelCase__												=     '''UNwant\u00E9d,running'''

													UpperCAmelCase__												=     tokenizer.tokenize(lowerCamelCase_					)
													UpperCAmelCase__												=     rust_tokenizer.tokenize(lowerCamelCase_					)
													self.assertListEqual(lowerCamelCase_		,			lowerCamelCase_					)

													UpperCAmelCase__												=     tokenizer.encode(lowerCamelCase_		,			add_special_tokens=lowerCamelCase_					)
													UpperCAmelCase__												=     rust_tokenizer.encode(lowerCamelCase_		,			add_special_tokens=lowerCamelCase_					)
													self.assertListEqual(lowerCamelCase_		,			lowerCamelCase_					)

													UpperCAmelCase__												=     self.get_rust_tokenizer()
													UpperCAmelCase__												=     tokenizer.encode(lowerCamelCase_					)
													UpperCAmelCase__												=     rust_tokenizer.encode(lowerCamelCase_					)
													self.assertListEqual(lowerCamelCase_		,			lowerCamelCase_					)






							def        lowerCAmelCase  (    self			:  List[Any]					):
													UpperCAmelCase__												=     BasicTokenizer()

													self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)		,			['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)






							def        lowerCAmelCase  (    self			:  Dict					):
													UpperCAmelCase__												=     BasicTokenizer(do_lower_case=lowerCamelCase_					)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)		,			['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
													self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)		,			['''hello''']					)






							def        lowerCAmelCase  (    self			:  List[Any]					):
													UpperCAmelCase__												=     BasicTokenizer(do_lower_case=lowerCamelCase_		,			strip_accents=lowerCamelCase_					)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)		,			['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
													self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)		,			['''h\u00E9llo''']					)






							def        lowerCAmelCase  (    self			:  List[Any]					):
													UpperCAmelCase__												=     BasicTokenizer(do_lower_case=lowerCamelCase_		,			strip_accents=lowerCamelCase_					)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)		,			['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
													self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)		,			['''hello''']					)






							def        lowerCAmelCase  (    self			:  Any					):
													UpperCAmelCase__												=     BasicTokenizer(do_lower_case=lowerCamelCase_					)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)		,			['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
													self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)		,			['''hello''']					)






							def        lowerCAmelCase  (    self			:  int					):
													UpperCAmelCase__												=     BasicTokenizer(do_lower_case=lowerCamelCase_					)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)		,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)






							def        lowerCAmelCase  (    self			:  int					):
													UpperCAmelCase__												=     BasicTokenizer(do_lower_case=lowerCamelCase_		,			strip_accents=lowerCamelCase_					)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)		,			['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)






							def        lowerCAmelCase  (    self			:  str					):
													UpperCAmelCase__												=     BasicTokenizer(do_lower_case=lowerCamelCase_		,			strip_accents=lowerCamelCase_					)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)		,			['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)






							def        lowerCAmelCase  (    self			:  Dict					):
													UpperCAmelCase__												=     BasicTokenizer(do_lower_case=lowerCamelCase_		,			never_split=['''[UNK]''']					)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)		,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)






							def        lowerCAmelCase  (    self			:  Optional[Any]					):
													UpperCAmelCase__												=     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

													UpperCAmelCase__												=     {}
													for i, token in enumerate(lowerCamelCase_					):
																			UpperCAmelCase__												=     i
													UpperCAmelCase__												=     WordpieceTokenizer(vocab=lowerCamelCase_		,			unk_token='''[UNK]'''					)

													self.assertListEqual(tokenizer.tokenize(''''''					)		,			[]					)

													self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)		,			['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)

													self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)		,			['''[UNK]''', '''runn''', '''##ing''']					)






							def        lowerCAmelCase  (    self			:  List[Any]					):
													self.assertTrue(_is_whitespace(''' '''					)					)
													self.assertTrue(_is_whitespace('''\t'''					)					)
													self.assertTrue(_is_whitespace('''\r'''					)					)
													self.assertTrue(_is_whitespace('''\n'''					)					)
													self.assertTrue(_is_whitespace('''\u00A0'''					)					)

													self.assertFalse(_is_whitespace('''A'''					)					)
													self.assertFalse(_is_whitespace('''-'''					)					)






							def        lowerCAmelCase  (    self			:  List[str]					):
													self.assertTrue(_is_control('''\u0005'''					)					)

													self.assertFalse(_is_control('''A'''					)					)
													self.assertFalse(_is_control(''' '''					)					)
													self.assertFalse(_is_control('''\t'''					)					)
													self.assertFalse(_is_control('''\r'''					)					)






							def        lowerCAmelCase  (    self			:  Union[str, Any]					):
													self.assertTrue(_is_punctuation('''-'''					)					)
													self.assertTrue(_is_punctuation('''$'''					)					)
													self.assertTrue(_is_punctuation('''`'''					)					)
													self.assertTrue(_is_punctuation('''.'''					)					)

													self.assertFalse(_is_punctuation('''A'''					)					)
													self.assertFalse(_is_punctuation(''' '''					)					)






							def        lowerCAmelCase  (    self			:  Any					):
													UpperCAmelCase__												=     self.get_tokenizer()
													UpperCAmelCase__												=     self.get_rust_tokenizer()

													# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
													self.assertListEqual([tokenizer.tokenize(lowerCamelCase_					) for t in ['''Test''', '''\xad''', '''test''']]		,			[['''[UNK]'''], [], ['''[UNK]''']]					)

													self.assertListEqual(
													    [rust_tokenizer.tokenize(lowerCamelCase_					) for t in ['''Test''', '''\xad''', '''test''']]		,			[['''[UNK]'''], [], ['''[UNK]''']]					)






							@slow
							def        lowerCAmelCase  (    self			:  Any					):
													UpperCAmelCase__												=     self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''					)

													UpperCAmelCase__												=     tokenizer.encode('''sequence builders'''		,			add_special_tokens=lowerCamelCase_					)
													UpperCAmelCase__												=     tokenizer.encode('''multi-sequence build'''		,			add_special_tokens=lowerCamelCase_					)

													UpperCAmelCase__												=     tokenizer.build_inputs_with_special_tokens(lowerCamelCase_					)
													UpperCAmelCase__												=     tokenizer.build_inputs_with_special_tokens(lowerCamelCase_		,			lowerCamelCase_					)

													assert encoded_sentence == [101] + text + [102]
													assert encoded_pair == [101] + text + [102] + text_a + [102]






							def        lowerCAmelCase  (    self			:  Tuple					):
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
																									UpperCAmelCase__												=     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_		,			**lowerCamelCase_					)

																									UpperCAmelCase__												=     f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																									UpperCAmelCase__												=     tokenizer_r.encode_plus(
																									    lowerCamelCase_		,			return_attention_mask=lowerCamelCase_		,			return_token_type_ids=lowerCamelCase_		,			return_offsets_mapping=lowerCamelCase_		,			add_special_tokens=lowerCamelCase_		,			)

																									UpperCAmelCase__												=     tokenizer_r.do_lower_case if hasattr(lowerCamelCase_		,			'''do_lower_case'''					) else False
																									UpperCAmelCase__												=     (
																									    [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), '''A'''),
																									        ((1, 2), ''','''),
																									        ((3, 5), '''na'''),
																									        ((5, 6), '''##ï'''),
																									        ((6, 8), '''##ve'''),
																									        ((9, 15), tokenizer_r.mask_token),
																									        ((16, 21), '''Allen'''),
																									        ((21, 23), '''##NL'''),
																									        ((23, 24), '''##P'''),
																									        ((25, 33), '''sentence'''),
																									        ((33, 34), '''.'''),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									    if not do_lower_case
																									    else [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), '''a'''),
																									        ((1, 2), ''','''),
																									        ((3, 8), '''naive'''),
																									        ((9, 15), tokenizer_r.mask_token),
																									        ((16, 21), '''allen'''),
																									        ((21, 23), '''##nl'''),
																									        ((23, 24), '''##p'''),
																									        ((25, 33), '''sentence'''),
																									        ((33, 34), '''.'''),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									)

																									self.assertEqual(
																									    [e[1] for e in expected_results]		,			tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
																									self.assertEqual([e[0] for e in expected_results]		,			tokens['''offset_mapping''']					)






							def        lowerCAmelCase  (    self			:  List[Any]					):
													UpperCAmelCase__												=     ['''的''', '''人''', '''有''']
													UpperCAmelCase__												=     ''''''.join(lowerCamelCase_					)
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
																									UpperCAmelCase__												=     True
																									UpperCAmelCase__												=     self.tokenizer_class.from_pretrained(lowerCamelCase_		,			**lowerCamelCase_					)
																									UpperCAmelCase__												=     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_		,			**lowerCamelCase_					)

																									UpperCAmelCase__												=     tokenizer_p.encode(lowerCamelCase_		,			add_special_tokens=lowerCamelCase_					)
																									UpperCAmelCase__												=     tokenizer_r.encode(lowerCamelCase_		,			add_special_tokens=lowerCamelCase_					)

																									UpperCAmelCase__												=     tokenizer_r.convert_ids_to_tokens(lowerCamelCase_					)
																									UpperCAmelCase__												=     tokenizer_p.convert_ids_to_tokens(lowerCamelCase_					)

																									# it is expected that each Chinese character is not preceded by "##"
																									self.assertListEqual(lowerCamelCase_		,			lowerCamelCase_					)
																									self.assertListEqual(lowerCamelCase_		,			lowerCamelCase_					)

																									UpperCAmelCase__												=     False
																									UpperCAmelCase__												=     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_		,			**lowerCamelCase_					)
																									UpperCAmelCase__												=     self.tokenizer_class.from_pretrained(lowerCamelCase_		,			**lowerCamelCase_					)

																									UpperCAmelCase__												=     tokenizer_r.encode(lowerCamelCase_		,			add_special_tokens=lowerCamelCase_					)
																									UpperCAmelCase__												=     tokenizer_p.encode(lowerCamelCase_		,			add_special_tokens=lowerCamelCase_					)

																									UpperCAmelCase__												=     tokenizer_r.convert_ids_to_tokens(lowerCamelCase_					)
																									UpperCAmelCase__												=     tokenizer_p.convert_ids_to_tokens(lowerCamelCase_					)

																									# it is expected that only the first Chinese character is not preceded by "##".
																									UpperCAmelCase__												=     [
																									    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase_					)
																									]
																									self.assertListEqual(lowerCamelCase_		,			lowerCamelCase_					)
																									self.assertListEqual(lowerCamelCase_		,			lowerCamelCase_					)



"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_torch_available


__a      			= {"configuration_bert_generation": ["BertGenerationConfig"]}

try:
						if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__a      			= ["BertGenerationTokenizer"]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__a      			= [
						    "BertGenerationDecoder",
						    "BertGenerationEncoder",
						    "BertGenerationPreTrainedModel",
						    "load_tf_weights_in_bert_generation",
						]


if TYPE_CHECKING:
						from .configuration_bert_generation import BertGenerationConfig

						try:
												if not is_sentencepiece_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_bert_generation import BertGenerationTokenizer

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_bert_generation import (
												    BertGenerationDecoder,
												    BertGenerationEncoder,
												    BertGenerationPreTrainedModel,
												    load_tf_weights_in_bert_generation,
												)

else:
						import sys

						__a      			= _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)