





import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 							snake_case_		(      UpperCAmelCase__    ,       unittest.TestCase):
		lowerCamelCase     :str							=  ElectraTokenizer
		lowerCamelCase     :List[Any]							=  ElectraTokenizerFast
		lowerCamelCase     :Any							=  True
		lowerCamelCase     :Tuple							=  True
		lowerCamelCase     :List[Any]							=  filter_non_english







		def       __lowercase (		self )					-> List[str]:
				super().setUp()

				lowerCamelCase :			Optional[Any]    		=[
				    '''[UNK]''',
				    '''[CLS]''',
				    '''[SEP]''',
				    '''[PAD]''',
				    '''[MASK]''',
				    '''want''',
				    '''##want''',
				    '''##ed''',
				    '''wa''',
				    '''un''',
				    '''runn''',
				    '''##ing''',
				    ''',''',
				    '''low''',
				    '''lowest''',
				]
				lowerCamelCase :			int    		=os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES['''vocab_file'''] )
				with open(self.vocab_file      ,  '''w'''      ,  encoding='''utf-8''' ) as vocab_writer:
						vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )







		def       __lowercase (		self      ,  __lowercase )					-> Dict:
				lowerCamelCase :			List[Any]    		='''UNwant\u00E9d,running'''
				lowerCamelCase :			int    		='''unwanted, running'''
				return input_text, output_text







		def       __lowercase (		self )					-> Dict:
				lowerCamelCase :			List[Any]    		=self.tokenizer_class(self.vocab_file )

				lowerCamelCase :			Any    		=tokenizer.tokenize('''UNwant\u00E9d,running''' )
				self.assertListEqual(_lowerCAmelCase      ,  ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''] )
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase )      ,  [9, 6, 7, 1_2, 1_0, 1_1] )







		def       __lowercase (		self )					-> int:
				if not self.test_rust_tokenizer:
						return

				lowerCamelCase :			str    		=self.get_tokenizer()
				lowerCamelCase :			Union[str, Any]    		=self.get_rust_tokenizer()

				lowerCamelCase :			str    		='''UNwant\u00E9d,running'''

				lowerCamelCase :			List[Any]    		=tokenizer.tokenize(_lowerCAmelCase )
				lowerCamelCase :			Dict    		=rust_tokenizer.tokenize(_lowerCAmelCase )
				self.assertListEqual(_lowerCAmelCase      ,  _lowerCAmelCase )

				lowerCamelCase :			Tuple    		=tokenizer.encode(_lowerCAmelCase      ,  add_special_tokens=_lowerCAmelCase )
				lowerCamelCase :			int    		=rust_tokenizer.encode(_lowerCAmelCase      ,  add_special_tokens=_lowerCAmelCase )
				self.assertListEqual(_lowerCAmelCase      ,  _lowerCAmelCase )

				lowerCamelCase :			Dict    		=self.get_rust_tokenizer()
				lowerCamelCase :			List[Any]    		=tokenizer.encode(_lowerCAmelCase )
				lowerCamelCase :			List[str]    		=rust_tokenizer.encode(_lowerCAmelCase )
				self.assertListEqual(_lowerCAmelCase      ,  _lowerCAmelCase )

				# With lower casing
				lowerCamelCase :			Optional[Any]    		=self.get_tokenizer(do_lower_case=_lowerCAmelCase )
				lowerCamelCase :			Any    		=self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase )

				lowerCamelCase :			List[str]    		='''UNwant\u00E9d,running'''

				lowerCamelCase :			Optional[int]    		=tokenizer.tokenize(_lowerCAmelCase )
				lowerCamelCase :			List[Any]    		=rust_tokenizer.tokenize(_lowerCAmelCase )
				self.assertListEqual(_lowerCAmelCase      ,  _lowerCAmelCase )

				lowerCamelCase :			Union[str, Any]    		=tokenizer.encode(_lowerCAmelCase      ,  add_special_tokens=_lowerCAmelCase )
				lowerCamelCase :			int    		=rust_tokenizer.encode(_lowerCAmelCase      ,  add_special_tokens=_lowerCAmelCase )
				self.assertListEqual(_lowerCAmelCase      ,  _lowerCAmelCase )

				lowerCamelCase :			Tuple    		=self.get_rust_tokenizer()
				lowerCamelCase :			int    		=tokenizer.encode(_lowerCAmelCase )
				lowerCamelCase :			List[str]    		=rust_tokenizer.encode(_lowerCAmelCase )
				self.assertListEqual(_lowerCAmelCase      ,  _lowerCAmelCase )







		def       __lowercase (		self )					-> Tuple:
				lowerCamelCase :			Union[str, Any]    		=BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''' )      ,  ['''ah''', '''\u535A''', '''\u63A8''', '''zz'''] )







		def       __lowercase (		self )					-> Optional[Any]:
				lowerCamelCase :			Tuple    		=BasicTokenizer(do_lower_case=_lowerCAmelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )      ,  ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )      ,  ['''hello'''] )







		def       __lowercase (		self )					-> int:
				lowerCamelCase :			Optional[int]    		=BasicTokenizer(do_lower_case=_lowerCAmelCase      ,  strip_accents=_lowerCAmelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )      ,  ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )      ,  ['''h\u00E9llo'''] )







		def       __lowercase (		self )					-> Any:
				lowerCamelCase :			Tuple    		=BasicTokenizer(do_lower_case=_lowerCAmelCase      ,  strip_accents=_lowerCAmelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )      ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )      ,  ['''hello'''] )







		def       __lowercase (		self )					-> int:
				lowerCamelCase :			int    		=BasicTokenizer(do_lower_case=_lowerCAmelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )      ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )      ,  ['''hello'''] )







		def       __lowercase (		self )					-> Optional[int]:
				lowerCamelCase :			Optional[Any]    		=BasicTokenizer(do_lower_case=_lowerCAmelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )      ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )







		def       __lowercase (		self )					-> Union[str, Any]:
				lowerCamelCase :			Optional[Any]    		=BasicTokenizer(do_lower_case=_lowerCAmelCase      ,  strip_accents=_lowerCAmelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )      ,  ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )







		def       __lowercase (		self )					-> Union[str, Any]:
				lowerCamelCase :			Optional[int]    		=BasicTokenizer(do_lower_case=_lowerCAmelCase      ,  strip_accents=_lowerCAmelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )      ,  ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )







		def       __lowercase (		self )					-> Tuple:
				lowerCamelCase :			Dict    		=BasicTokenizer(do_lower_case=_lowerCAmelCase      ,  never_split=['''[UNK]'''] )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''' )      ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''] )







		def       __lowercase (		self )					-> Union[str, Any]:
				lowerCamelCase :			List[str]    		=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

				lowerCamelCase :			int    		={}
				for i, token in enumerate(_lowerCAmelCase ):
						lowerCamelCase :			Optional[Any]    		=i
				lowerCamelCase :			Union[str, Any]    		=WordpieceTokenizer(vocab=_lowerCAmelCase      ,  unk_token='''[UNK]''' )

				self.assertListEqual(tokenizer.tokenize('''''' )      ,  [] )

				self.assertListEqual(tokenizer.tokenize('''unwanted running''' )      ,  ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''] )

				self.assertListEqual(tokenizer.tokenize('''unwantedX running''' )      ,  ['''[UNK]''', '''runn''', '''##ing'''] )







		def       __lowercase (		self )					-> Union[str, Any]:
				self.assertTrue(_is_whitespace(''' ''' ) )
				self.assertTrue(_is_whitespace('''\t''' ) )
				self.assertTrue(_is_whitespace('''\r''' ) )
				self.assertTrue(_is_whitespace('''\n''' ) )
				self.assertTrue(_is_whitespace('''\u00A0''' ) )

				self.assertFalse(_is_whitespace('''A''' ) )
				self.assertFalse(_is_whitespace('''-''' ) )







		def       __lowercase (		self )					-> Tuple:
				self.assertTrue(_is_control('''\u0005''' ) )

				self.assertFalse(_is_control('''A''' ) )
				self.assertFalse(_is_control(''' ''' ) )
				self.assertFalse(_is_control('''\t''' ) )
				self.assertFalse(_is_control('''\r''' ) )







		def       __lowercase (		self )					-> List[str]:
				self.assertTrue(_is_punctuation('''-''' ) )
				self.assertTrue(_is_punctuation('''$''' ) )
				self.assertTrue(_is_punctuation('''`''' ) )
				self.assertTrue(_is_punctuation('''.''' ) )

				self.assertFalse(_is_punctuation('''A''' ) )
				self.assertFalse(_is_punctuation(''' ''' ) )







		def       __lowercase (		self )					-> List[str]:
				lowerCamelCase :			List[Any]    		=self.get_tokenizer()
				lowerCamelCase :			int    		=self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase ) for t in ['''Test''', '''\xad''', '''test''']]      ,  [['''[UNK]'''], [], ['''[UNK]''']] )

				self.assertListEqual(
				    [rust_tokenizer.tokenize(_lowerCAmelCase ) for t in ['''Test''', '''\xad''', '''test''']]      ,  [['''[UNK]'''], [], ['''[UNK]''']] )







		@slow
		def       __lowercase (		self )					-> Optional[Any]:
				lowerCamelCase :			Any    		=self.tokenizer_class.from_pretrained('''google/electra-base-discriminator''' )

				lowerCamelCase :			Any    		=tokenizer.encode('''sequence builders'''      ,  add_special_tokens=_lowerCAmelCase )
				lowerCamelCase :			int    		=tokenizer.encode('''multi-sequence build'''      ,  add_special_tokens=_lowerCAmelCase )

				lowerCamelCase :			Any    		=tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase )
				lowerCamelCase :			Any    		=tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase      ,  _lowerCAmelCase )

				assert encoded_sentence == [1_0_1] + text + [1_0_2]
				assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]







		def       __lowercase (		self )					-> Dict:
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
								lowerCamelCase :			Optional[Any]    		=self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase      ,  **_lowerCAmelCase )

								lowerCamelCase :			List[str]    		=F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
								lowerCamelCase :			Dict    		=tokenizer_r.encode_plus(
								    _lowerCAmelCase      ,  return_attention_mask=_lowerCAmelCase      ,  return_token_type_ids=_lowerCAmelCase      ,  return_offsets_mapping=_lowerCAmelCase      ,  add_special_tokens=_lowerCAmelCase      ,  )

								lowerCamelCase :			Optional[int]    		=tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase      ,  '''do_lower_case''' ) else False
								lowerCamelCase :			Dict    		=(
								    [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), '''A'''),
								        ((1, 2), ''','''),
								        ((3, 5), '''na'''),
								        ((5, 6), '''##ï'''),
								        ((6, 8), '''##ve'''),
								        ((9, 1_5), tokenizer_r.mask_token),
								        ((1_6, 2_1), '''Allen'''),
								        ((2_1, 2_3), '''##NL'''),
								        ((2_3, 2_4), '''##P'''),
								        ((2_5, 3_3), '''sentence'''),
								        ((3_3, 3_4), '''.'''),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								    if not do_lower_case
								    else [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), '''a'''),
								        ((1, 2), ''','''),
								        ((3, 8), '''naive'''),
								        ((9, 1_5), tokenizer_r.mask_token),
								        ((1_6, 2_1), '''allen'''),
								        ((2_1, 2_3), '''##nl'''),
								        ((2_3, 2_4), '''##p'''),
								        ((2_5, 3_3), '''sentence'''),
								        ((3_3, 3_4), '''.'''),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								)

								self.assertEqual(
								    [e[1] for e in expected_results]      ,  tokenizer_r.convert_ids_to_tokens(tokens['''input_ids'''] ) )
								self.assertEqual([e[0] for e in expected_results]      ,  tokens['''offset_mapping'''] )







		def       __lowercase (		self )					-> Any:
				lowerCamelCase :			str    		=['''的''', '''人''', '''有''']
				lowerCamelCase :			Any    		=''''''.join(_lowerCAmelCase )
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
								lowerCamelCase :			Dict    		=True
								lowerCamelCase :			Any    		=self.tokenizer_class.from_pretrained(_lowerCAmelCase      ,  **_lowerCAmelCase )
								lowerCamelCase :			Any    		=self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase      ,  **_lowerCAmelCase )

								lowerCamelCase :			Optional[Any]    		=tokenizer_p.encode(_lowerCAmelCase      ,  add_special_tokens=_lowerCAmelCase )
								lowerCamelCase :			str    		=tokenizer_r.encode(_lowerCAmelCase      ,  add_special_tokens=_lowerCAmelCase )

								lowerCamelCase :			Optional[Any]    		=tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase )
								lowerCamelCase :			Tuple    		=tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase )

								# it is expected that each Chinese character is not preceded by "##"
								self.assertListEqual(_lowerCAmelCase      ,  _lowerCAmelCase )
								self.assertListEqual(_lowerCAmelCase      ,  _lowerCAmelCase )

								lowerCamelCase :			Optional[int]    		=False
								lowerCamelCase :			Dict    		=self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase      ,  **_lowerCAmelCase )
								lowerCamelCase :			Any    		=self.tokenizer_class.from_pretrained(_lowerCAmelCase      ,  **_lowerCAmelCase )

								lowerCamelCase :			int    		=tokenizer_r.encode(_lowerCAmelCase      ,  add_special_tokens=_lowerCAmelCase )
								lowerCamelCase :			int    		=tokenizer_p.encode(_lowerCAmelCase      ,  add_special_tokens=_lowerCAmelCase )

								lowerCamelCase :			List[Any]    		=tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase )
								lowerCamelCase :			Optional[int]    		=tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase )

								# it is expected that only the first Chinese character is not preceded by "##".
								lowerCamelCase :			str    		=[
								    F"##{token}" if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase )
								]
								self.assertListEqual(_lowerCAmelCase      ,  _lowerCAmelCase )
								self.assertListEqual(_lowerCAmelCase      ,  _lowerCAmelCase )







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


_lowerCamelCase							=      {'''configuration_cvt''': ['''CVT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''CvtConfig''']}


try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _lowerCamelCase							=      [
        '''CVT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''CvtForImageClassification''',
        '''CvtModel''',
        '''CvtPreTrainedModel''',
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _lowerCamelCase							=      [
        '''TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''TFCvtForImageClassification''',
        '''TFCvtModel''',
        '''TFCvtPreTrainedModel''',
    ]

if TYPE_CHECKING:
    from .configuration_cvt import CVT_PRETRAINED_CONFIG_ARCHIVE_MAP, CvtConfig

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_cvt import (
            CVT_PRETRAINED_MODEL_ARCHIVE_LIST,
            CvtForImageClassification,
            CvtModel,
            CvtPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_cvt import (
            TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFCvtForImageClassification,
            TFCvtModel,
            TFCvtPreTrainedModel,
        )


else:
    import sys

    _lowerCamelCase							=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
