




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class        UpperCAmelCase__				(							UpperCAmelCase__      ,		unittest.TestCase				):



     '''simple docstring'''
     _SCREAMING_SNAKE_CASE      :					str     					=    ElectraTokenizer
     _SCREAMING_SNAKE_CASE      :					List[Any]     					=    ElectraTokenizerFast
     _SCREAMING_SNAKE_CASE      :					Any     					=    True
     _SCREAMING_SNAKE_CASE      :					Tuple     					=    True
     _SCREAMING_SNAKE_CASE      :					List[Any]     					=    filter_non_english
     def        lowerCAmelCase__  (				self	):
            super().setUp()

            a			   =[
                """[UNK]""",
                """[CLS]""",
                """[SEP]""",
                """[PAD]""",
                """[MASK]""",
                """want""",
                """##want""",
                """##ed""",
                """wa""",
                """un""",
                """runn""",
                """##ing""",
                """,""",
                """low""",
                """lowest""",
            ]
            a			   =os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES["""vocab_file"""]	)
            with open(self.vocab_file ,    """w""" ,    encoding="""utf-8"""	) as vocab_writer:
                   vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase	):
            a			   ="""UNwant\u00E9d,running"""
            a			   ="""unwanted, running"""
            return input_text, output_text
     def        lowerCAmelCase__  (				self	):
            a			   =self.tokenizer_class(self.vocab_file	)

            a			   =tokenizer.tokenize("""UNwant\u00E9d,running"""	)
            self.assertListEqual(_lowerCAmelCase ,    ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]	)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase	) ,    [9, 6, 7, 12, 10, 11]	)
     def        lowerCAmelCase__  (				self	):
            if not self.test_rust_tokenizer:
                   return

            a			   =self.get_tokenizer()
            a			   =self.get_rust_tokenizer()

            a			   ="""UNwant\u00E9d,running"""

            a			   =tokenizer.tokenize(_lowerCAmelCase	)
            a			   =rust_tokenizer.tokenize(_lowerCAmelCase	)
            self.assertListEqual(_lowerCAmelCase ,    _lowerCAmelCase	)

            a			   =tokenizer.encode(_lowerCAmelCase ,    add_special_tokens=_lowerCAmelCase	)
            a			   =rust_tokenizer.encode(_lowerCAmelCase ,    add_special_tokens=_lowerCAmelCase	)
            self.assertListEqual(_lowerCAmelCase ,    _lowerCAmelCase	)

            a			   =self.get_rust_tokenizer()
            a			   =tokenizer.encode(_lowerCAmelCase	)
            a			   =rust_tokenizer.encode(_lowerCAmelCase	)
            self.assertListEqual(_lowerCAmelCase ,    _lowerCAmelCase	)

            # With lower casing
            a			   =self.get_tokenizer(do_lower_case=_lowerCAmelCase	)
            a			   =self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase	)

            a			   ="""UNwant\u00E9d,running"""

            a			   =tokenizer.tokenize(_lowerCAmelCase	)
            a			   =rust_tokenizer.tokenize(_lowerCAmelCase	)
            self.assertListEqual(_lowerCAmelCase ,    _lowerCAmelCase	)

            a			   =tokenizer.encode(_lowerCAmelCase ,    add_special_tokens=_lowerCAmelCase	)
            a			   =rust_tokenizer.encode(_lowerCAmelCase ,    add_special_tokens=_lowerCAmelCase	)
            self.assertListEqual(_lowerCAmelCase ,    _lowerCAmelCase	)

            a			   =self.get_rust_tokenizer()
            a			   =tokenizer.encode(_lowerCAmelCase	)
            a			   =rust_tokenizer.encode(_lowerCAmelCase	)
            self.assertListEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =BasicTokenizer()

            self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""	) ,    ["""ah""", """\u535A""", """\u63A8""", """zz"""]	)
     def        lowerCAmelCase__  (				self	):
            a			   =BasicTokenizer(do_lower_case=_lowerCAmelCase	)

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	) ,    ["""hello""", """!""", """how""", """are""", """you""", """?"""]	)
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	) ,    ["""hello"""]	)
     def        lowerCAmelCase__  (				self	):
            a			   =BasicTokenizer(do_lower_case=_lowerCAmelCase ,    strip_accents=_lowerCAmelCase	)

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	) ,    ["""hällo""", """!""", """how""", """are""", """you""", """?"""]	)
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	) ,    ["""h\u00E9llo"""]	)
     def        lowerCAmelCase__  (				self	):
            a			   =BasicTokenizer(do_lower_case=_lowerCAmelCase ,    strip_accents=_lowerCAmelCase	)

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	) ,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	) ,    ["""hello"""]	)
     def        lowerCAmelCase__  (				self	):
            a			   =BasicTokenizer(do_lower_case=_lowerCAmelCase	)

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	) ,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	) ,    ["""hello"""]	)
     def        lowerCAmelCase__  (				self	):
            a			   =BasicTokenizer(do_lower_case=_lowerCAmelCase	)

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	) ,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)
     def        lowerCAmelCase__  (				self	):
            a			   =BasicTokenizer(do_lower_case=_lowerCAmelCase ,    strip_accents=_lowerCAmelCase	)

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	) ,    ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)
     def        lowerCAmelCase__  (				self	):
            a			   =BasicTokenizer(do_lower_case=_lowerCAmelCase ,    strip_accents=_lowerCAmelCase	)

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	) ,    ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)
     def        lowerCAmelCase__  (				self	):
            a			   =BasicTokenizer(do_lower_case=_lowerCAmelCase ,    never_split=["""[UNK]"""]	)

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""	) ,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]	)
     def        lowerCAmelCase__  (				self	):
            a			   =["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

            a			   ={}
            for i, token in enumerate(_lowerCAmelCase	):
                   a			   =i
            a			   =WordpieceTokenizer(vocab=_lowerCAmelCase ,    unk_token="""[UNK]"""	)

            self.assertListEqual(tokenizer.tokenize(""""""	) ,    []	)

            self.assertListEqual(tokenizer.tokenize("""unwanted running"""	) ,    ["""un""", """##want""", """##ed""", """runn""", """##ing"""]	)

            self.assertListEqual(tokenizer.tokenize("""unwantedX running"""	) ,    ["""[UNK]""", """runn""", """##ing"""]	)
     def        lowerCAmelCase__  (				self	):
            self.assertTrue(_is_whitespace(""" """	)	)
            self.assertTrue(_is_whitespace("""\t"""	)	)
            self.assertTrue(_is_whitespace("""\r"""	)	)
            self.assertTrue(_is_whitespace("""\n"""	)	)
            self.assertTrue(_is_whitespace("""\u00A0"""	)	)

            self.assertFalse(_is_whitespace("""A"""	)	)
            self.assertFalse(_is_whitespace("""-"""	)	)
     def        lowerCAmelCase__  (				self	):
            self.assertTrue(_is_control("""\u0005"""	)	)

            self.assertFalse(_is_control("""A"""	)	)
            self.assertFalse(_is_control(""" """	)	)
            self.assertFalse(_is_control("""\t"""	)	)
            self.assertFalse(_is_control("""\r"""	)	)
     def        lowerCAmelCase__  (				self	):
            self.assertTrue(_is_punctuation("""-"""	)	)
            self.assertTrue(_is_punctuation("""$"""	)	)
            self.assertTrue(_is_punctuation("""`"""	)	)
            self.assertTrue(_is_punctuation("""."""	)	)

            self.assertFalse(_is_punctuation("""A"""	)	)
            self.assertFalse(_is_punctuation(""" """	)	)
     def        lowerCAmelCase__  (				self	):
            a			   =self.get_tokenizer()
            a			   =self.get_rust_tokenizer()

            # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
            self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase	) for t in ["""Test""", """\xad""", """test"""]] ,    [["""[UNK]"""], [], ["""[UNK]"""]]	)

            self.assertListEqual(
                [rust_tokenizer.tokenize(_lowerCAmelCase	) for t in ["""Test""", """\xad""", """test"""]] ,    [["""[UNK]"""], [], ["""[UNK]"""]]	)
     @slow
     def        lowerCAmelCase__  (				self	):
            a			   =self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""	)

            a			   =tokenizer.encode("""sequence builders""" ,    add_special_tokens=_lowerCAmelCase	)
            a			   =tokenizer.encode("""multi-sequence build""" ,    add_special_tokens=_lowerCAmelCase	)

            a			   =tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase	)
            a			   =tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase ,    _lowerCAmelCase	)

            assert encoded_sentence == [101] + text + [102]
            assert encoded_pair == [101] + text + [102] + text_a + [102]
     def        lowerCAmelCase__  (				self	):
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
                          a			   =self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase ,    **_lowerCAmelCase	)

                          a			   =F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                          a			   =tokenizer_r.encode_plus(
                              _lowerCAmelCase ,    return_attention_mask=_lowerCAmelCase ,    return_token_type_ids=_lowerCAmelCase ,    return_offsets_mapping=_lowerCAmelCase ,    add_special_tokens=_lowerCAmelCase ,    )

                          a			   =tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase ,    """do_lower_case"""	) else False
                          a			   =(
                              [
                                  ((0, 0), tokenizer_r.cls_token),
                                  ((0, 1), """A"""),
                                  ((1, 2), ""","""),
                                  ((3, 5), """na"""),
                                  ((5, 6), """##ï"""),
                                  ((6, 8), """##ve"""),
                                  ((9, 15), tokenizer_r.mask_token),
                                  ((16, 21), """Allen"""),
                                  ((21, 23), """##NL"""),
                                  ((23, 24), """##P"""),
                                  ((25, 33), """sentence"""),
                                  ((33, 34), """."""),
                                  ((0, 0), tokenizer_r.sep_token),
                              ]
                              if not do_lower_case
                              else [
                                  ((0, 0), tokenizer_r.cls_token),
                                  ((0, 1), """a"""),
                                  ((1, 2), ""","""),
                                  ((3, 8), """naive"""),
                                  ((9, 15), tokenizer_r.mask_token),
                                  ((16, 21), """allen"""),
                                  ((21, 23), """##nl"""),
                                  ((23, 24), """##p"""),
                                  ((25, 33), """sentence"""),
                                  ((33, 34), """."""),
                                  ((0, 0), tokenizer_r.sep_token),
                              ]
                          )

                          self.assertEqual(
                              [e[1] for e in expected_results] ,    tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]	)	)
                          self.assertEqual([e[0] for e in expected_results] ,    tokens["""offset_mapping"""]	)






     def        lowerCAmelCase__  (				self	):
            a			   =["""的""", """人""", """有"""]
            a			   ="""""".join(_lowerCAmelCase	)
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
                          a			   =True
                          a			   =self.tokenizer_class.from_pretrained(_lowerCAmelCase ,    **_lowerCAmelCase	)
                          a			   =self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase ,    **_lowerCAmelCase	)

                          a			   =tokenizer_p.encode(_lowerCAmelCase ,    add_special_tokens=_lowerCAmelCase	)
                          a			   =tokenizer_r.encode(_lowerCAmelCase ,    add_special_tokens=_lowerCAmelCase	)

                          a			   =tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase	)
                          a			   =tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase	)

                          # it is expected that each Chinese character is not preceded by "##"
                          self.assertListEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
                          self.assertListEqual(_lowerCAmelCase ,    _lowerCAmelCase	)

                          a			   =False
                          a			   =self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase ,    **_lowerCAmelCase	)
                          a			   =self.tokenizer_class.from_pretrained(_lowerCAmelCase ,    **_lowerCAmelCase	)

                          a			   =tokenizer_r.encode(_lowerCAmelCase ,    add_special_tokens=_lowerCAmelCase	)
                          a			   =tokenizer_p.encode(_lowerCAmelCase ,    add_special_tokens=_lowerCAmelCase	)

                          a			   =tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase	)
                          a			   =tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase	)

                          # it is expected that only the first Chinese character is not preceded by "##".
                          a			   =[
                              F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase	)
                          ]
                          self.assertListEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
                          self.assertListEqual(_lowerCAmelCase ,    _lowerCAmelCase	)






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


_lowerCamelCase							=      {'''configuration_cvt''': ['''CVT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''CvtConfig''']}


try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _lowerCamelCase							=      [
        '''CVT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''CvtForImageClassification''',
        '''CvtModel''',
        '''CvtPreTrainedModel''',
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _lowerCamelCase							=      [
        '''TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''TFCvtForImageClassification''',
        '''TFCvtModel''',
        '''TFCvtPreTrainedModel''',
    ]

if TYPE_CHECKING:
    from .configuration_cvt import CVT_PRETRAINED_CONFIG_ARCHIVE_MAP, CvtConfig

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_cvt import (
            CVT_PRETRAINED_MODEL_ARCHIVE_LIST,
            CvtForImageClassification,
            CvtModel,
            CvtPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_cvt import (
            TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFCvtForImageClassification,
            TFCvtModel,
            TFCvtPreTrainedModel,
        )


else:
    import sys

    _lowerCamelCase							=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
