


import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


A       :		int    =   get_tests_dir('fixtures/test_sentencepiece.model')



@require_sentencepiece
@require_tokenizers
class 							A    (       UpperCamelCase__ ,  unittest.TestCase							):
					'''simple docstring'''






					A__			= XLNetTokenizer
					A__			= XLNetTokenizerFast
					A__			= True
					A__			= True

					def   lowerCamelCase__     (self    :						Union[str, Any]		)						->   Dict:




							"""simple docstring"""


							super().setUp()

							# We have a SentencePiece fixture for testing
							lowercase__     			=		XLNetTokenizer(__a      ,					keep_accents=__a		)
							tokenizer.sanitize_special_tokens()
							tokenizer.save_pretrained(self.tmpdirname		)

					def   lowerCamelCase__     (self    :						Dict		)						->   int:




							"""simple docstring"""


							lowercase__     			=		"""<s>"""
							lowercase__     			=		1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a		)      ,					__a		)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a		)      ,					__a		)

					def   lowerCamelCase__     (self    :						Optional[Any]		)						->   Dict:




							"""simple docstring"""


							lowercase__     			=		list(self.get_tokenizer().get_vocab().keys()		)

							self.assertEqual(vocab_keys[0]      ,					"""<unk>"""		)
							self.assertEqual(vocab_keys[1]      ,					"""<s>"""		)
							self.assertEqual(vocab_keys[-1]      ,					"""<eod>"""		)
							self.assertEqual(len(__a		)      ,					1006		)

					def   lowerCamelCase__     (self    :						Tuple		)						->   Optional[Any]:




							"""simple docstring"""


							self.assertEqual(self.get_tokenizer().vocab_size      ,					1000		)

					def   lowerCamelCase__     (self    :						Tuple		)						->   int:




							"""simple docstring"""


							lowercase__     			=		XLNetTokenizer(__a      ,					keep_accents=__a		)

							lowercase__     			=		tokenizer.tokenize("""This is a test"""		)
							self.assertListEqual(__a      ,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]		)

							self.assertListEqual(tokenizer.convert_tokens_to_ids(__a		)      ,					[285, 46, 10, 170, 382]		)

							lowercase__     			=		tokenizer.tokenize("""I was born in 92000, and this is falsé."""		)
							self.assertListEqual(
							    __a      ,					[
							        SPIECE_UNDERLINE + """I""",
							        SPIECE_UNDERLINE + """was""",
							        SPIECE_UNDERLINE + """b""",
							        """or""",
							        """n""",
							        SPIECE_UNDERLINE + """in""",
							        SPIECE_UNDERLINE + """""",
							        """9""",
							        """2""",
							        """0""",
							        """0""",
							        """0""",
							        """,""",
							        SPIECE_UNDERLINE + """and""",
							        SPIECE_UNDERLINE + """this""",
							        SPIECE_UNDERLINE + """is""",
							        SPIECE_UNDERLINE + """f""",
							        """al""",
							        """s""",
							        """é""",
							        """.""",
							    ]      ,					)
							lowercase__     			=		tokenizer.convert_tokens_to_ids(__a		)
							self.assertListEqual(__a      ,					[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]		)

							lowercase__     			=		tokenizer.convert_ids_to_tokens(__a		)
							self.assertListEqual(
							    __a      ,					[
							        SPIECE_UNDERLINE + """I""",
							        SPIECE_UNDERLINE + """was""",
							        SPIECE_UNDERLINE + """b""",
							        """or""",
							        """n""",
							        SPIECE_UNDERLINE + """in""",
							        SPIECE_UNDERLINE + """""",
							        """<unk>""",
							        """2""",
							        """0""",
							        """0""",
							        """0""",
							        """,""",
							        SPIECE_UNDERLINE + """and""",
							        SPIECE_UNDERLINE + """this""",
							        SPIECE_UNDERLINE + """is""",
							        SPIECE_UNDERLINE + """f""",
							        """al""",
							        """s""",
							        """<unk>""",
							        """.""",
							    ]      ,					)

					def   lowerCamelCase__     (self    :						Any		)						->   Any:




							"""simple docstring"""


							lowercase__     			=		XLNetTokenizer(__a      ,					do_lower_case=__a		)
							lowercase__     			=		tokenizer.tokenize("""I was born in 92000, and this is falsé."""		)
							self.assertListEqual(
							    __a      ,					[
							        SPIECE_UNDERLINE + """""",
							        """i""",
							        SPIECE_UNDERLINE + """was""",
							        SPIECE_UNDERLINE + """b""",
							        """or""",
							        """n""",
							        SPIECE_UNDERLINE + """in""",
							        SPIECE_UNDERLINE + """""",
							        """9""",
							        """2""",
							        """0""",
							        """0""",
							        """0""",
							        """,""",
							        SPIECE_UNDERLINE + """and""",
							        SPIECE_UNDERLINE + """this""",
							        SPIECE_UNDERLINE + """is""",
							        SPIECE_UNDERLINE + """f""",
							        """al""",
							        """se""",
							        """.""",
							    ]      ,					)
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)      ,					["""▁he""", """ll""", """o"""]		)

					def   lowerCamelCase__     (self    :						int		)						->   Dict:




							"""simple docstring"""


							lowercase__     			=		XLNetTokenizer(__a      ,					do_lower_case=__a		)
							lowercase__     			=		tokenizer.tokenize("""I was born in 92000, and this is falsé."""		)
							self.assertListEqual(
							    __a      ,					[
							        SPIECE_UNDERLINE + """I""",
							        SPIECE_UNDERLINE + """was""",
							        SPIECE_UNDERLINE + """b""",
							        """or""",
							        """n""",
							        SPIECE_UNDERLINE + """in""",
							        SPIECE_UNDERLINE + """""",
							        """9""",
							        """2""",
							        """0""",
							        """0""",
							        """0""",
							        """,""",
							        SPIECE_UNDERLINE + """and""",
							        SPIECE_UNDERLINE + """this""",
							        SPIECE_UNDERLINE + """is""",
							        SPIECE_UNDERLINE + """f""",
							        """al""",
							        """se""",
							        """.""",
							    ]      ,					)

					@slow
					def   lowerCamelCase__     (self    :						str		)						->   Optional[Any]:




							"""simple docstring"""


							lowercase__     			=		XLNetTokenizer.from_pretrained("""xlnet-base-cased"""		)

							lowercase__     			=		tokenizer.encode("""sequence builders"""      ,					add_special_tokens=__a		)
							lowercase__     			=		tokenizer.encode("""multi-sequence build"""      ,					add_special_tokens=__a		)

							lowercase__     			=		tokenizer.build_inputs_with_special_tokens(__a		)
							lowercase__     			=		tokenizer.build_inputs_with_special_tokens(__a      ,					__a		)

							assert encoded_sentence == text + [4, 3]
							assert encoded_pair == text + [4] + text_a + [4, 3]


					@slow
					def   lowerCamelCase__     (self    :						Optional[int]		)						->   List[str]:




							"""simple docstring"""


							lowercase__     			=		{"""input_ids""": [[17, 2_1442, 270, 17, 10, 1_4645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 2_2018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 1_4431, 13, 5500, 11, 1176, 580, 13, 1_6819, 4797, 23, 17, 10, 1_7135, 658, 19, 457, 7932, 13, 184, 19, 3154, 1_7135, 6468, 19, 1404, 1_2269, 19, 4229, 5356, 1_6264, 46, 19, 17, 2_0545, 1_0395, 9, 9, 9, 11, 28, 6421, 9531, 2_0729, 17, 10, 353, 1_7022, 11, 21, 6421, 9531, 1_6949, 17, 10, 1_1509, 753, 11, 33, 95, 2421, 7385, 956, 1_4431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 2_4738, 19, 1_3203, 658, 218, 787, 21, 430, 1_8482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 2_2178, 27, 1064, 22, 956, 13, 1_1101, 1429, 5854, 2_4313, 1_8953, 40, 422, 2_4366, 68, 1758, 37, 1_0483, 1_4257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 1_3894, 3380, 23, 95, 18, 1_7634, 2288, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=__a      ,					model_name="""xlnet-base-cased"""      ,					revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""      ,					)


'''simple docstring'''





import inspect
import unittest

import numpy as np

from transformers import ViTConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor


if is_flax_available():
    import jax

    from transformers.models.vit.modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel




class        __A      ( unittest.TestCase	):



    def __init__(self    :    str   ,			__a    :    Optional[Any]   ,			__a    :    Optional[Any]=13   ,			__a    :    int=30   ,			__a    :    Union[str, Any]=2   ,			__a    :    Dict=3   ,			__a    :    List[Any]=True   ,			__a    :    Optional[Any]=True   ,			__a    :    List[Any]=32   ,			__a    :    Any=5   ,			__a    :    str=4   ,			__a    :    Optional[int]=37   ,			__a    :    Optional[int]="gelu"   ,			__a    :    List[str]=0.1   ,			__a    :    Tuple=0.1   ,			__a    :    List[str]=10   ,			__a    :    Optional[int]=0.02   ,			):
         UpperCAmelCase_           =   parent
         UpperCAmelCase_           =   batch_size
         UpperCAmelCase_           =   image_size
         UpperCAmelCase_           =   patch_size
         UpperCAmelCase_           =   num_channels
         UpperCAmelCase_           =   is_training
         UpperCAmelCase_           =   use_labels
         UpperCAmelCase_           =   hidden_size
         UpperCAmelCase_           =   num_hidden_layers
         UpperCAmelCase_           =   num_attention_heads
         UpperCAmelCase_           =   intermediate_size
         UpperCAmelCase_           =   hidden_act
         UpperCAmelCase_           =   hidden_dropout_prob
         UpperCAmelCase_           =   attention_probs_dropout_prob
         UpperCAmelCase_           =   type_sequence_label_size
         UpperCAmelCase_           =   initializer_range

         # in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
         UpperCAmelCase_           =   (image_size // patch_size) ** 2
         UpperCAmelCase_           =   num_patches + 1



    def 					_lowercase						(self    :    Any ):
         UpperCAmelCase_           =   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

         UpperCAmelCase_           =   ViTConfig(
             image_size=self.image_size   ,			patch_size=self.patch_size   ,			num_channels=self.num_channels   ,			hidden_size=self.hidden_size   ,			num_hidden_layers=self.num_hidden_layers   ,			num_attention_heads=self.num_attention_heads   ,			intermediate_size=self.intermediate_size   ,			hidden_act=self.hidden_act   ,			hidden_dropout_prob=self.hidden_dropout_prob   ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,			is_decoder=__a   ,			initializer_range=self.initializer_range   ,			)

         return config, pixel_values



    def 					_lowercase						(self    :    Dict   ,			__a    :    Any   ,			__a    :    List[Any] ):
         UpperCAmelCase_           =   FlaxViTModel(config=__a )
         UpperCAmelCase_           =   model(__a )
         # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token)
         UpperCAmelCase_           =   (self.image_size, self.image_size)
         UpperCAmelCase_           =   (self.patch_size, self.patch_size)
         UpperCAmelCase_           =   (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
         self.parent.assertEqual(result.last_hidden_state.shape   ,			(self.batch_size, num_patches + 1, self.hidden_size) )



    def 					_lowercase						(self    :    Tuple   ,			__a    :    str   ,			__a    :    Any ):
         UpperCAmelCase_           =   self.type_sequence_label_size
         UpperCAmelCase_           =   FlaxViTForImageClassification(config=__a )
         UpperCAmelCase_           =   model(__a )
         self.parent.assertEqual(result.logits.shape   ,			(self.batch_size, self.type_sequence_label_size) )

         # test greyscale images
         UpperCAmelCase_           =   1
         UpperCAmelCase_           =   FlaxViTForImageClassification(__a )

         UpperCAmelCase_           =   floats_tensor([self.batch_size, 1, self.image_size, self.image_size] )
         UpperCAmelCase_           =   model(__a )






    def 					_lowercase						(self    :    Optional[Any] ):
         UpperCAmelCase_           =   self.prepare_config_and_inputs()
         (
             (
             UpperCAmelCase_
         )    , (
             UpperCAmelCase_
         )    , 
         )           =   config_and_inputs
         UpperCAmelCase_           =   {"pixel_values": pixel_values}
         return config, inputs_dict




@require_flax
class        __A      ( UpperCamelCase__		,	unittest.TestCase	):
    a__       :						Tuple          =  (FlaxViTModel, FlaxViTForImageClassification) if is_flax_available() else ()



    def 					_lowercase						(self    :    Any ):
         UpperCAmelCase_           =   FlaxViTModelTester(self )
         UpperCAmelCase_           =   ConfigTester(self   ,			config_class=__a   ,			has_text_modality=__a   ,			hidden_size=37 )



    def 					_lowercase						(self    :    Tuple ):
         self.config_tester.run_common_tests()



    def 					_lowercase						(self    :    str ):
         UpperCAmelCase_           =   self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*__a )



    def 					_lowercase						(self    :    str ):
         UpperCAmelCase_           =   self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_image_classification(*__a )



    def 					_lowercase						(self    :    Tuple ):
         UpperCAmelCase_    , UpperCAmelCase_           =   self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
              UpperCAmelCase_           =   model_class(__a )
              UpperCAmelCase_           =   inspect.signature(model.__call__ )
              # signature.parameters is an OrderedDict => so arg_names order is deterministic
              UpperCAmelCase_           =   [*signature.parameters.keys()]

              UpperCAmelCase_           =   ["pixel_values"]
              self.assertListEqual(arg_names[:1]   ,			__a )



    def 					_lowercase						(self    :    Optional[Any] ):
         UpperCAmelCase_    , UpperCAmelCase_           =   self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
              with self.subTest(model_class.__name__ ):
                   UpperCAmelCase_           =   self._prepare_for_class(__a   ,			__a )
                   UpperCAmelCase_           =   model_class(__a )

                   @jax.jit
                   def model_jitted(__a    :    Tuple   ,			**__a    :    List[Any] ):
                        return model(pixel_values=__a   ,			**__a )

                   with self.subTest("JIT Enabled" ):
                        UpperCAmelCase_           =   model_jitted(**__a ).to_tuple()

                   with self.subTest("JIT Disabled" ):
                        with jax.disable_jit():
                             UpperCAmelCase_           =   model_jitted(**__a ).to_tuple()

                   self.assertEqual(len(__a )   ,			len(__a ) )
                   for jitted_output, output in zip(__a   ,			__a ):
                        self.assertEqual(jitted_output.shape   ,			output.shape )






    @slow
    def 					_lowercase						(self    :    Tuple ):
         for model_class_name in self.all_model_classes:
              UpperCAmelCase_           =   model_class_name.from_pretrained("google/vit-base-patch16-224" )
              UpperCAmelCase_           =   model(np.ones((1, 3, 224, 224) ) )
              self.assertIsNotNone(__a )




