




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__       :      Union[str, Any]  =   get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
@require_tokenizers
class 		UpperCamelCase__	(lowerCAmelCase__    ,				unittest.TestCase     ):

    '''simple docstring'''



    lowerCamelCase_      :	Optional[Any]	     =      XLNetTokenizer
    lowerCamelCase_      :	Union[str, Any]	     =      XLNetTokenizerFast
    lowerCamelCase_      :	Optional[int]	     =      True
    lowerCamelCase_      :	Optional[int]	     =      True







    def     _lowercase     (    self     )     ->      str:
           super().setUp()

           # We have a SentencePiece fixture for testing
           lowerCamelCase   :      int			 =					XLNetTokenizer(UpperCamelCase__     ,  keep_accents=UpperCamelCase__     )
           tokenizer.sanitize_special_tokens()
           tokenizer.save_pretrained(self.tmpdirname     )







    def     _lowercase     (    self     )     ->      str:
           lowerCamelCase   :      str			 =					"<s>"
           lowerCamelCase   :      Any			 =					1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__     )     ,  UpperCamelCase__     )
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__     )     ,  UpperCamelCase__     )







    def     _lowercase     (    self     )     ->      Any:
           lowerCamelCase   :      Union[str, Any]			 =					list(self.get_tokenizer().get_vocab().keys()     )

           self.assertEqual(vocab_keys[0]     ,  "<unk>"     )
           self.assertEqual(vocab_keys[1]     ,  "<s>"     )
           self.assertEqual(vocab_keys[-1]     ,  "<eod>"     )
           self.assertEqual(len(UpperCamelCase__     )     ,  1006     )







    def     _lowercase     (    self     )     ->      Optional[int]:
           self.assertEqual(self.get_tokenizer().vocab_size     ,  1000     )







    def     _lowercase     (    self     )     ->      List[Any]:
           lowerCamelCase   :      int			 =					XLNetTokenizer(UpperCamelCase__     ,  keep_accents=UpperCamelCase__     )

           lowerCamelCase   :      str			 =					tokenizer.tokenize("This is a test"     )
           self.assertListEqual(UpperCamelCase__     ,  ["▁This", "▁is", "▁a", "▁t", "est"]     )

           self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__     )     ,  [285, 46, 10, 170, 382]     )

           lowerCamelCase   :      int			 =					tokenizer.tokenize("I was born in 92000, and this is falsé."     )
           self.assertListEqual(
               UpperCamelCase__     ,  [
                   SPIECE_UNDERLINE + "I",
                   SPIECE_UNDERLINE + "was",
                   SPIECE_UNDERLINE + "b",
                   "or",
                   "n",
                   SPIECE_UNDERLINE + "in",
                   SPIECE_UNDERLINE + "",
                   "9",
                   "2",
                   "0",
                   "0",
                   "0",
                   ",",
                   SPIECE_UNDERLINE + "and",
                   SPIECE_UNDERLINE + "this",
                   SPIECE_UNDERLINE + "is",
                   SPIECE_UNDERLINE + "f",
                   "al",
                   "s",
                   "é",
                   ".",
               ]     ,  )
           lowerCamelCase   :      int			 =					tokenizer.convert_tokens_to_ids(UpperCamelCase__     )
           self.assertListEqual(UpperCamelCase__     ,  [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]     )

           lowerCamelCase   :      Dict			 =					tokenizer.convert_ids_to_tokens(UpperCamelCase__     )
           self.assertListEqual(
               UpperCamelCase__     ,  [
                   SPIECE_UNDERLINE + "I",
                   SPIECE_UNDERLINE + "was",
                   SPIECE_UNDERLINE + "b",
                   "or",
                   "n",
                   SPIECE_UNDERLINE + "in",
                   SPIECE_UNDERLINE + "",
                   "<unk>",
                   "2",
                   "0",
                   "0",
                   "0",
                   ",",
                   SPIECE_UNDERLINE + "and",
                   SPIECE_UNDERLINE + "this",
                   SPIECE_UNDERLINE + "is",
                   SPIECE_UNDERLINE + "f",
                   "al",
                   "s",
                   "<unk>",
                   ".",
               ]     ,  )







    def     _lowercase     (    self     )     ->      Any:
           lowerCamelCase   :      Union[str, Any]			 =					XLNetTokenizer(UpperCamelCase__     ,  do_lower_case=UpperCamelCase__     )
           lowerCamelCase   :      Dict			 =					tokenizer.tokenize("I was born in 92000, and this is falsé."     )
           self.assertListEqual(
               UpperCamelCase__     ,  [
                   SPIECE_UNDERLINE + "",
                   "i",
                   SPIECE_UNDERLINE + "was",
                   SPIECE_UNDERLINE + "b",
                   "or",
                   "n",
                   SPIECE_UNDERLINE + "in",
                   SPIECE_UNDERLINE + "",
                   "9",
                   "2",
                   "0",
                   "0",
                   "0",
                   ",",
                   SPIECE_UNDERLINE + "and",
                   SPIECE_UNDERLINE + "this",
                   SPIECE_UNDERLINE + "is",
                   SPIECE_UNDERLINE + "f",
                   "al",
                   "se",
                   ".",
               ]     ,  )
           self.assertListEqual(tokenizer.tokenize("H\u00E9llo"     )     ,  ["▁he", "ll", "o"]     )







    def     _lowercase     (    self     )     ->      int:
           lowerCamelCase   :      str			 =					XLNetTokenizer(UpperCamelCase__     ,  do_lower_case=UpperCamelCase__     )
           lowerCamelCase   :      Optional[int]			 =					tokenizer.tokenize("I was born in 92000, and this is falsé."     )
           self.assertListEqual(
               UpperCamelCase__     ,  [
                   SPIECE_UNDERLINE + "I",
                   SPIECE_UNDERLINE + "was",
                   SPIECE_UNDERLINE + "b",
                   "or",
                   "n",
                   SPIECE_UNDERLINE + "in",
                   SPIECE_UNDERLINE + "",
                   "9",
                   "2",
                   "0",
                   "0",
                   "0",
                   ",",
                   SPIECE_UNDERLINE + "and",
                   SPIECE_UNDERLINE + "this",
                   SPIECE_UNDERLINE + "is",
                   SPIECE_UNDERLINE + "f",
                   "al",
                   "se",
                   ".",
               ]     ,  )







    @slow
    def     _lowercase     (    self     )     ->      Tuple:
           lowerCamelCase   :      List[Any]			 =					XLNetTokenizer.from_pretrained("xlnet-base-cased"     )

           lowerCamelCase   :      List[str]			 =					tokenizer.encode("sequence builders"     ,  add_special_tokens=UpperCamelCase__     )
           lowerCamelCase   :      Dict			 =					tokenizer.encode("multi-sequence build"     ,  add_special_tokens=UpperCamelCase__     )

           lowerCamelCase   :      str			 =					tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     )
           lowerCamelCase   :      List[str]			 =					tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     ,  UpperCamelCase__     )

           assert encoded_sentence == text + [4, 3]
           assert encoded_pair == text + [4] + text_a + [4, 3]







    @slow
    def     _lowercase     (    self     )     ->      Union[str, Any]:
           # fmt: off
           lowerCamelCase   :      str			 =					{"input_ids": [[17, 2_1442, 270, 17, 10, 1_4645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 2_2018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 1_4431, 13, 5500, 11, 1176, 580, 13, 1_6819, 4797, 23, 17, 10, 1_7135, 658, 19, 457, 7932, 13, 184, 19, 3154, 1_7135, 6468, 19, 1404, 1_2269, 19, 4229, 5356, 1_6264, 46, 19, 17, 2_0545, 1_0395, 9, 9, 9, 11, 28, 6421, 9531, 2_0729, 17, 10, 353, 1_7022, 11, 21, 6421, 9531, 1_6949, 17, 10, 1_1509, 753, 11, 33, 95, 2421, 7385, 956, 1_4431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 2_4738, 19, 1_3203, 658, 218, 787, 21, 430, 1_8482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 2_2178, 27, 1064, 22, 956, 13, 1_1101, 1429, 5854, 2_4313, 1_8953, 40, 422, 2_4366, 68, 1758, 37, 1_0483, 1_4257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 1_3894, 3380, 23, 95, 18, 1_7634, 2288, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCamelCase__     ,  model_name="xlnet-base-cased"     ,  revision="c841166438c31ec7ca9a106dee7bb312b73ae511"     ,  )











"""simple docstring"""







from ..utils import DummyObject, requires_backends
class        __A       (	metaclass=A_    ):







		'''simple docstring'''





		lowerCAmelCase			:					List[str]   	=				["torch", "torchsde"]







		def __init__(	self						:  Tuple			,*_snake_case						:  Union[str, Any]			,**_snake_case						:  Any						)					->		Union[str, Any]:






				"""simple docstring"""



				requires_backends(self			,['''torch''', '''torchsde''']						)







		@classmethod
		def 					UpperCAmelCase    (	cls						:  List[str]			,*_snake_case						:  int			,**_snake_case						:  Union[str, Any]						)					->		str:






				"""simple docstring"""



				requires_backends(cls			,['''torch''', '''torchsde''']						)







		@classmethod
		def 					UpperCAmelCase    (	cls						:  List[Any]			,*_snake_case						:  List[Any]			,**_snake_case						:  List[str]						)					->		List[Any]:






				"""simple docstring"""



				requires_backends(cls			,['''torch''', '''torchsde''']						)
