





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__A									=				get_tests_dir("fixtures/test_sentencepiece.model")



@require_sentencepiece
@require_tokenizers
class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ,							unittest.TestCase    ):







 '''simple docstring'''
 lowercase_		=			XLNetTokenizer
 lowercase_		=			XLNetTokenizerFast
 lowercase_		=			True
 lowercase_		=			True

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[Any])       ->Optional[Any]:



   '''simple docstring'''



   super().setUp()

   # We have a SentencePiece fixture for testing
   lowerCamelCase__:      Optional[Any]			  =XLNetTokenizer(UpperCAmelCase_       , keep_accents=UpperCAmelCase_)
   tokenizer.sanitize_special_tokens()
   tokenizer.save_pretrained(self.tmpdirname)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Union[str, Any])       ->int:



   '''simple docstring'''



   lowerCamelCase__:      Any			  ="<s>"
   lowerCamelCase__:      Optional[Any]			  =1

   self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase_)       , UpperCAmelCase_)
   self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase_)       , UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[Any])       ->int:



   '''simple docstring'''



   lowerCamelCase__:      List[Any]			  =list(self.get_tokenizer().get_vocab().keys())

   self.assertEqual(vocab_keys[0]       , "<unk>")
   self.assertEqual(vocab_keys[1]       , "<s>")
   self.assertEqual(vocab_keys[-1]       , "<eod>")
   self.assertEqual(len(UpperCAmelCase_)       , 1_006)

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->List[Any]:



   '''simple docstring'''



   self.assertEqual(self.get_tokenizer().vocab_size       , 1_000)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Any)       ->str:



   '''simple docstring'''



   lowerCamelCase__:      Dict			  =XLNetTokenizer(UpperCAmelCase_       , keep_accents=UpperCAmelCase_)

   lowerCamelCase__:      List[str]			  =tokenizer.tokenize("This is a test")
   self.assertListEqual(UpperCAmelCase_       , ["▁This", "▁is", "▁a", "▁t", "est"])

   self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_)       , [285, 46, 10, 170, 382])

   lowerCamelCase__:      List[Any]			  =tokenizer.tokenize("I was born in 92000, and this is falsé.")
   self.assertListEqual(
       UpperCAmelCase_       , [
           SPIECE_UNDERLINE + "I",
           SPIECE_UNDERLINE + "was",
           SPIECE_UNDERLINE + "b",
           "or",
           "n",
           SPIECE_UNDERLINE + "in",
           SPIECE_UNDERLINE + "",
           "9",
           "2",
           "0",
           "0",
           "0",
           ",",
           SPIECE_UNDERLINE + "and",
           SPIECE_UNDERLINE + "this",
           SPIECE_UNDERLINE + "is",
           SPIECE_UNDERLINE + "f",
           "al",
           "s",
           "é",
           ".",
       ]       , )
   lowerCamelCase__:      Dict			  =tokenizer.convert_tokens_to_ids(UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4])

   lowerCamelCase__:      List[Any]			  =tokenizer.convert_ids_to_tokens(UpperCAmelCase_)
   self.assertListEqual(
       UpperCAmelCase_       , [
           SPIECE_UNDERLINE + "I",
           SPIECE_UNDERLINE + "was",
           SPIECE_UNDERLINE + "b",
           "or",
           "n",
           SPIECE_UNDERLINE + "in",
           SPIECE_UNDERLINE + "",
           "<unk>",
           "2",
           "0",
           "0",
           "0",
           ",",
           SPIECE_UNDERLINE + "and",
           SPIECE_UNDERLINE + "this",
           SPIECE_UNDERLINE + "is",
           SPIECE_UNDERLINE + "f",
           "al",
           "s",
           "<unk>",
           ".",
       ]       , )

 def 							SCREAMING_SNAKE_CASE_					(self				:   str)       ->List[str]:



   '''simple docstring'''



   lowerCamelCase__:      int			  =XLNetTokenizer(UpperCAmelCase_       , do_lower_case=UpperCAmelCase_)
   lowerCamelCase__:      List[Any]			  =tokenizer.tokenize("I was born in 92000, and this is falsé.")
   self.assertListEqual(
       UpperCAmelCase_       , [
           SPIECE_UNDERLINE + "",
           "i",
           SPIECE_UNDERLINE + "was",
           SPIECE_UNDERLINE + "b",
           "or",
           "n",
           SPIECE_UNDERLINE + "in",
           SPIECE_UNDERLINE + "",
           "9",
           "2",
           "0",
           "0",
           "0",
           ",",
           SPIECE_UNDERLINE + "and",
           SPIECE_UNDERLINE + "this",
           SPIECE_UNDERLINE + "is",
           SPIECE_UNDERLINE + "f",
           "al",
           "se",
           ".",
       ]       , )
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo")       , ["▁he", "ll", "o"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Any:



   '''simple docstring'''



   lowerCamelCase__:      List[Any]			  =XLNetTokenizer(UpperCAmelCase_       , do_lower_case=UpperCAmelCase_)
   lowerCamelCase__:      Any			  =tokenizer.tokenize("I was born in 92000, and this is falsé.")
   self.assertListEqual(
       UpperCAmelCase_       , [
           SPIECE_UNDERLINE + "I",
           SPIECE_UNDERLINE + "was",
           SPIECE_UNDERLINE + "b",
           "or",
           "n",
           SPIECE_UNDERLINE + "in",
           SPIECE_UNDERLINE + "",
           "9",
           "2",
           "0",
           "0",
           "0",
           ",",
           SPIECE_UNDERLINE + "and",
           SPIECE_UNDERLINE + "this",
           SPIECE_UNDERLINE + "is",
           SPIECE_UNDERLINE + "f",
           "al",
           "se",
           ".",
       ]       , )

 @slow
 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[int])       ->str:



   '''simple docstring'''



   lowerCamelCase__:      Dict			  =XLNetTokenizer.from_pretrained("xlnet-base-cased")

   lowerCamelCase__:      Tuple			  =tokenizer.encode("sequence builders"       , add_special_tokens=UpperCAmelCase_)
   lowerCamelCase__:      Optional[Any]			  =tokenizer.encode("multi-sequence build"       , add_special_tokens=UpperCAmelCase_)

   lowerCamelCase__:      Optional[Any]			  =tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_)
   lowerCamelCase__:      Optional[Any]			  =tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_       , UpperCAmelCase_)

   assert encoded_sentence == text + [4, 3]
   assert encoded_pair == text + [4] + text_a + [4, 3]







 @slow
 def 							SCREAMING_SNAKE_CASE_					(self				:   Any)       ->Optional[int]:



   '''simple docstring'''



   lowerCamelCase__:      Optional[Any]			  ={"input_ids": [[17, 21_442, 270, 17, 10, 14_645, 318, 34, 17, 4_546, 3_145, 787, 13, 7_752, 22_018, 23, 21, 17, 4_546, 3_145, 787, 13, 3_352, 14_431, 13, 5_500, 11, 1_176, 580, 13, 16_819, 4_797, 23, 17, 10, 17_135, 658, 19, 457, 7_932, 13, 184, 19, 3_154, 17_135, 6_468, 19, 1_404, 12_269, 19, 4_229, 5_356, 16_264, 46, 19, 17, 20_545, 10_395, 9, 9, 9, 11, 28, 6_421, 9_531, 20_729, 17, 10, 353, 17_022, 11, 21, 6_421, 9_531, 16_949, 17, 10, 11_509, 753, 11, 33, 95, 2_421, 7_385, 956, 14_431, 2_626, 25, 842, 7_385, 4_836, 21, 1_429, 2_272, 9_855, 3_120, 161, 24_738, 19, 13_203, 658, 218, 787, 21, 430, 18_482, 847, 2_637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22_178, 27, 1_064, 22, 956, 13, 11_101, 1_429, 5_854, 24_313, 18_953, 40, 422, 24_366, 68, 1_758, 37, 10_483, 14_257, 31, 207, 263, 21, 203, 3_773, 25, 71, 9_735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2_049, 3_442, 17, 13_894, 3_380, 23, 95, 18, 17_634, 2_288, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
   # fmt: on

   self.tokenizer_integration_test_util(
       expected_encoding=UpperCAmelCase_       , model_name="xlnet-base-cased"       , revision="c841166438c31ec7ca9a106dee7bb312b73ae511"       , )








import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


__A									=				get_tests_dir("fixtures/test_sentencepiece_bpe.model")



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ,							unittest.TestCase    ):







 '''simple docstring'''
 lowercase_		=			BartphoTokenizer
 lowercase_		=			False
 lowercase_		=			True

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Tuple:



   '''simple docstring'''



   super().setUp()

   lowerCamelCase__:      int			  =["▁This", "▁is", "▁a", "▁t", "est"]
   lowerCamelCase__:      Tuple			  =dict(zip(UpperCAmelCase_       , range(len(UpperCAmelCase_))))
   lowerCamelCase__:      List[Any]			  ={"unk_token": "<unk>"}

   lowerCamelCase__:      Dict			  =os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES["monolingual_vocab_file"])
   with open(self.monolingual_vocab_file       , "w"       , encoding="utf-8") as fp:
     for token in vocab_tokens:
       fp.write(F"""{token} {vocab_tokens[token]}\n""")

   lowerCamelCase__:      Dict			  =BartphoTokenizer(UpperCAmelCase_       , self.monolingual_vocab_file       , **self.special_tokens_map)
   tokenizer.save_pretrained(self.tmpdirname)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[int]       , **UpperCAmelCase_				:   Optional[Any])       ->str:



   '''simple docstring'''



   kwargs.update(self.special_tokens_map)
   return BartphoTokenizer.from_pretrained(self.tmpdirname       , **UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Union[str, Any]       , UpperCAmelCase_				:   Optional[Any])       ->List[Any]:



   '''simple docstring'''



   lowerCamelCase__:      Optional[int]			  ="This is a là test"
   lowerCamelCase__:      Optional[Any]			  ="This is a<unk><unk> test"
   return input_text, output_text







 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Optional[Any]:



   '''simple docstring'''



   lowerCamelCase__:      str			  =BartphoTokenizer(UpperCAmelCase_       , self.monolingual_vocab_file       , **self.special_tokens_map)
   lowerCamelCase__:      List[Any]			  ="This is a là test"
   lowerCamelCase__:      Optional[int]			  ="▁This ▁is ▁a ▁l à ▁t est".split()
   lowerCamelCase__:      Optional[int]			  =tokenizer.tokenize(UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

   lowerCamelCase__:      Tuple			  =tokens + [tokenizer.unk_token]
   lowerCamelCase__:      List[Any]			  =[4, 5, 6, 3, 3, 7, 8, 3]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_)       , UpperCAmelCase_)

