




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_ : int    		=       get_tests_dir("fixtures/test_sentencepiece.model")







@require_sentencepiece
@require_tokenizers
class __snake_case			(			a ,  unittest.TestCase		):
   UpperCAmelCase__  :    str          =  XLNetTokenizer
   UpperCAmelCase__  :    Optional[Any]          =  XLNetTokenizerFast
   UpperCAmelCase__  :    str          =  True
   UpperCAmelCase__  :    Optional[int]          =  True







   def lowerCamelCase						(      self		:						str):

        """simple docstring"""


        super().setUp()

        # We have a SentencePiece fixture for testing
        UpperCAmelCase_          =							XLNetTokenizer(_snake_case					,						keep_accents=_snake_case)
        tokenizer.sanitize_special_tokens()
        tokenizer.save_pretrained(self.tmpdirname)







   def lowerCamelCase						(      self		:						Union[str, Any]):

        """simple docstring"""


        UpperCAmelCase_          =							'''<s>'''
        UpperCAmelCase_          =							1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(_snake_case)					,						_snake_case)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(_snake_case)					,						_snake_case)







   def lowerCamelCase						(      self		:						Any):

        """simple docstring"""


        UpperCAmelCase_          =							list(self.get_tokenizer().get_vocab().keys())

        self.assertEqual(vocab_keys[0]					,						'''<unk>''')
        self.assertEqual(vocab_keys[1]					,						'''<s>''')
        self.assertEqual(vocab_keys[-1]					,						'''<eod>''')
        self.assertEqual(len(_snake_case)					,						1006)







   def lowerCamelCase						(      self		:						int):

        """simple docstring"""


        self.assertEqual(self.get_tokenizer().vocab_size					,						1000)







   def lowerCamelCase						(      self		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							XLNetTokenizer(_snake_case					,						keep_accents=_snake_case)

        UpperCAmelCase_          =							tokenizer.tokenize('''This is a test''')
        self.assertListEqual(_snake_case					,						['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

        self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case)					,						[285, 46, 10, 170, 382])

        UpperCAmelCase_          =							tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
        self.assertListEqual(
            _snake_case					,						[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''é''',
                '''.''',
            ]					,						)
        UpperCAmelCase_          =							tokenizer.convert_tokens_to_ids(_snake_case)
        self.assertListEqual(_snake_case					,						[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4])

        UpperCAmelCase_          =							tokenizer.convert_ids_to_tokens(_snake_case)
        self.assertListEqual(
            _snake_case					,						[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''<unk>''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''<unk>''',
                '''.''',
            ]					,						)







   def lowerCamelCase						(      self		:						Optional[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							XLNetTokenizer(_snake_case					,						do_lower_case=_snake_case)
        UpperCAmelCase_          =							tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
        self.assertListEqual(
            _snake_case					,						[
                SPIECE_UNDERLINE + '''''',
                '''i''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''se''',
                '''.''',
            ]					,						)
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')					,						['''▁he''', '''ll''', '''o'''])







   def lowerCamelCase						(      self		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							XLNetTokenizer(_snake_case					,						do_lower_case=_snake_case)
        UpperCAmelCase_          =							tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
        self.assertListEqual(
            _snake_case					,						[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''se''',
                '''.''',
            ]					,						)







   @slow
   def lowerCamelCase						(      self		:						List[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							XLNetTokenizer.from_pretrained('''xlnet-base-cased''')

        UpperCAmelCase_          =							tokenizer.encode('''sequence builders'''					,						add_special_tokens=_snake_case)
        UpperCAmelCase_          =							tokenizer.encode('''multi-sequence build'''					,						add_special_tokens=_snake_case)

        UpperCAmelCase_          =							tokenizer.build_inputs_with_special_tokens(_snake_case)
        UpperCAmelCase_          =							tokenizer.build_inputs_with_special_tokens(_snake_case					,						_snake_case)

        assert encoded_sentence == text + [4, 3]
        assert encoded_pair == text + [4] + text_a + [4, 3]







   @slow
   def lowerCamelCase						(      self		:						List[str]):

        """simple docstring"""


        UpperCAmelCase_          =							{'''input_ids''': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=_snake_case					,						model_name='''xlnet-base-cased'''					,						revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''					,						)












import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin







class __snake_case			(			a ,  unittest.TestCase		):
   UpperCAmelCase__  :    Any          =  PhobertTokenizer
   UpperCAmelCase__  :    List[str]          =  False







   def lowerCamelCase						(      self		:						str):

        """simple docstring"""


        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        UpperCAmelCase_          =							['''T@@''', '''i''', '''I''', '''R@@''', '''r''', '''e@@''']
        UpperCAmelCase_          =							dict(zip(_snake_case					,						range(len(_snake_case))))
        UpperCAmelCase_          =							['''#version: 0.2''', '''l à</w>''']
        UpperCAmelCase_          =							{'''unk_token''': '''<unk>'''}

        UpperCAmelCase_          =							os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES['''vocab_file'''])
        UpperCAmelCase_          =							os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES['''merges_file'''])

        with open(self.vocab_file					,						'''w'''					,						encoding='''utf-8''') as fp:
             for token in vocab_tokens:
                  fp.write(F"""{token} {vocab_tokens[token]}\n""")
        with open(self.merges_file					,						'''w'''					,						encoding='''utf-8''') as fp:
             fp.write('''\n'''.join(_snake_case))







   def lowerCamelCase						(      self		:						int					,						**_snake_case		:						Any):

        """simple docstring"""


        kwargs.update(self.special_tokens_map)
        return PhobertTokenizer.from_pretrained(self.tmpdirname					,						**_snake_case)







   def lowerCamelCase						(      self		:						Optional[Any]					,						_snake_case		:						Union[str, Any]):

        """simple docstring"""


        UpperCAmelCase_          =							'''Tôi là VinAI Research'''
        UpperCAmelCase_          =							'''T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'''
        return input_text, output_text







   def lowerCamelCase						(      self		:						List[str]):

        """simple docstring"""


        UpperCAmelCase_          =							PhobertTokenizer(self.vocab_file					,						self.merges_file					,						**self.special_tokens_map)
        UpperCAmelCase_          =							'''Tôi là VinAI Research'''
        UpperCAmelCase_          =							'''T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'''.split()
        UpperCAmelCase_          =							tokenizer.tokenize(_snake_case)
        print(_snake_case)
        self.assertListEqual(_snake_case					,						_snake_case)

        UpperCAmelCase_          =							tokens + [tokenizer.unk_token]

        UpperCAmelCase_          =							[4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case)					,						_snake_case)






