


'''simple docstring'''

import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class   __lowerCAmelCase						(lowercase_  ,     unittest.TestCase     ):





      '''simple docstring'''






      lowerCAmelCase__					:     List[Any]  					=							TransfoXLTokenizer
      lowerCAmelCase__					:     int  					=							False
      lowerCAmelCase__					:     List[str]  					=							False



      def       UpperCamelCase__		(self    :    Dict ):

             '''simple docstring'''







             super().setUp()

             lowercase__      			= [
                 '''<unk>''',
                 '''[CLS]''',
                 '''[SEP]''',
                 '''want''',
                 '''unwanted''',
                 '''wa''',
                 '''un''',
                 '''running''',
                 ''',''',
                 '''low''',
                 '''l''',
             ]
             lowercase__      			= os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['''vocab_file'''] )
             with open(self.vocab_file   ,						'''w'''   ,						encoding='''utf-8''' ) as vocab_writer:
                    vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )



      def       UpperCamelCase__		(self    :    List[str]   ,						**UpperCamelCase    :    int ):

             '''simple docstring'''







             lowercase__      			= True
             return TransfoXLTokenizer.from_pretrained(self.tmpdirname   ,						**UpperCamelCase )



      def       UpperCamelCase__		(self    :    int   ,						UpperCamelCase    :    int ):

             '''simple docstring'''







             lowercase__      			= '''<unk> UNwanted , running'''
             lowercase__      			= '''<unk> unwanted, running'''
             return input_text, output_text



      def       UpperCamelCase__		(self    :    Dict ):

             '''simple docstring'''







             lowercase__      			= TransfoXLTokenizer(vocab_file=self.vocab_file   ,						lower_case=UpperCamelCase )

             lowercase__      			= tokenizer.tokenize('''<unk> UNwanted , running''' )
             self.assertListEqual(UpperCamelCase   ,						['''<unk>''', '''unwanted''', ''',''', '''running'''] )

             self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase )   ,						[0, 4, 8, 7] )



      def       UpperCamelCase__		(self    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= TransfoXLTokenizer(lower_case=UpperCamelCase )

             self.assertListEqual(
                 tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  ''' )   ,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''] )



      def       UpperCamelCase__		(self    :    List[str] ):

             '''simple docstring'''







             lowercase__      			= TransfoXLTokenizer(lower_case=UpperCamelCase )

             self.assertListEqual(
                 tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  ''' )   ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )



      def       UpperCamelCase__		(self    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= TransfoXLTokenizer(lower_case=UpperCamelCase )
             lowercase__      			= '''Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'''
             lowercase__      			= [
                 '''Hello''',
                 '''(''',
                 '''bracket''',
                 ''')''',
                 '''and''',
                 '''side''',
                 '''@-@''',
                 '''scrolled''',
                 '''[''',
                 '''and''',
                 ''']''',
                 '''Henry''',
                 '''\'s''',
                 '''$''',
                 '''5''',
                 '''@,@''',
                 '''000''',
                 '''with''',
                 '''3''',
                 '''@.@''',
                 '''34''',
                 '''m''',
                 '''.''',
                 '''What''',
                 '''\'s''',
                 '''up''',
                 '''!''',
                 '''?''',
             ]

             self.assertListEqual(tokenizer.tokenize(UpperCamelCase )   ,						UpperCamelCase )

             self.assertEqual(tokenizer.convert_tokens_to_string(UpperCamelCase )   ,						UpperCamelCase )






      def       UpperCamelCase__		(self    :    Tuple ):

             '''simple docstring'''







             lowercase__      			= self.get_tokenizer()
             lowercase__      			= len(UpperCamelCase )

             tokenizer.add_tokens(['''new1''', '''new2'''] )
             tokenizer.move_added_token('''new1'''   ,						1 )

             # Check that moved token is not copied (duplicate)
             self.assertEqual(len(UpperCamelCase )   ,						original_len + 2 )
             # Check that token is moved to specified id
             self.assertEqual(tokenizer.encode('''new1''' )   ,						[1] )
             self.assertEqual(tokenizer.decode([1] )   ,						'''new1''' )








'''simple docstring'''

import unittest

from transformers import DonutProcessor


lowerCamelCase							:   Tuple  						=	'naver-clova-ix/donut-base'
class   __lowerCAmelCase						(unittest.TestCase     ):





      '''simple docstring'''






      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= DonutProcessor.from_pretrained(UpperCamelCase )






      def       UpperCamelCase__		(self    :    Tuple ):

             '''simple docstring'''







             lowercase__      			= {
                 '''name''': '''John Doe''',
                 '''age''': '''99''',
                 '''city''': '''Atlanta''',
                 '''state''': '''GA''',
                 '''zip''': '''30301''',
                 '''phone''': '''123-4567''',
                 '''nicknames''': [{'''nickname''': '''Johnny'''}, {'''nickname''': '''JD'''}],
             }

             lowercase__      			= (
                 '''<s_name>John Doe</s_name><s_age>99</s_age><s_city>Atlanta</s_city>'''
                 '''<s_state>GA</s_state><s_zip>30301</s_zip><s_phone>123-4567</s_phone>'''
                 '''<s_nicknames><s_nickname>Johnny</s_nickname>'''
                 '''<sep/><s_nickname>JD</s_nickname></s_nicknames>'''
             )
             lowercase__      			= self.processor.tokenajson(UpperCamelCase )

             self.assertDictEqual(UpperCamelCase   ,						UpperCamelCase )




