

'''simple docstring'''
import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
     lowerCamelCase :      Tuple					 =     "pt"
elif is_tf_available():
     lowerCamelCase :      Optional[int]					 =     "tf"
else:
     lowerCamelCase :      Tuple					 =     "jax"







class 							A__    (       A__     ,   unittest.TestCase  ):
  A__		 =				ByTaTokenizer
  A__		 =				False





  def 						A  (		self      :	str )	->    Any:





       '''simple docstring'''






       super().setUp()
       _SCREAMING_SNAKE_CASE							    =ByTaTokenizer()
       tokenizer.save_pretrained(self.tmpdirname )





  @cached_property
  def 						A  (		self      :	List[Any] )	->    Union[str, Any]:





       '''simple docstring'''






       return ByTaTokenizer.from_pretrained('google/byt5-small' )





  def 						A  (		self      :	Dict							,					**_a      :	Union[str, Any] )	->    ByTaTokenizer:





       '''simple docstring'''






       return self.tokenizer_class.from_pretrained(self.tmpdirname							,					**_a )





  def 						A  (		self      :	List[str]							,					_a      :	Dict							,					_a      :	str=False							,					_a      :	Optional[int]=20							,					_a      :	int=5 )	->    Tuple[str, list]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =[]
       for i in range(len(_a ) ):
            try:
                 _SCREAMING_SNAKE_CASE							    =tokenizer.decode([i]							,					clean_up_tokenization_spaces=_a )
            except UnicodeDecodeError:
                 pass
            toks.append((i, tok) )

       _SCREAMING_SNAKE_CASE							    =list(filter(lambda _a : re.match(r'^[ a-zA-Z]+$'							,					t[1] )							,					_a ) )
       _SCREAMING_SNAKE_CASE							    =list(filter(lambda _a : [t[0]] == tokenizer.encode(t[1]							,					add_special_tokens=_a )							,					_a ) )
       if max_length is not None and len(_a ) > max_length:
            _SCREAMING_SNAKE_CASE							    =toks[:max_length]
       if min_length is not None and len(_a ) < min_length and len(_a ) > 0:
            while len(_a ) < min_length:
                 _SCREAMING_SNAKE_CASE							    =toks + toks
        # toks_str = [t[1] for t in toks]
       _SCREAMING_SNAKE_CASE							    =[t[0] for t in toks]

       # Ensure consistency
       _SCREAMING_SNAKE_CASE							    =tokenizer.decode(_a							,					clean_up_tokenization_spaces=_a )
       if " " not in output_txt and len(_a ) > 1:
            _SCREAMING_SNAKE_CASE							    =(
                tokenizer.decode([toks_ids[0]]							,					clean_up_tokenization_spaces=_a )
                + ' '
                + tokenizer.decode(toks_ids[1:]							,					clean_up_tokenization_spaces=_a )
            )
       if with_prefix_space:
            _SCREAMING_SNAKE_CASE							    =' ' + output_txt
       _SCREAMING_SNAKE_CASE							    =tokenizer.encode(_a							,					add_special_tokens=_a )
       return output_txt, output_ids





  def 						A  (		self      :	Any )	->    List[str]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.ta_base_tokenizer
       _SCREAMING_SNAKE_CASE							    =tokenizer(['hi</s>', 'I went to the gym</s>', '</s>'] )
       _SCREAMING_SNAKE_CASE							    =tokenizer(['hi', 'I went to the gym', ''] )
       self.assertListEqual(batch_with_eos_added['input_ids']							,					batch_without_eos_added['input_ids'] )





  def 						A  (		self      :	Optional[int] )	->    str:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.ta_base_tokenizer
       _SCREAMING_SNAKE_CASE							    ='Unicode €.'
       _SCREAMING_SNAKE_CASE							    =tokenizer(_a )
       _SCREAMING_SNAKE_CASE							    =[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
       self.assertEqual(encoded['input_ids']							,					_a )

       # decoding
       _SCREAMING_SNAKE_CASE							    =tokenizer.decode(_a )
       self.assertEqual(_a							,					'Unicode €.</s>' )

       _SCREAMING_SNAKE_CASE							    =tokenizer('e è é ê ë' )
       _SCREAMING_SNAKE_CASE							    =[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
       self.assertEqual(encoded['input_ids']							,					_a )
       # decoding
       _SCREAMING_SNAKE_CASE							    =tokenizer.decode(_a )
       self.assertEqual(_a							,					'e è é ê ë</s>' )

       # encode/decode, but with `encode` instead of `__call__`
       self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë' ) )							,					'e è é ê ë</s>' )





  def 						A  (		self      :	Union[str, Any] )	->    List[str]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.ta_base_tokenizer
       _SCREAMING_SNAKE_CASE							    =['A long paragraph for summarization.', 'Another paragraph for summarization.']
       # fmt: off
       _SCREAMING_SNAKE_CASE							    =[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
       # fmt: on
       _SCREAMING_SNAKE_CASE							    =tokenizer(_a							,					padding=_a							,					return_tensors=_a )
       self.assertIsInstance(_a							,					_a )

       if FRAMEWORK != "jax":
            _SCREAMING_SNAKE_CASE							    =list(batch.input_ids.numpy()[0] )
       else:
            _SCREAMING_SNAKE_CASE							    =list(batch.input_ids.tolist()[0] )

       self.assertListEqual(_a							,					_a )

       self.assertEqual((2, 37)							,					batch.input_ids.shape )
       self.assertEqual((2, 37)							,					batch.attention_mask.shape )





  def 						A  (		self      :	Dict )	->    Union[str, Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.ta_base_tokenizer
       _SCREAMING_SNAKE_CASE							    =['A long paragraph for summarization.', 'Another paragraph for summarization.']
       _SCREAMING_SNAKE_CASE							    =tokenizer(_a							,					padding=_a							,					return_tensors=_a )
       # check if input_ids are returned and no decoder_input_ids
       self.assertIn('input_ids'							,					_a )
       self.assertIn('attention_mask'							,					_a )
       self.assertNotIn('decoder_input_ids'							,					_a )
       self.assertNotIn('decoder_attention_mask'							,					_a )





  def 						A  (		self      :	List[str] )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.ta_base_tokenizer
       _SCREAMING_SNAKE_CASE							    =[
           'Summary of the text.',
           'Another summary.',
       ]
       _SCREAMING_SNAKE_CASE							    =tokenizer(
           text_target=_a							,					max_length=32							,					padding='max_length'							,					truncation=_a							,					return_tensors=_a )
       self.assertEqual(32							,					targets['input_ids'].shape[1] )





  def 						A  (		self      :	Union[str, Any] )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.ta_base_tokenizer
       _SCREAMING_SNAKE_CASE							    =['A long paragraph for summarization. </s>']
       _SCREAMING_SNAKE_CASE							    =['Summary of the text. </s>']
       # fmt: off
       _SCREAMING_SNAKE_CASE							    =[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
       _SCREAMING_SNAKE_CASE							    =[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
       # fmt: on

       _SCREAMING_SNAKE_CASE							    =tokenizer(_a							,					text_target=_a )

       self.assertEqual(_a							,					batch['input_ids'][0] )
       self.assertEqual(_a							,					batch['labels'][0] )





  def 						A  (		self      :	Tuple )	->    List[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.get_tokenizers()
       for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}" ):
                 self.assertNotEqual(tokenizer.model_max_length							,					42 )

        # Now let's start the test
       _SCREAMING_SNAKE_CASE							    =self.get_tokenizers()
       for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}" ):
                 # Isolate this from the other tests because we save additional tokens/etc
                 _SCREAMING_SNAKE_CASE							    =tempfile.mkdtemp()

                 _SCREAMING_SNAKE_CASE							    =' He is very happy, UNwant\u00E9d,running'
                 _SCREAMING_SNAKE_CASE							    =tokenizer.encode(_a							,					add_special_tokens=_a )
                 tokenizer.save_pretrained(_a )

                 _SCREAMING_SNAKE_CASE							    =tokenizer.__class__.from_pretrained(_a )
                 _SCREAMING_SNAKE_CASE							    =after_tokenizer.encode(_a							,					add_special_tokens=_a )
                 self.assertListEqual(_a							,					_a )

                 shutil.rmtree(_a )

       _SCREAMING_SNAKE_CASE							    =self.get_tokenizers(model_max_length=42 )
       for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}" ):
                 # Isolate this from the other tests because we save additional tokens/etc
                 _SCREAMING_SNAKE_CASE							    =tempfile.mkdtemp()

                 _SCREAMING_SNAKE_CASE							    =' He is very happy, UNwant\u00E9d,running'
                 tokenizer.add_tokens(['bim', 'bambam'] )
                 _SCREAMING_SNAKE_CASE							    =tokenizer.additional_special_tokens
                 additional_special_tokens.append('new_additional_special_token' )
                 tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens} )
                 _SCREAMING_SNAKE_CASE							    =tokenizer.encode(_a							,					add_special_tokens=_a )
                 tokenizer.save_pretrained(_a )

                 _SCREAMING_SNAKE_CASE							    =tokenizer.__class__.from_pretrained(_a )
                 _SCREAMING_SNAKE_CASE							    =after_tokenizer.encode(_a							,					add_special_tokens=_a )
                 self.assertListEqual(_a							,					_a )
                 self.assertIn('new_additional_special_token'							,					after_tokenizer.additional_special_tokens )
                 self.assertEqual(after_tokenizer.model_max_length							,					42 )

                 _SCREAMING_SNAKE_CASE							    =tokenizer.__class__.from_pretrained(_a							,					model_max_length=43 )
                 self.assertEqual(tokenizer.model_max_length							,					43 )

                 shutil.rmtree(_a )





  def 						A  (		self      :	int )	->    List[str]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =[]
       if self.test_slow_tokenizer:
            tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()) )

       if self.test_rust_tokenizer:
            tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()) )

       for tokenizer_class, tokenizer_utils in tokenizer_list:
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer_utils.save_pretrained(_a )

                 with open(os.path.join(_a							,					'special_tokens_map.json' )							,					encoding='utf-8' ) as json_file:
                      _SCREAMING_SNAKE_CASE							    =json.load(_a )

                 with open(os.path.join(_a							,					'tokenizer_config.json' )							,					encoding='utf-8' ) as json_file:
                      _SCREAMING_SNAKE_CASE							    =json.load(_a )

                 _SCREAMING_SNAKE_CASE							    =[f"<extra_id_{i}>" for i in range(125 )]

                 _SCREAMING_SNAKE_CASE							    =added_tokens_extra_ids + [
                     'an_additional_special_token'
                 ]
                 _SCREAMING_SNAKE_CASE							    =added_tokens_extra_ids + [
                     'an_additional_special_token'
                 ]

                 with open(os.path.join(_a							,					'special_tokens_map.json' )							,					'w'							,					encoding='utf-8' ) as outfile:
                      json.dump(_a							,					_a )
                 with open(os.path.join(_a							,					'tokenizer_config.json' )							,					'w'							,					encoding='utf-8' ) as outfile:
                      json.dump(_a							,					_a )

                 # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                 # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                 # "special_tokens_map.json" files
                 _SCREAMING_SNAKE_CASE							    =tokenizer_class.from_pretrained(
                     _a							,					)
                 self.assertIn(
                     'an_additional_special_token'							,					tokenizer_without_change_in_init.additional_special_tokens )
                 # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                 self.assertEqual(
                     ['an_additional_special_token']							,					tokenizer_without_change_in_init.convert_ids_to_tokens(
                         tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token'] ) )							,					)

                 # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                 _SCREAMING_SNAKE_CASE							    =added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'							,					lstrip=_a )]
                 _SCREAMING_SNAKE_CASE							    =tokenizer_class.from_pretrained(
                     _a							,					additional_special_tokens=_a							,					)

                 self.assertIn('a_new_additional_special_token'							,					tokenizer.additional_special_tokens )
                 self.assertEqual(
                     ['a_new_additional_special_token']							,					tokenizer.convert_ids_to_tokens(
                         tokenizer.convert_tokens_to_ids(['a_new_additional_special_token'] ) )							,					)





  def 						A  (		self      :	Union[str, Any] )	->    int:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =[]
       if self.test_slow_tokenizer:
            tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()) )

       if self.test_rust_tokenizer:
            tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()) )

       for tokenizer_class, tokenizer_utils in tokenizer_list:
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer_utils.save_pretrained(_a )

                 _SCREAMING_SNAKE_CASE							    =tokenizer_class.from_pretrained(_a )

                 self.assertTrue(tokenizer.decode([255] ) == '' )





  def 						A  (		self      :	int )	->    List[Any]:





       '''simple docstring'''






       pass





  def 						A  (		self      :	Optional[int] )	->    Optional[Any]:





       '''simple docstring'''






       pass





  def 						A  (		self      :	List[Any] )	->    Tuple:





       '''simple docstring'''






       pass





  def 						A  (		self      :	Tuple )	->    Optional[Any]:





       '''simple docstring'''






       pass





  def 						A  (		self      :	Optional[int] )	->    List[str]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.get_tokenizers(fast=_a							,					do_lower_case=_a )
       for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}" ):
                 _SCREAMING_SNAKE_CASE							    =['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
                 _SCREAMING_SNAKE_CASE							    =tokenizer.convert_tokens_to_string(_a )

                 self.assertIsInstance(_a							,					_a )





  def 						A  (		self      :	List[str] )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.get_tokenizers()
       for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}" ):
                 _SCREAMING_SNAKE_CASE							    =[
                     'bos_token',
                     'eos_token',
                     'unk_token',
                     'sep_token',
                     'pad_token',
                     'cls_token',
                     'mask_token',
                 ]

                 _SCREAMING_SNAKE_CASE							    =0
                 _SCREAMING_SNAKE_CASE							    =tokenizer.convert_ids_to_tokens(
                     _a							,					skip_special_tokens=_a )

                 for attr in attributes_list:
                      setattr(_a							,					attr + '_id'							,					_a )
                      self.assertEqual(getattr(_a							,					_a )							,					_a )
                      self.assertEqual(getattr(_a							,					attr + '_id' )							,					_a )

                      setattr(_a							,					attr + '_id'							,					_a )
                      self.assertEqual(getattr(_a							,					_a )							,					_a )
                      self.assertEqual(getattr(_a							,					attr + '_id' )							,					_a )

                 setattr(_a							,					'additional_special_tokens_ids'							,					[] )
                 self.assertListEqual(getattr(_a							,					'additional_special_tokens' )							,					[] )
                 self.assertListEqual(getattr(_a							,					'additional_special_tokens_ids' )							,					[] )

                 setattr(_a							,					'additional_special_tokens_ids'							,					[token_id_to_test_setters] )
                 self.assertListEqual(getattr(_a							,					'additional_special_tokens' )							,					[token_to_test_setters] )
                 self.assertListEqual(getattr(_a							,					'additional_special_tokens_ids' )							,					[token_id_to_test_setters] )




import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class        _lowercase      (			A__						,			unittest.TestCase       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			Optional[int]						=		TransfoXLTokenizer
				SCREAMING_SNAKE_CASE__				:			int						=		False
				SCREAMING_SNAKE_CASE__				:			Optional[Any]						=		False


				def 		__magic_name__(       self			:str	)		->			Dict:
										super().setUp()

										__SCREAMING_SNAKE_CASE						:     List[str]					=		[
										    '''<unk>''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''want''',
										    '''unwanted''',
										    '''wa''',
										    '''un''',
										    '''running''',
										    ''',''',
										    '''low''',
										    '''l''',
										]
										__SCREAMING_SNAKE_CASE						:     Any					=		os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES['''vocab_file''']	)
										with open(self.vocab_file			,			'''w'''			,			encoding='''utf-8'''	) as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)


				def 		__magic_name__(       self			:Any			,			**lowerCAmelCase__			:int	)		->			str:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		True
										return TransfoXLTokenizer.from_pretrained(self.tmpdirname			,			**lowerCAmelCase__	)


				def 		__magic_name__(       self			:Union[str, Any]			,			lowerCAmelCase__			:List[Any]	)		->			Tuple:
										__SCREAMING_SNAKE_CASE						:     Dict					=		'''<unk> UNwanted , running'''
										__SCREAMING_SNAKE_CASE						:     List[str]					=		'''<unk> unwanted, running'''
										return input_text, output_text


				def 		__magic_name__(       self			:Any	)		->			Union[str, Any]:
										__SCREAMING_SNAKE_CASE						:     int					=		TransfoXLTokenizer(vocab_file=self.vocab_file			,			lower_case=lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     int					=		tokenizer.tokenize('''<unk> UNwanted , running'''	)
										self.assertListEqual(lowerCAmelCase__			,			['''<unk>''', '''unwanted''', ''',''', '''running''']	)

										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__	)			,			[0, 4, 8, 7]	)


				def 		__magic_name__(       self			:Tuple	)		->			Union[str, Any]:
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		TransfoXLTokenizer(lower_case=lowerCAmelCase__	)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''	)			,			['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)


				def 		__magic_name__(       self			:Tuple	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		TransfoXLTokenizer(lower_case=lowerCAmelCase__	)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''	)			,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)


				def 		__magic_name__(       self			:Dict	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     List[str]					=		TransfoXLTokenizer(lower_case=lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		'''Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'''
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		[
										    '''Hello''',
										    '''(''',
										    '''bracket''',
										    ''')''',
										    '''and''',
										    '''side''',
										    '''@-@''',
										    '''scrolled''',
										    '''[''',
										    '''and''',
										    ''']''',
										    '''Henry''',
										    '''\'s''',
										    '''$''',
										    '''5''',
										    '''@,@''',
										    '''000''',
										    '''with''',
										    '''3''',
										    '''@.@''',
										    '''34''',
										    '''m''',
										    '''.''',
										    '''What''',
										    '''\'s''',
										    '''up''',
										    '''!''',
										    '''?''',
										]

										self.assertListEqual(tokenizer.tokenize(lowerCAmelCase__	)			,			lowerCAmelCase__	)

										self.assertEqual(tokenizer.convert_tokens_to_string(lowerCAmelCase__	)			,			lowerCAmelCase__	)



				def 		__magic_name__(       self			:str	)		->			int:
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		self.get_tokenizer()
										__SCREAMING_SNAKE_CASE						:     Any					=		len(lowerCAmelCase__	)

										tokenizer.add_tokens(['''new1''', '''new2''']	)
										tokenizer.move_added_token('''new1'''			,			1	)

										# Check that moved token is not copied (duplicate)
										self.assertEqual(len(lowerCAmelCase__	)			,			original_len + 2	)
										# Check that token is moved to specified id
										self.assertEqual(tokenizer.encode('''new1'''	)			,			[1]	)
										self.assertEqual(tokenizer.decode([1]	)			,			'''new1'''	)
