import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 							a_					(		lowercase__			,	unittest.TestCase		):






 '''simple docstring'''






 UpperCAmelCase_      					=   PreTrainedTokenizerFast
 UpperCAmelCase_      					=   False
 UpperCAmelCase_      					=   True
 UpperCAmelCase_      					=   'tokenizer_file'




 def 						__snake_case		(     self   :    int):







        '''simple docstring'''



        lowerCAmelCase__	      =   False  # because we don't have pretrained_vocab_files_map
        super().setUp()
        lowerCAmelCase__	      =   True

        lowerCAmelCase__	      =   ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
        lowerCAmelCase__	      =   'SaulLu/dummy-tokenizer-bytelevel-bpe'

        # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
        lowerCAmelCase__	      =   [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

        lowerCAmelCase__	      =   PreTrainedTokenizerFast.from_pretrained(model_paths[0])
        tokenizer.save_pretrained(self.tmpdirname)




 def 						__snake_case		(     self   :    int):







        '''simple docstring'''



        pass




 def 						__snake_case		(     self   :    Union[str, Any]):







        '''simple docstring'''



        pass




 def 						__snake_case		(     self   :    Optional[int]):







        '''simple docstring'''



        pass




 def 						__snake_case		(     self   :    List[str]):







        '''simple docstring'''



        pass




 def 						__snake_case		(     self   :    Any):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                      try:
                             lowerCAmelCase__	      =   tempfile.mkdtemp()
                             lowerCAmelCase__	      =   self.rust_tokenizer_class.from_pretrained(_lowercase     ,       **_lowercase)

                             tokenizer.save_pretrained(self.tmpdirname)
                             super().test_training_new_tokenizer()
                      finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname)
                             lowerCAmelCase__	      =   tmpdirname_orig




 def 						__snake_case		(     self   :    List[str]):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                      try:
                             lowerCAmelCase__	      =   tempfile.mkdtemp()
                             lowerCAmelCase__	      =   self.rust_tokenizer_class.from_pretrained(_lowercase     ,       **_lowercase)

                             tokenizer.save_pretrained(self.tmpdirname)
                             super().test_training_new_tokenizer_with_special_tokens_change()
                      finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname)
                             lowerCAmelCase__	      =   tmpdirname_orig




 def 						__snake_case		(     self   :    Tuple):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name)

        lowerCAmelCase__	      =   ('a' for _ in range(1_000))
        lowerCAmelCase__	      =   tokenizer.train_new_from_iterator(text_iterator=_lowercase     ,       length=1_000     ,       vocab_size=50)

        lowerCAmelCase__	      =   new_tokenizer.encode('a🤗')
        self.assertEqual(_lowercase     ,       [64, 172, 253, 97, 245])






@require_tokenizers
class 							a_					(		unittest.TestCase		):






 '''simple docstring'''






 def 						__snake_case		(     self   :    Dict):







        '''simple docstring'''



        lowerCAmelCase__	      =   AutoTokenizer.from_pretrained('bert-base-cased')
        lowerCAmelCase__	      =   json.loads(tokenizer._tokenizer.to_str())
        lowerCAmelCase__	      =   len(_lowercase)

        with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               lowerCAmelCase__	      =   ['tokenizer.4.0.0.json']
               tokenizer.save_pretrained(_lowercase)
               json.dump(_lowercase     ,       open(os.path.join(_lowercase     ,       'tokenizer.4.0.0.json')     ,       'w'))

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               lowerCAmelCase__	      =   AutoTokenizer.from_pretrained(_lowercase)
               self.assertEqual(len(_lowercase)     ,       len(_lowercase) + 1)
               lowerCAmelCase__	      =   json.loads(new_tokenizer._tokenizer.to_str())
               self.assertIn('huggingface'     ,       json_tokenizer['model']['vocab'])

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(_lowercase     ,       'tokenizer.4.0.0.json')     ,       os.path.join(_lowercase     ,       'tokenizer.42.0.0.json'))
               lowerCAmelCase__	      =   ['tokenizer.42.0.0.json']
               tokenizer.save_pretrained(_lowercase)
               lowerCAmelCase__	      =   AutoTokenizer.from_pretrained(_lowercase)
               self.assertEqual(len(_lowercase)     ,       len(_lowercase))
               lowerCAmelCase__	      =   json.loads(new_tokenizer._tokenizer.to_str())
               self.assertNotIn('huggingface'     ,       json_tokenizer['model']['vocab'])




 def 						__snake_case		(     self   :    Tuple):







        '''simple docstring'''



        lowerCAmelCase__	      =   'hf-internal-testing/test-two-tokenizers'

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        lowerCAmelCase__	      =   AutoTokenizer.from_pretrained(_lowercase)
        self.assertEqual(len(_lowercase)     ,       28_997)
        lowerCAmelCase__	      =   json.loads(tokenizer._tokenizer.to_str())
        self.assertIn('huggingface'     ,       json_tokenizer['model']['vocab'])

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        lowerCAmelCase__	      =   '3.0.0'
        lowerCAmelCase__	      =   old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowercase)
        self.assertEqual(len(_lowercase)     ,       28_996)
        lowerCAmelCase__	      =   json.loads(old_tokenizer._tokenizer.to_str())
        self.assertNotIn('huggingface'     ,       json_tokenizer['model']['vocab'])






@require_tokenizers
class 							a_					(		unittest.TestCase		):






 '''simple docstring'''






 def 						__snake_case		(     self   :    Optional[int]):







        '''simple docstring'''



        lowerCAmelCase__	      =   PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel')
        lowerCAmelCase__	      =   'The Matrix is a 1999 science fiction action film.'

        with concurrent.futures.ThreadPoolExecutor() as executor:
               lowerCAmelCase__	      =   [executor.submit(self.fetch     ,       _lowercase     ,       _lowercase) for i in range(10)]
               lowerCAmelCase__	      =   [future.result() for future in futures]
               self.assertEqual(_lowercase     ,       [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10)])




 def 						__snake_case		(     self   :    Optional[int]     ,       lowercase__   :    List[Any]     ,       lowercase__   :    List[Any]):







        '''simple docstring'''



        return tokenizer.encode(_lowercase     ,       truncation='longest_first'     ,       padding='longest')

"""simple docstring"""


__UpperCamelCase       :    List[Any]         =		"\n# Diffusers installation\n! pip install diffusers transformers datasets accelerate\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/diffusers.git\n"

__UpperCamelCase       :    Any         =		[{"type": "code", "content": INSTALL_CONTENT}]