
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		lowercase_							(							__SCREAMING_SNAKE_CASE				,						unittest.TestCase					):
 A__    :					Dict    =     PreTrainedTokenizerFast
 A__    :					Optional[Any]    =     False
 A__    :					Optional[Any]    =     True
 A__    :					List[str]    =     """tokenizer_file"""

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        UpperCamelCase_											=							False  # because we don't have pretrained_vocab_files_map
        super().setUp()
        UpperCamelCase_											=							True

        UpperCamelCase_											=							["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
        UpperCamelCase_											=							"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

        # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
        UpperCamelCase_											=							[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

        UpperCamelCase_											=							PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
        tokenizer.save_pretrained(self.tmpdirname  )

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        pass

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        pass

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        pass

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        pass

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        UpperCamelCase_											=							self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                      try:
                             UpperCamelCase_											=							tempfile.mkdtemp()
                             UpperCamelCase_											=							self.rust_tokenizer_class.from_pretrained(__UpperCamelCase  ,						**__UpperCamelCase  )

                             tokenizer.save_pretrained(self.tmpdirname  )
                             super().test_training_new_tokenizer()
                      finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname  )
                             UpperCamelCase_											=							tmpdirname_orig

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        UpperCamelCase_											=							self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                      try:
                             UpperCamelCase_											=							tempfile.mkdtemp()
                             UpperCamelCase_											=							self.rust_tokenizer_class.from_pretrained(__UpperCamelCase  ,						**__UpperCamelCase  )

                             tokenizer.save_pretrained(self.tmpdirname  )
                             super().test_training_new_tokenizer_with_special_tokens_change()
                      finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname  )
                             UpperCamelCase_											=							tmpdirname_orig

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        UpperCamelCase_											=							self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

        UpperCamelCase_											=							("""a""" for _ in range(1_0_0_0  ))
        UpperCamelCase_											=							tokenizer.train_new_from_iterator(text_iterator=__UpperCamelCase  ,						length=1_0_0_0  ,						vocab_size=5_0  )

        UpperCamelCase_											=							new_tokenizer.encode("""a🤗"""  )
        self.assertEqual(__UpperCamelCase  ,						[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]  )



@require_tokenizers
class 		lowercase_							(							unittest.TestCase					):

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        UpperCamelCase_											=							AutoTokenizer.from_pretrained("""bert-base-cased"""  )
        UpperCamelCase_											=							json.loads(tokenizer._tokenizer.to_str()  )
        UpperCamelCase_											=							len(__UpperCamelCase  )

        with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               UpperCamelCase_											=							["""tokenizer.4.0.0.json"""]
               tokenizer.save_pretrained(__UpperCamelCase  )
               json.dump(__UpperCamelCase  ,						open(os.path.join(__UpperCamelCase  ,						"""tokenizer.4.0.0.json"""  )  ,						"""w"""  )  )

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               UpperCamelCase_											=							AutoTokenizer.from_pretrained(__UpperCamelCase  )
               self.assertEqual(len(__UpperCamelCase  )  ,						len(__UpperCamelCase  ) + 1  )
               UpperCamelCase_											=							json.loads(new_tokenizer._tokenizer.to_str()  )
               self.assertIn("""huggingface"""  ,						json_tokenizer["""model"""]["""vocab"""]  )

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(__UpperCamelCase  ,						"""tokenizer.4.0.0.json"""  )  ,						os.path.join(__UpperCamelCase  ,						"""tokenizer.42.0.0.json"""  )  )
               UpperCamelCase_											=							["""tokenizer.42.0.0.json"""]
               tokenizer.save_pretrained(__UpperCamelCase  )
               UpperCamelCase_											=							AutoTokenizer.from_pretrained(__UpperCamelCase  )
               self.assertEqual(len(__UpperCamelCase  )  ,						len(__UpperCamelCase  )  )
               UpperCamelCase_											=							json.loads(new_tokenizer._tokenizer.to_str()  )
               self.assertNotIn("""huggingface"""  ,						json_tokenizer["""model"""]["""vocab"""]  )

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        UpperCamelCase_											=							"""hf-internal-testing/test-two-tokenizers"""

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        UpperCamelCase_											=							AutoTokenizer.from_pretrained(__UpperCamelCase  )
        self.assertEqual(len(__UpperCamelCase  )  ,						2_8_9_9_7  )
        UpperCamelCase_											=							json.loads(tokenizer._tokenizer.to_str()  )
        self.assertIn("""huggingface"""  ,						json_tokenizer["""model"""]["""vocab"""]  )

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        UpperCamelCase_											=							"""3.0.0"""
        UpperCamelCase_											=							old_transformers.models.auto.AutoTokenizer.from_pretrained(__UpperCamelCase  )
        self.assertEqual(len(__UpperCamelCase  )  ,						2_8_9_9_6  )
        UpperCamelCase_											=							json.loads(old_tokenizer._tokenizer.to_str()  )
        self.assertNotIn("""huggingface"""  ,						json_tokenizer["""model"""]["""vocab"""]  )



@require_tokenizers
class 		lowercase_							(							unittest.TestCase					):

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        UpperCamelCase_											=							PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""  )
        UpperCamelCase_											=							"""The Matrix is a 1999 science fiction action film."""

        with concurrent.futures.ThreadPoolExecutor() as executor:
               UpperCamelCase_											=							[executor.submit(self.fetch  ,						__UpperCamelCase  ,						__UpperCamelCase  ) for i in range(1_0  )]
               UpperCamelCase_											=							[future.result() for future in futures]
               self.assertEqual(__UpperCamelCase  ,						[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0  )]  )

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase  ):




        """simple docstring"""


        return tokenizer.encode(__UpperCamelCase  ,						truncation="""longest_first"""  ,						padding="""longest"""  )





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_A							=       {'''configuration_dinat''': ['''DINAT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''DinatConfig''']}


try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _A							=       [
          '''DINAT_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''DinatForImageClassification''',
          '''DinatModel''',
          '''DinatPreTrainedModel''',
          '''DinatBackbone''',
      ]

if TYPE_CHECKING:
      from .configuration_dinat import DINAT_PRETRAINED_CONFIG_ARCHIVE_MAP, DinatConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_dinat import (
                DINAT_PRETRAINED_MODEL_ARCHIVE_LIST,
                DinatBackbone,
                DinatForImageClassification,
                DinatModel,
                DinatPreTrainedModel,
            )

else:
      import sys

      _A							=       _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



