



'''simple docstring'''







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class    lowerCamelCase_				(						lowerCAmelCase_       ,     unittest.TestCase	):






   '''simple docstring'''

   A			:  List[str]					=						PreTrainedTokenizerFast
   A			:  Dict					=						False
   A			:  int					=						True
   A			:  List[Any]					=						"tokenizer_file"


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      False  # because we don't have pretrained_vocab_files_map
       super().setUp()
       UpperCamelCase__	     =      True

       UpperCamelCase__	     =      ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
       UpperCamelCase__	     =      "SaulLu/dummy-tokenizer-bytelevel-bpe"

       # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
       UpperCamelCase__	     =      [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

       UpperCamelCase__	     =      PreTrainedTokenizerFast.from_pretrained(model_paths[0]			)
       tokenizer.save_pretrained(self.tmpdirname			)


   def 						_UpperCamelCase							(	self			):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass


   def 						_UpperCamelCase							(	self			):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass


   def 						_UpperCamelCase							(	self			):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass


   def 						_UpperCamelCase							(	self			):
       # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
       pass


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
               try:
                   UpperCamelCase__	     =      tempfile.mkdtemp()
                   UpperCamelCase__	     =      self.rust_tokenizer_class.from_pretrained(__UpperCamelCase  ,    **__UpperCamelCase			)

                   tokenizer.save_pretrained(self.tmpdirname			)
                   super().test_training_new_tokenizer()
               finally:
                   # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                   # is restored
                   shutil.rmtree(self.tmpdirname			)
                   UpperCamelCase__	     =      tmpdirname_orig


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
               try:
                   UpperCamelCase__	     =      tempfile.mkdtemp()
                   UpperCamelCase__	     =      self.rust_tokenizer_class.from_pretrained(__UpperCamelCase  ,    **__UpperCamelCase			)

                   tokenizer.save_pretrained(self.tmpdirname			)
                   super().test_training_new_tokenizer_with_special_tokens_change()
               finally:
                   # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                   # is restored
                   shutil.rmtree(self.tmpdirname			)
                   UpperCamelCase__	     =      tmpdirname_orig







   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name			)

       UpperCamelCase__	     =      ("a" for _ in range(10_00			))
       UpperCamelCase__	     =      tokenizer.train_new_from_iterator(text_iterator=__UpperCamelCase  ,    length=10_00  ,    vocab_size=50			)

       UpperCamelCase__	     =      new_tokenizer.encode("a🤗"			)
       self.assertEqual(__UpperCamelCase  ,    [64, 1_72, 2_53, 97, 2_45]			)

@require_tokenizers
class    lowerCamelCase_				(						unittest.TestCase	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-cased"			)
       UpperCamelCase__	     =      json.loads(tokenizer._tokenizer.to_str()			)
       UpperCamelCase__	     =      len(__UpperCamelCase			)

       with tempfile.TemporaryDirectory() as tmp_dir:
           # Hack to save this in the tokenizer_config.json
           UpperCamelCase__	     =      ["tokenizer.4.0.0.json"]
           tokenizer.save_pretrained(__UpperCamelCase			)
           json.dump(__UpperCamelCase  ,    open(os.path.join(__UpperCamelCase  ,    "tokenizer.4.0.0.json"			)  ,    "w"			)			)

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           UpperCamelCase__	     =      AutoTokenizer.from_pretrained(__UpperCamelCase			)
           self.assertEqual(len(__UpperCamelCase			)  ,    len(__UpperCamelCase			) + 1			)
           UpperCamelCase__	     =      json.loads(new_tokenizer._tokenizer.to_str()			)
           self.assertIn("huggingface"  ,    json_tokenizer["model"]["vocab"]			)

           # Will need to be adjusted if we reach v42 and this test is still here.
           # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
           shutil.move(os.path.join(__UpperCamelCase  ,    "tokenizer.4.0.0.json"			)  ,    os.path.join(__UpperCamelCase  ,    "tokenizer.42.0.0.json"			)			)
           UpperCamelCase__	     =      ["tokenizer.42.0.0.json"]
           tokenizer.save_pretrained(__UpperCamelCase			)
           UpperCamelCase__	     =      AutoTokenizer.from_pretrained(__UpperCamelCase			)
           self.assertEqual(len(__UpperCamelCase			)  ,    len(__UpperCamelCase			)			)
           UpperCamelCase__	     =      json.loads(new_tokenizer._tokenizer.to_str()			)
           self.assertNotIn("huggingface"  ,    json_tokenizer["model"]["vocab"]			)







   def 						_UpperCamelCase							(	self			):
       # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
       UpperCamelCase__	     =      "hf-internal-testing/test-two-tokenizers"

       # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained(__UpperCamelCase			)
       self.assertEqual(len(__UpperCamelCase			)  ,    2_89_97			)
       UpperCamelCase__	     =      json.loads(tokenizer._tokenizer.to_str()			)
       self.assertIn("huggingface"  ,    json_tokenizer["model"]["vocab"]			)

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       UpperCamelCase__	     =      "3.0.0"
       UpperCamelCase__	     =      old_transformers.models.auto.AutoTokenizer.from_pretrained(__UpperCamelCase			)
       self.assertEqual(len(__UpperCamelCase			)  ,    2_89_96			)
       UpperCamelCase__	     =      json.loads(old_tokenizer._tokenizer.to_str()			)
       self.assertNotIn("huggingface"  ,    json_tokenizer["model"]["vocab"]			)

@require_tokenizers
class    lowerCamelCase_				(						unittest.TestCase	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       # See https://github.com/huggingface/transformers/pull/12550
       # and https://github.com/huggingface/tokenizers/issues/537
       UpperCamelCase__	     =      PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"			)
       UpperCamelCase__	     =      "The Matrix is a 1999 science fiction action film."

       with concurrent.futures.ThreadPoolExecutor() as executor:
           UpperCamelCase__	     =      [executor.submit(self.fetch  ,    __UpperCamelCase  ,    __UpperCamelCase			) for i in range(10			)]
           UpperCamelCase__	     =      [future.result() for future in futures]
           self.assertEqual(__UpperCamelCase  ,    [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10			)]			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase			):
       return tokenizer.encode(__UpperCamelCase  ,    truncation="longest_first"  ,    padding="longest"			)









'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase__  :	Optional[Any]            =      {
    'configuration_marian': ['MARIAN_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MarianConfig', 'MarianOnnxConfig'],
}

try:
      if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase__  :	List[Any]            =      ['MarianTokenizer']

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase__  :	List[str]            =      [
          'MARIAN_PRETRAINED_MODEL_ARCHIVE_LIST',
          'MarianForCausalLM',
          'MarianModel',
          'MarianMTModel',
          'MarianPreTrainedModel',
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase__  :	Optional[Any]            =      ['TFMarianModel', 'TFMarianMTModel', 'TFMarianPreTrainedModel']

try:
      if not is_flax_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase__  :	Any            =      ['FlaxMarianModel', 'FlaxMarianMTModel', 'FlaxMarianPreTrainedModel']

if TYPE_CHECKING:
      from .configuration_marian import MARIAN_PRETRAINED_CONFIG_ARCHIVE_MAP, MarianConfig, MarianOnnxConfig

      try:
            if not is_sentencepiece_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_marian import MarianTokenizer

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_marian import (
                MARIAN_PRETRAINED_MODEL_ARCHIVE_LIST,
                MarianForCausalLM,
                MarianModel,
                MarianMTModel,
                MarianPreTrainedModel,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_marian import TFMarianModel, TFMarianMTModel, TFMarianPreTrainedModel

      try:
            if not is_flax_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_flax_marian import FlaxMarianModel, FlaxMarianMTModel, FlaxMarianPreTrainedModel

else:
      import sys

      lowercase__  :	Any            =      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




