'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


snake_case__	:     Any      =    {
    '''configuration_prophetnet''': ['''PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ProphetNetConfig'''],
    '''tokenization_prophetnet''': ['''ProphetNetTokenizer'''],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 snake_case__	:     Dict      =    [
     '''PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
     '''ProphetNetDecoder''',
     '''ProphetNetEncoder''',
     '''ProphetNetForCausalLM''',
     '''ProphetNetForConditionalGeneration''',
     '''ProphetNetModel''',
     '''ProphetNetPreTrainedModel''',
 ]


if TYPE_CHECKING:
 from .configuration_prophetnet import PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ProphetNetConfig
 from .tokenization_prophetnet import ProphetNetTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_prophetnet import (
      PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
      ProphetNetDecoder,
      ProphetNetEncoder,
      ProphetNetForCausalLM,
      ProphetNetForConditionalGeneration,
      ProphetNetModel,
      ProphetNetPreTrainedModel,
  )

else:
 import sys

 snake_case__	:     int      =    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)













import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 						UpperCamelCase__					( lowerCAmelCase_				, unittest.TestCase  ):



     '''simple docstring'''

     __snake_case       :      Dict						      =     PreTrainedTokenizerFast
     __snake_case       :      Optional[int]						      =     False
     __snake_case       :      List[Any]						      =     True
     __snake_case       :      str						      =     "tokenizer_file"






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Union[str, Any]			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		False  # because we don't have pretrained_vocab_files_map
           super().setUp()
           SCREAMING_SNAKE_CASE						=		True

           SCREAMING_SNAKE_CASE						=		["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
           SCREAMING_SNAKE_CASE						=		"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

           # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
           SCREAMING_SNAKE_CASE						=		[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

           SCREAMING_SNAKE_CASE						=		PreTrainedTokenizerFast.from_pretrained(model_paths[0]			)
           tokenizer.save_pretrained(self.tmpdirname			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						Tuple:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						List[Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						Union[str, Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						List[Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Any			)	->						Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tmpdirname
           # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                       try:
                             SCREAMING_SNAKE_CASE						=		tempfile.mkdtemp()
                             SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,**lowerCamelCase__			)

                             tokenizer.save_pretrained(self.tmpdirname			)
                             super().test_training_new_tokenizer()
                       finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname			)
                             SCREAMING_SNAKE_CASE						=		tmpdirname_orig






     def 							SCREAMING_SNAKE_CASE__    (				self		:       int			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tmpdirname
           # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                       try:
                             SCREAMING_SNAKE_CASE						=		tempfile.mkdtemp()
                             SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,**lowerCamelCase__			)

                             tokenizer.save_pretrained(self.tmpdirname			)
                             super().test_training_new_tokenizer_with_special_tokens_change()
                       finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname			)
                             SCREAMING_SNAKE_CASE						=		tmpdirname_orig






     def 							SCREAMING_SNAKE_CASE__    (				self		:       int			)	->						Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name			)

           SCREAMING_SNAKE_CASE						=		("""a""" for _ in range(1000			))
           SCREAMING_SNAKE_CASE						=		tokenizer.train_new_from_iterator(text_iterator=lowerCamelCase__		,length=1000		,vocab_size=50			)

           SCREAMING_SNAKE_CASE						=		new_tokenizer.encode("""a🤗"""			)
           self.assertEqual(lowerCamelCase__		,[64, 172, 253, 97, 245]			)

@require_tokenizers
class 						UpperCamelCase__					( unittest.TestCase  ):



     '''simple docstring'''

     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained("""bert-base-cased"""			)
           SCREAMING_SNAKE_CASE						=		json.loads(tokenizer._tokenizer.to_str()			)
           SCREAMING_SNAKE_CASE						=		len(lowerCamelCase__			)

           with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 SCREAMING_SNAKE_CASE						=		["""tokenizer.4.0.0.json"""]
                 tokenizer.save_pretrained(lowerCamelCase__			)
                 json.dump(lowerCamelCase__		,open(os.path.join(lowerCamelCase__		,"""tokenizer.4.0.0.json"""			)		,"""w"""			)			)

                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
                 self.assertEqual(len(lowerCamelCase__			)		,len(lowerCamelCase__			) + 1			)
                 SCREAMING_SNAKE_CASE						=		json.loads(new_tokenizer._tokenizer.to_str()			)
                 self.assertIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(lowerCamelCase__		,"""tokenizer.4.0.0.json"""			)		,os.path.join(lowerCamelCase__		,"""tokenizer.42.0.0.json"""			)			)
                 SCREAMING_SNAKE_CASE						=		["""tokenizer.42.0.0.json"""]
                 tokenizer.save_pretrained(lowerCamelCase__			)
                 SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
                 self.assertEqual(len(lowerCamelCase__			)		,len(lowerCamelCase__			)			)
                 SCREAMING_SNAKE_CASE						=		json.loads(new_tokenizer._tokenizer.to_str()			)
                 self.assertNotIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Dict			)	->						Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		"""hf-internal-testing/test-two-tokenizers"""

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
           self.assertEqual(len(lowerCamelCase__			)		,28997			)
           SCREAMING_SNAKE_CASE						=		json.loads(tokenizer._tokenizer.to_str()			)
           self.assertIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

           # Testing an older version by monkey-patching the version in the module it's used.
           import transformers as old_transformers

           SCREAMING_SNAKE_CASE						=		"""3.0.0"""
           SCREAMING_SNAKE_CASE						=		old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCamelCase__			)
           self.assertEqual(len(lowerCamelCase__			)		,28996			)
           SCREAMING_SNAKE_CASE						=		json.loads(old_tokenizer._tokenizer.to_str()			)
           self.assertNotIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

@require_tokenizers
class 						UpperCamelCase__					( unittest.TestCase  ):



     '''simple docstring'''

     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""			)
           SCREAMING_SNAKE_CASE						=		"""The Matrix is a 1999 science fiction action film."""

           with concurrent.futures.ThreadPoolExecutor() as executor:
                 SCREAMING_SNAKE_CASE						=		[executor.submit(self.fetch		,lowerCamelCase__		,lowerCamelCase__			) for i in range(10			)]
                 SCREAMING_SNAKE_CASE						=		[future.result() for future in futures]
                 self.assertEqual(lowerCamelCase__		,[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10			)]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Optional[int]		,lowerCamelCase__		:       Optional[int]		,lowerCamelCase__		:       str			)	->						Union[str, Any]:




           '''simple docstring'''


           return tokenizer.encode(lowerCamelCase__		,truncation="""longest_first"""		,padding="""longest"""			)


