



'''simple docstring'''


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 						UpperCAmelCase						(		__a       ,						unittest.TestCase):


 '''simple docstring'''




 __magic_name__       :			Optional[Any]	=   PreTrainedTokenizerFast
 __magic_name__       :			Union[str, Any]	=   False
 __magic_name__       :			Union[str, Any]	=   True
 __magic_name__       :			int	=   "tokenizer_file"




 def    lowercase_	( self)			->   Tuple:



      """simple docstring"""







      a_									=False  # because we don't have pretrained_vocab_files_map
      super().setUp()
      a_									=True

      a_									=["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
      a_									="SaulLu/dummy-tokenizer-bytelevel-bpe"

      # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
      a_									=[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

      a_									=PreTrainedTokenizerFast.from_pretrained(model_paths[0])
      tokenizer.save_pretrained(self.tmpdirname)




 def    lowercase_	( self)			->   str:



      """simple docstring"""







      pass




 def    lowercase_	( self)			->   Optional[Any]:



      """simple docstring"""







      pass




 def    lowercase_	( self)			->   Any:



      """simple docstring"""







      pass




 def    lowercase_	( self)			->   Union[str, Any]:



      """simple docstring"""







      pass




 def    lowercase_	( self)			->   Tuple:



      """simple docstring"""







      a_									=self.tmpdirname
      # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                try:
                     a_									=tempfile.mkdtemp()
                     a_									=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,	**lowerCAmelCase_)

                     tokenizer.save_pretrained(self.tmpdirname)
                     super().test_training_new_tokenizer()
                finally:
                     # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                     # is restored
                     shutil.rmtree(self.tmpdirname)
                     a_									=tmpdirname_orig




 def    lowercase_	( self)			->   Optional[Any]:



      """simple docstring"""







      a_									=self.tmpdirname
      # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                try:
                     a_									=tempfile.mkdtemp()
                     a_									=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,	**lowerCAmelCase_)

                     tokenizer.save_pretrained(self.tmpdirname)
                     super().test_training_new_tokenizer_with_special_tokens_change()
                finally:
                     # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                     # is restored
                     shutil.rmtree(self.tmpdirname)
                     a_									=tmpdirname_orig




 def    lowercase_	( self)			->   List[Any]:



      """simple docstring"""







      a_									=self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name)

      a_									=("a" for _ in range(1_0_0_0))
      a_									=tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase_				,	length=1_0_0_0				,	vocab_size=5_0)

      a_									=new_tokenizer.encode("a🤗")
      self.assertEqual(lowerCAmelCase_				,	[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5])







@require_tokenizers
class 						UpperCAmelCase						(		unittest.TestCase):


 '''simple docstring'''




 def    lowercase_	( self)			->   Dict:



      """simple docstring"""







      a_									=AutoTokenizer.from_pretrained("bert-base-cased")
      a_									=json.loads(tokenizer._tokenizer.to_str())
      a_									=len(lowerCAmelCase_)

      with tempfile.TemporaryDirectory() as tmp_dir:
           # Hack to save this in the tokenizer_config.json
           a_									=["tokenizer.4.0.0.json"]
           tokenizer.save_pretrained(lowerCAmelCase_)
           json.dump(lowerCAmelCase_				,	open(os.path.join(lowerCAmelCase_				,	"tokenizer.4.0.0.json")				,	"w"))

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           a_									=AutoTokenizer.from_pretrained(lowerCAmelCase_)
           self.assertEqual(len(lowerCAmelCase_)				,	len(lowerCAmelCase_) + 1)
           a_									=json.loads(new_tokenizer._tokenizer.to_str())
           self.assertIn("huggingface"				,	json_tokenizer["model"]["vocab"])

           # Will need to be adjusted if we reach v42 and this test is still here.
           # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
           shutil.move(os.path.join(lowerCAmelCase_				,	"tokenizer.4.0.0.json")				,	os.path.join(lowerCAmelCase_				,	"tokenizer.42.0.0.json"))
           a_									=["tokenizer.42.0.0.json"]
           tokenizer.save_pretrained(lowerCAmelCase_)
           a_									=AutoTokenizer.from_pretrained(lowerCAmelCase_)
           self.assertEqual(len(lowerCAmelCase_)				,	len(lowerCAmelCase_))
           a_									=json.loads(new_tokenizer._tokenizer.to_str())
           self.assertNotIn("huggingface"				,	json_tokenizer["model"]["vocab"])




 def    lowercase_	( self)			->   str:



      """simple docstring"""







      a_									="hf-internal-testing/test-two-tokenizers"

      # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
      a_									=AutoTokenizer.from_pretrained(lowerCAmelCase_)
      self.assertEqual(len(lowerCAmelCase_)				,	2_8_9_9_7)
      a_									=json.loads(tokenizer._tokenizer.to_str())
      self.assertIn("huggingface"				,	json_tokenizer["model"]["vocab"])

      # Testing an older version by monkey-patching the version in the module it's used.
      import transformers as old_transformers

      a_									="3.0.0"
      a_									=old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase_)
      self.assertEqual(len(lowerCAmelCase_)				,	2_8_9_9_6)
      a_									=json.loads(old_tokenizer._tokenizer.to_str())
      self.assertNotIn("huggingface"				,	json_tokenizer["model"]["vocab"])







@require_tokenizers
class 						UpperCAmelCase						(		unittest.TestCase):


 '''simple docstring'''




 def    lowercase_	( self)			->   Union[str, Any]:



      """simple docstring"""







      a_									=PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel")
      a_									="The Matrix is a 1999 science fiction action film."

      with concurrent.futures.ThreadPoolExecutor() as executor:
           a_									=[executor.submit(self.fetch				,	lowerCAmelCase_				,	lowerCAmelCase_) for i in range(1_0)]
           a_									=[future.result() for future in futures]
           self.assertEqual(lowerCAmelCase_				,	[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0)])




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_)			->   List[Any]:



      """simple docstring"""







      return tokenizer.encode(lowerCAmelCase_				,	truncation="longest_first"				,	padding="longest")







'''simple docstring'''


lowercase											=      (1, 153, 370, 371, 1_634, 24_678_051, 115_132_219_018_763_992_565_095_597_973_971_522_401)
lowercase											=      (-153, -1, 0, 1.2, 200, "A", [], {}, None)






def 			UpperCAmelCase_							(		lowercase__							):


     '''simple docstring'''







     if not isinstance(lowercase__							,						lowercase__							) or n < 1:
          return False

     # Initialization of sum and number of digits.
     a_									=0
     a_									=0
     a_									=n
     # Calculation of digits of the number
     while temp > 0:
          number_of_digits += 1
          temp //= 1_0
     # Dividing number into separate digits and find Armstrong number
     a_									=n
     while temp > 0:
          a_									=temp % 1_0
          total += rem**number_of_digits
          temp //= 1_0
     return n == total






def 			UpperCAmelCase_							(		lowercase__							):


     '''simple docstring'''







     if not isinstance(lowercase__							,						lowercase__							) or n < 1:
          return False

     # Init a "histogram" of the digits
     a_									=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
     a_									=0
     a_									=0
     a_									=n
     while temp > 0:
          a_		,    a_									=divmod(lowercase__							,						1_0							)
          digit_histogram[rem] += 1
          digit_total += 1

     for cnt, i in zip(lowercase__							,						range(len(lowercase__							)							)							):
          total += cnt * i**digit_total

     return n == total






def 			UpperCAmelCase_							(		lowercase__							):


     '''simple docstring'''







     if not isinstance(lowercase__							,						lowercase__							) or n < 1:
          return False
     a_									=len(str(lowercase__							)							)  # the power that all digits will be raised to
     # check if sum of each digit multiplied expo times is equal to number
     return n == sum(int(lowercase__							) ** expo for i in str(lowercase__							)							)






def 			UpperCAmelCase_							(		):


     '''simple docstring'''







     a_									=int(input("Enter an integer to see if it is an Armstrong number: "							).strip()							)
     print(F"""{num} is {'' if armstrong_number(lowercase__							) else 'not '}an Armstrong number."""							)
     print(F"""{num} is {'' if narcissistic_number(lowercase__							) else 'not '}an Armstrong number."""							)
     print(F"""{num} is {'' if pluperfect_number(lowercase__							) else 'not '}an Armstrong number."""							)


if __name__ == "__main__":
  import doctest

  doctest.testmod()
  main()


