






"""simple docstring"""


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 		snake_case    (    UpperCAmelCase  ,					unittest.TestCase			):
    __magic_name__       	=	PreTrainedTokenizerFast
    __magic_name__       	=	False
    __magic_name__       	=	True
    __magic_name__       	=	'''tokenizer_file'''





    def 							lowerCamelCase__ (				self      :   Any     ):


      '''simple docstring'''




      a	:     Optional[int]							=  False  # because we don't have pretrained_vocab_files_map
      super().setUp()
      a	:     int							=  True

      a	:     Tuple							=  ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
      a	:     Tuple							=  'SaulLu/dummy-tokenizer-bytelevel-bpe'

      # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
      a	:     Tuple							=  [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

      a	:     str							=  PreTrainedTokenizerFast.from_pretrained(model_paths[0]     )
      tokenizer.save_pretrained(self.tmpdirname     )





    def 							lowerCamelCase__ (				self      :   str     ):


      '''simple docstring'''




      pass





    def 							lowerCamelCase__ (				self      :   Optional[int]     ):


      '''simple docstring'''




      pass





    def 							lowerCamelCase__ (				self      :   Optional[int]     ):


      '''simple docstring'''




      pass





    def 							lowerCamelCase__ (				self      :   str     ):


      '''simple docstring'''




      pass





    def 							lowerCamelCase__ (				self      :   Union[str, Any]     ):


      '''simple docstring'''




      a	:     Optional[Any]							=  self.tmpdirname
      # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
          try:
            a	:     str							=  tempfile.mkdtemp()
            a	:     Dict							=  self.rust_tokenizer_class.from_pretrained(A							,						**A     )

            tokenizer.save_pretrained(self.tmpdirname     )
            super().test_training_new_tokenizer()
          finally:
            # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
            # is restored
            shutil.rmtree(self.tmpdirname     )
            a	:     int							=  tmpdirname_orig





    def 							lowerCamelCase__ (				self      :   str     ):


      '''simple docstring'''




      a	:     Any							=  self.tmpdirname
      # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
          try:
            a	:     Optional[Any]							=  tempfile.mkdtemp()
            a	:     List[Any]							=  self.rust_tokenizer_class.from_pretrained(A							,						**A     )

            tokenizer.save_pretrained(self.tmpdirname     )
            super().test_training_new_tokenizer_with_special_tokens_change()
          finally:
            # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
            # is restored
            shutil.rmtree(self.tmpdirname     )
            a	:     Union[str, Any]							=  tmpdirname_orig





    def 							lowerCamelCase__ (				self      :   Optional[int]     ):


      '''simple docstring'''




      a	:     Tuple							=  self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name     )

      a	:     Dict							=  ('a' for _ in range(1_0_0_0     ))
      a	:     List[Any]							=  tokenizer.train_new_from_iterator(text_iterator=A							,						length=1_0_0_0							,						vocab_size=5_0     )

      a	:     List[str]							=  new_tokenizer.encode('a🤗'     )
      self.assertEqual(A							,						[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]     )






@require_tokenizers
class 		snake_case    (    unittest.TestCase			):





    def 							lowerCamelCase__ (				self      :   Dict     ):


      '''simple docstring'''




      a	:     List[Any]							=  AutoTokenizer.from_pretrained('bert-base-cased'     )
      a	:     str							=  json.loads(tokenizer._tokenizer.to_str()     )
      a	:     Optional[int]							=  len(A     )

      with tempfile.TemporaryDirectory() as tmp_dir:
        # Hack to save this in the tokenizer_config.json
        a	:     List[Any]							=  ['tokenizer.4.0.0.json']
        tokenizer.save_pretrained(A     )
        json.dump(A							,						open(os.path.join(A							,						'tokenizer.4.0.0.json'     )							,						'w'     )     )

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        a	:     Tuple							=  AutoTokenizer.from_pretrained(A     )
        self.assertEqual(len(A     )							,						len(A     ) + 1     )
        a	:     Tuple							=  json.loads(new_tokenizer._tokenizer.to_str()     )
        self.assertIn('huggingface'							,						json_tokenizer['model']['vocab']     )

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
        shutil.move(os.path.join(A							,						'tokenizer.4.0.0.json'     )							,						os.path.join(A							,						'tokenizer.42.0.0.json'     )     )
        a	:     Any							=  ['tokenizer.42.0.0.json']
        tokenizer.save_pretrained(A     )
        a	:     int							=  AutoTokenizer.from_pretrained(A     )
        self.assertEqual(len(A     )							,						len(A     )     )
        a	:     List[str]							=  json.loads(new_tokenizer._tokenizer.to_str()     )
        self.assertNotIn('huggingface'							,						json_tokenizer['model']['vocab']     )





    def 							lowerCamelCase__ (				self      :   Optional[int]     ):


      '''simple docstring'''




      a	:     int							=  'hf-internal-testing/test-two-tokenizers'

      # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
      a	:     Tuple							=  AutoTokenizer.from_pretrained(A     )
      self.assertEqual(len(A     )							,						2_8_9_9_7     )
      a	:     List[str]							=  json.loads(tokenizer._tokenizer.to_str()     )
      self.assertIn('huggingface'							,						json_tokenizer['model']['vocab']     )

      # Testing an older version by monkey-patching the version in the module it's used.
      import transformers as old_transformers

      a	:     Any							=  '3.0.0'
      a	:     int							=  old_transformers.models.auto.AutoTokenizer.from_pretrained(A     )
      self.assertEqual(len(A     )							,						2_8_9_9_6     )
      a	:     List[Any]							=  json.loads(old_tokenizer._tokenizer.to_str()     )
      self.assertNotIn('huggingface'							,						json_tokenizer['model']['vocab']     )






@require_tokenizers
class 		snake_case    (    unittest.TestCase			):





    def 							lowerCamelCase__ (				self      :   Optional[int]     ):


      '''simple docstring'''




      a	:     List[Any]							=  PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'     )
      a	:     Optional[Any]							=  'The Matrix is a 1999 science fiction action film.'

      with concurrent.futures.ThreadPoolExecutor() as executor:
        a	:     Optional[Any]							=  [executor.submit(self.fetch							,						A							,						A     ) for i in range(1_0     )]
        a	:     List[Any]							=  [future.result() for future in futures]
        self.assertEqual(A							,						[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0     )]     )





    def 							lowerCamelCase__ (				self      :   str							,						A      :   str							,						A      :   List[str]     ):


      '''simple docstring'''




      return tokenizer.encode(A							,						truncation='longest_first'							,						padding='longest'     )












"""simple docstring"""


from __future__ import annotations




def      snake_case    (A_ :int ):







  '''simple docstring'''

  assert (
      isinstance(A_      ,     A_ ) and mask > 0
  ), f'''mask needs to be positive integer, your input {mask}'''
  a	:     Any							=  []
  a	:     Union[str, Any]							=  mask

  while submask:
    all_submasks.append(A_ )
    a	:     Any							=  (submask - 1) & mask

  return all_submasks


if __name__ == "__main__":
   import doctest

   doctest.testmod()




