
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		__magic_name__				( lowercase_						,unittest.TestCase					):


  """simple docstring"""





  _UpperCamelCase						 = PreTrainedTokenizerFast
  _UpperCamelCase						 = False
  _UpperCamelCase						 = True
  _UpperCamelCase						 = "tokenizer_file"


  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						False  # because we don't have pretrained_vocab_files_map
    super().setUp()
    _lowerCamelCase      =						True

    _lowerCamelCase      =						['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
    _lowerCamelCase      =						'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

    # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
    _lowerCamelCase      =						[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

    _lowerCamelCase      =						PreTrainedTokenizerFast.from_pretrained(model_paths[0]		)
    tokenizer.save_pretrained(self.tmpdirname		)


  def     _UpperCAmelCase  (					self		):
    # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
    # model
    pass


  def     _UpperCAmelCase  (					self		):
    # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
    # model
    pass


  def     _UpperCAmelCase  (					self		):
    # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
    # model
    pass


  def     _UpperCAmelCase  (					self		):
    # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
    pass


  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						self.tmpdirname
    # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
        try:
          _lowerCamelCase      =						tempfile.mkdtemp()
          _lowerCamelCase      =						self.rust_tokenizer_class.from_pretrained(a__	,		**a__		)

          tokenizer.save_pretrained(self.tmpdirname		)
          super().test_training_new_tokenizer()
        finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname		)
          _lowerCamelCase      =						tmpdirname_orig


  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						self.tmpdirname
    # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
        try:
          _lowerCamelCase      =						tempfile.mkdtemp()
          _lowerCamelCase      =						self.rust_tokenizer_class.from_pretrained(a__	,		**a__		)

          tokenizer.save_pretrained(self.tmpdirname		)
          super().test_training_new_tokenizer_with_special_tokens_change()
        finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname		)
          _lowerCamelCase      =						tmpdirname_orig


  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name		)

    _lowerCamelCase      =						('''a''' for _ in range(10_00		))
    _lowerCamelCase      =						tokenizer.train_new_from_iterator(text_iterator=a__	,		length=10_00	,		vocab_size=50		)

    _lowerCamelCase      =						new_tokenizer.encode('''a🤗'''		)
    self.assertEqual(a__	,		[64, 1_72, 2_53, 97, 2_45]		)







@require_tokenizers
class 		__magic_name__				( unittest.TestCase					):


  """simple docstring"""





  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						AutoTokenizer.from_pretrained('''bert-base-cased'''		)
    _lowerCamelCase      =						json.loads(tokenizer._tokenizer.to_str()		)
    _lowerCamelCase      =						len(a__		)

    with tempfile.TemporaryDirectory() as tmp_dir:
      # Hack to save this in the tokenizer_config.json
      _lowerCamelCase      =						['''tokenizer.4.0.0.json''']
      tokenizer.save_pretrained(a__		)
      json.dump(a__	,		open(os.path.join(a__	,		'''tokenizer.4.0.0.json'''		)	,		'''w'''		)		)

      # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
      _lowerCamelCase      =						AutoTokenizer.from_pretrained(a__		)
      self.assertEqual(len(a__		)	,		len(a__		) + 1		)
      _lowerCamelCase      =						json.loads(new_tokenizer._tokenizer.to_str()		)
      self.assertIn('''huggingface'''	,		json_tokenizer['''model''']['''vocab''']		)

      # Will need to be adjusted if we reach v42 and this test is still here.
      # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
      shutil.move(os.path.join(a__	,		'''tokenizer.4.0.0.json'''		)	,		os.path.join(a__	,		'''tokenizer.42.0.0.json'''		)		)
      _lowerCamelCase      =						['''tokenizer.42.0.0.json''']
      tokenizer.save_pretrained(a__		)
      _lowerCamelCase      =						AutoTokenizer.from_pretrained(a__		)
      self.assertEqual(len(a__		)	,		len(a__		)		)
      _lowerCamelCase      =						json.loads(new_tokenizer._tokenizer.to_str()		)
      self.assertNotIn('''huggingface'''	,		json_tokenizer['''model''']['''vocab''']		)


  def     _UpperCAmelCase  (					self		):
    # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
    _lowerCamelCase      =						'''hf-internal-testing/test-two-tokenizers'''

    # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
    _lowerCamelCase      =						AutoTokenizer.from_pretrained(a__		)
    self.assertEqual(len(a__		)	,		2_89_97		)
    _lowerCamelCase      =						json.loads(tokenizer._tokenizer.to_str()		)
    self.assertIn('''huggingface'''	,		json_tokenizer['''model''']['''vocab''']		)

    # Testing an older version by monkey-patching the version in the module it's used.
    import transformers as old_transformers

    _lowerCamelCase      =						'''3.0.0'''
    _lowerCamelCase      =						old_transformers.models.auto.AutoTokenizer.from_pretrained(a__		)
    self.assertEqual(len(a__		)	,		2_89_96		)
    _lowerCamelCase      =						json.loads(old_tokenizer._tokenizer.to_str()		)
    self.assertNotIn('''huggingface'''	,		json_tokenizer['''model''']['''vocab''']		)







@require_tokenizers
class 		__magic_name__				( unittest.TestCase					):


  """simple docstring"""





  def     _UpperCAmelCase  (					self		):
    # See https://github.com/huggingface/transformers/pull/12550
    # and https://github.com/huggingface/tokenizers/issues/537
    _lowerCamelCase      =						PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''		)
    _lowerCamelCase      =						'''The Matrix is a 1999 science fiction action film.'''

    with concurrent.futures.ThreadPoolExecutor() as executor:
      _lowerCamelCase      =						[executor.submit(self.fetch	,		a__	,		a__		) for i in range(10		)]
      _lowerCamelCase      =						[future.result() for future in futures]
      self.assertEqual(a__	,		[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10		)]		)


  def     _UpperCAmelCase  (					self	,		a__	,		a__		):
    return tokenizer.encode(a__	,		truncation='''longest_first'''	,		padding='''longest'''		)






import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin

class 		__magic_name__				( lowercase_					):


  """simple docstring"""





  _UpperCamelCase						 = ["image_processor", "tokenizer"]
  _UpperCamelCase						 = "AutoImageProcessor"
  _UpperCamelCase						 = "AutoTokenizer"


  def __init__(					self	,		a__=None	,		a__=None	,		**a__		):
    _lowerCamelCase      =						None
    if "feature_extractor" in kwargs:
      warnings.warn(
          '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
          ''' instead.'''	,		a__	,		)
      _lowerCamelCase      =						kwargs.pop('''feature_extractor'''		)

    _lowerCamelCase      =						image_processor if image_processor is not None else feature_extractor
    if image_processor is None:
      raise ValueError('''You need to specify an `image_processor`.'''		)
    if tokenizer is None:
      raise ValueError('''You need to specify a `tokenizer`.'''		)

    super().__init__(a__	,		a__		)
    _lowerCamelCase      =						self.image_processor
    _lowerCamelCase      =						False


  def __call__(					self	,		*a__	,		**a__		):
    # For backward compatibility
    if self._in_target_context_manager:
      return self.current_processor(*a__	,		**a__		)

    _lowerCamelCase      =						kwargs.pop('''images'''	,		a__		)
    _lowerCamelCase      =						kwargs.pop('''text'''	,		a__		)
    if len(a__		) > 0:
      _lowerCamelCase      =						args[0]
      _lowerCamelCase      =						args[1:]

    if images is None and text is None:
      raise ValueError('''You need to specify either an `images` or `text` input to process.'''		)

    if images is not None:
      _lowerCamelCase      =						self.image_processor(a__	,		*a__	,		**a__		)
    if text is not None:
      _lowerCamelCase      =						self.tokenizer(a__	,		**a__		)

    if text is None:
      return inputs
    elif images is None:
      return encodings
    else:
      _lowerCamelCase      =						encodings['''input_ids''']
      return inputs


  def     _UpperCAmelCase  (					self	,		*a__	,		**a__		):
    return self.tokenizer.batch_decode(*a__	,		**a__		)


  def     _UpperCAmelCase  (					self	,		*a__	,		**a__		):
    return self.tokenizer.decode(*a__	,		**a__		)


  @contextmanager
  def     _UpperCAmelCase  (					self		):
    warnings.warn(
        '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
        '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
        '''your images inputs, or in a separate call.'''		)
    _lowerCamelCase      =						True
    _lowerCamelCase      =						self.tokenizer
    yield
    _lowerCamelCase      =						self.image_processor
    _lowerCamelCase      =						False


  @property
  def     _UpperCAmelCase  (					self		):
    warnings.warn(
        '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''	,		a__	,		)
    return self.image_processor_class


  @property
  def     _UpperCAmelCase  (					self		):
    warnings.warn(
        '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''	,		a__	,		)
    return self.image_processor




