
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class       A__  (		lowerCAmelCase__       ,		unittest.TestCase							):
							lowerCAmelCase__						:		List[Any]            =				PreTrainedTokenizerFast
							lowerCAmelCase__						:		List[Any]            =				False
							lowerCAmelCase__						:		Union[str, Any]            =				True
							lowerCAmelCase__						:		int            =				"tokenizer_file"





							def      a__					(					self    :  str							)		->						int:



											"""simple docstring"""


											__lowercase								=   False  # because we don't have pretrained_vocab_files_map
											super().setUp()
											__lowercase								=   True

											__lowercase								=   ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
											__lowercase								=   'SaulLu/dummy-tokenizer-bytelevel-bpe'

											# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
											__lowercase								=   [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

											__lowercase								=   PreTrainedTokenizerFast.from_pretrained(model_paths[0]							)
											tokenizer.save_pretrained(self.tmpdirname							)





							def      a__					(					self    :  str							)		->						Union[str, Any]:



											"""simple docstring"""


											pass





							def      a__					(					self    :  str							)		->						Optional[Any]:



											"""simple docstring"""


											pass





							def      a__					(					self    :  Optional[Any]							)		->						str:



											"""simple docstring"""


											pass





							def      a__					(					self    :  Dict							)		->						Any:



											"""simple docstring"""


											pass





							def      a__					(					self    :  Tuple							)		->						Union[str, Any]:



											"""simple docstring"""


											__lowercase								=   self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																			try:
																							__lowercase								=   tempfile.mkdtemp()
																							__lowercase								=   self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase	,					**_UpperCAmelCase							)

																							tokenizer.save_pretrained(self.tmpdirname							)
																							super().test_training_new_tokenizer()
																			finally:
																							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																							# is restored
																							shutil.rmtree(self.tmpdirname							)
																							__lowercase								=   tmpdirname_orig





							def      a__					(					self    :  Any							)		->						Union[str, Any]:



											"""simple docstring"""


											__lowercase								=   self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																			try:
																							__lowercase								=   tempfile.mkdtemp()
																							__lowercase								=   self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase	,					**_UpperCAmelCase							)

																							tokenizer.save_pretrained(self.tmpdirname							)
																							super().test_training_new_tokenizer_with_special_tokens_change()
																			finally:
																							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																							# is restored
																							shutil.rmtree(self.tmpdirname							)
																							__lowercase								=   tmpdirname_orig





							def      a__					(					self    :  Dict							)		->						str:



											"""simple docstring"""


											__lowercase								=   self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name							)

											__lowercase								=   ('a' for _ in range(10_00							))
											__lowercase								=   tokenizer.train_new_from_iterator(text_iterator=_UpperCAmelCase	,					length=10_00	,					vocab_size=50							)

											__lowercase								=   new_tokenizer.encode('a🤗'							)
											self.assertEqual(_UpperCAmelCase	,					[64, 1_72, 2_53, 97, 2_45]							)






@require_tokenizers
class       A__  (		unittest.TestCase							):





							def      a__					(					self    :  str							)		->						str:



											"""simple docstring"""


											__lowercase								=   AutoTokenizer.from_pretrained('bert-base-cased'							)
											__lowercase								=   json.loads(tokenizer._tokenizer.to_str()							)
											__lowercase								=   len(_UpperCAmelCase							)

											with tempfile.TemporaryDirectory() as tmp_dir:
															# Hack to save this in the tokenizer_config.json
															__lowercase								=   ['tokenizer.4.0.0.json']
															tokenizer.save_pretrained(_UpperCAmelCase							)
															json.dump(_UpperCAmelCase	,					open(os.path.join(_UpperCAmelCase	,					'tokenizer.4.0.0.json'							)	,					'w'							)							)

															# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
															__lowercase								=   AutoTokenizer.from_pretrained(_UpperCAmelCase							)
															self.assertEqual(len(_UpperCAmelCase							)	,					len(_UpperCAmelCase							) + 1							)
															__lowercase								=   json.loads(new_tokenizer._tokenizer.to_str()							)
															self.assertIn('huggingface'	,					json_tokenizer['model']['vocab']							)

															# Will need to be adjusted if we reach v42 and this test is still here.
															# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
															shutil.move(os.path.join(_UpperCAmelCase	,					'tokenizer.4.0.0.json'							)	,					os.path.join(_UpperCAmelCase	,					'tokenizer.42.0.0.json'							)							)
															__lowercase								=   ['tokenizer.42.0.0.json']
															tokenizer.save_pretrained(_UpperCAmelCase							)
															__lowercase								=   AutoTokenizer.from_pretrained(_UpperCAmelCase							)
															self.assertEqual(len(_UpperCAmelCase							)	,					len(_UpperCAmelCase							)							)
															__lowercase								=   json.loads(new_tokenizer._tokenizer.to_str()							)
															self.assertNotIn('huggingface'	,					json_tokenizer['model']['vocab']							)





							def      a__					(					self    :  Union[str, Any]							)		->						Any:



											"""simple docstring"""


											__lowercase								=   'hf-internal-testing/test-two-tokenizers'

											# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
											__lowercase								=   AutoTokenizer.from_pretrained(_UpperCAmelCase							)
											self.assertEqual(len(_UpperCAmelCase							)	,					2_89_97							)
											__lowercase								=   json.loads(tokenizer._tokenizer.to_str()							)
											self.assertIn('huggingface'	,					json_tokenizer['model']['vocab']							)

											# Testing an older version by monkey-patching the version in the module it's used.
											import transformers as old_transformers

											__lowercase								=   '3.0.0'
											__lowercase								=   old_transformers.models.auto.AutoTokenizer.from_pretrained(_UpperCAmelCase							)
											self.assertEqual(len(_UpperCAmelCase							)	,					2_89_96							)
											__lowercase								=   json.loads(old_tokenizer._tokenizer.to_str()							)
											self.assertNotIn('huggingface'	,					json_tokenizer['model']['vocab']							)






@require_tokenizers
class       A__  (		unittest.TestCase							):





							def      a__					(					self    :  Any							)		->						str:



											"""simple docstring"""


											__lowercase								=   PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'							)
											__lowercase								=   'The Matrix is a 1999 science fiction action film.'

											with concurrent.futures.ThreadPoolExecutor() as executor:
															__lowercase								=   [executor.submit(self.fetch	,					_UpperCAmelCase	,					_UpperCAmelCase							) for i in range(10							)]
															__lowercase								=   [future.result() for future in futures]
															self.assertEqual(_UpperCAmelCase	,					[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10							)]							)





							def      a__					(					self    :  Optional[int]	,					_UpperCAmelCase    :  str	,					_UpperCAmelCase    :  List[str]							)		->						str:



											"""simple docstring"""


											return tokenizer.encode(_UpperCAmelCase	,					truncation='longest_first'	,					padding='longest'							)







from ctypes import c_float, sizeof
from enum import Enum
from typing import TYPE_CHECKING, Optional, Union


if TYPE_CHECKING:
							from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer  # tests_ignore
class       A__  (		lowerCAmelCase__							):
							lowerCAmelCase__						:		Any            =				c_float





							@property
							def      a__					(					self    :  List[str]							)		->						int:



											"""simple docstring"""


											return sizeof(self.value							)






def 				__SCREAMING_SNAKE_CASE  (						SCREAMING_SNAKE_CASE			: int       ,				SCREAMING_SNAKE_CASE			: int       ,				SCREAMING_SNAKE_CASE			: int = 0     )	->		int:
				# < 0 is possible if using a dynamic axis
				if dimension <= 0:
								__lowercase								=   fixed_dimension

				dimension -= num_token_to_add
				return dimension




def 				__SCREAMING_SNAKE_CASE  (						SCREAMING_SNAKE_CASE			: int       ,				SCREAMING_SNAKE_CASE			: ParameterFormat     )	->		int:
				return num_parameters * dtype.size




def 				__SCREAMING_SNAKE_CASE  (						SCREAMING_SNAKE_CASE			: str     )	->		Optional[Union["AutoTokenizer", "AutoFeatureExtractor", "AutoProcessor"]]:
				from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer  # tests_ignore

				try:
								return AutoProcessor.from_pretrained(SCREAMING_SNAKE_CASE     )
				except (ValueError, OSError, KeyError):
								__lowercase						,   __lowercase								=   None, None
								try:
												__lowercase								=   AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE     )
								except (OSError, KeyError):
												pass
								try:
												__lowercase								=   AutoFeatureExtractor.from_pretrained(SCREAMING_SNAKE_CASE     )
								except (OSError, KeyError):
												pass

								if tokenizer is not None and feature_extractor is not None:
												raise ValueError(
												    F"""Couldn't auto-detect preprocessor for {model_name}. Found both a tokenizer and a feature extractor."""     )
								elif tokenizer is None and feature_extractor is None:
												return None
								elif tokenizer is not None:
												return tokenizer
								else:
												return feature_extractor





