





'''simple docstring'''






import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import OPTConfig, is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
	import os

	# The slow tests are often failing with OOM error on GPU
	# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
	# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
	__UpperCamelCase      :   List[Any]								=     '''platform'''

	import jax
	import jax.numpy as jnp

	from transformers import FlaxOPTForCausalLM, FlaxOPTModel, GPTaTokenizer





def      lowercase  (  lowerCAmelCase :						List[str]   , lowerCAmelCase :						str   , lowerCAmelCase :						Any=None   , lowerCAmelCase :						str=None):



				"""simple docstring"""


				if attention_mask is None:
								_A :				Optional[Any]				=  np.where(input_ids != config.pad_token_id   , 1   , 0)
				return {
				    "input_ids": input_ids,
				    "attention_mask": attention_mask,
				}



@require_flax
class  lowerCamelCase__				:


	"""simple docstring"""





	def __init__(					self   ,     UpperCAmelCase__   ,     UpperCAmelCase__=1_3   ,     UpperCAmelCase__=7   ,     UpperCAmelCase__=True   ,     UpperCAmelCase__=False   ,     UpperCAmelCase__=9_9   ,     UpperCAmelCase__=1_6   ,     UpperCAmelCase__=2   ,     UpperCAmelCase__=4   ,     UpperCAmelCase__=4   ,     UpperCAmelCase__="gelu"   ,     UpperCAmelCase__=0.1   ,     UpperCAmelCase__=0.1   ,     UpperCAmelCase__=2_0   ,     UpperCAmelCase__=2   ,     UpperCAmelCase__=1   ,     UpperCAmelCase__=0   ,     UpperCAmelCase__=1_6   ,     UpperCAmelCase__=1_6   ,     UpperCAmelCase__=0.0_2   ,     )       ->				Dict:
					_A :				Optional[Any]				=  parent
					_A :				Optional[Any]				=  batch_size
					_A :				List[str]				=  seq_length
					_A :				Any				=  is_training
					_A :				Dict				=  use_labels
					_A :				Any				=  vocab_size
					_A :				Union[str, Any]				=  hidden_size
					_A :				Dict				=  num_hidden_layers
					_A :				Optional[int]				=  num_attention_heads
					_A :				int				=  intermediate_size
					_A :				Optional[int]				=  hidden_act
					_A :				Optional[Any]				=  hidden_dropout_prob
					_A :				Any				=  attention_probs_dropout_prob
					_A :				List[Any]				=  max_position_embeddings
					_A :				Union[str, Any]				=  eos_token_id
					_A :				Union[str, Any]				=  pad_token_id
					_A :				List[str]				=  bos_token_id
					_A :				List[str]				=  embed_dim
					_A :				Optional[int]				=  word_embed_proj_dim
					_A :				str				=  initializer_range
					_A :				Tuple				=  False




	def 				_lowerCamelCase				(					self			)       ->				Optional[Any]:
					_A :				Optional[int]				=  np.clip(ids_tensor([self.batch_size, self.seq_length - 1]   ,     self.vocab_size			)   ,     3   ,     self.vocab_size			)
					_A :				List[Any]				=  np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)   ,     dtype=np.intaa			))   ,     -1			)

					_A :				Optional[int]				=  OPTConfig(
					    vocab_size=self.vocab_size   ,     hidden_size=self.hidden_size   ,     num_hidden_layers=self.num_hidden_layers   ,     num_attention_heads=self.num_attention_heads   ,     ffn_dim=self.intermediate_size   ,     dropout=self.hidden_dropout_prob   ,     attention_dropout=self.attention_probs_dropout_prob   ,     max_position_embeddings=self.max_position_embeddings   ,     eos_token_id=self.eos_token_id   ,     bos_token_id=self.bos_token_id   ,     pad_token_id=self.pad_token_id   ,     embed_dim=self.embed_dim   ,     is_encoder_decoder=lowerCamelCase__   ,     word_embed_proj_dim=self.word_embed_proj_dim   ,     initializer_range=self.initializer_range   ,     use_cache=lowerCamelCase__   ,     )
					_A :				int				=  prepare_opt_inputs_dict(lowerCamelCase__   ,     lowerCamelCase__			)
					return config, inputs_dict




	def 				_lowerCamelCase				(					self			)       ->				Tuple:
					_A	,       _A :				Dict				=  self.prepare_config_and_inputs()
					return config, inputs_dict




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__   ,     UpperCAmelCase__   ,     UpperCAmelCase__			)       ->				Optional[Any]:
					_A :				Tuple				=  2_0
					_A :				Tuple				=  model_class_name(lowerCamelCase__			)

					_A :				Tuple				=  inputs_dict['''input_ids''']
					_A :				str				=  inputs_dict['''attention_mask''']

					_A :				List[Any]				=  model.init_cache(input_ids.shape[0]   ,     lowerCamelCase__			)
					_A :				List[Any]				=  jnp.ones((input_ids.shape[0], max_length)   ,     dtype='''i4'''			)

					_A :				List[Any]				=  jnp.broadcast_to(
					    jnp.arange(input_ids.shape[-1] - 1			)[None, :]   ,     (input_ids.shape[0], input_ids.shape[-1] - 1)   ,     )
					_A :				List[Any]				=  model(
					    input_ids[:, :-1]   ,     attention_mask=lowerCamelCase__   ,     past_key_values=lowerCamelCase__   ,     position_ids=lowerCamelCase__   ,     )

					_A :				Optional[int]				=  jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]   ,     dtype='''i4'''			)
					_A :				Optional[int]				=  model(
					    input_ids[:, -1:]   ,     attention_mask=lowerCamelCase__   ,     past_key_values=outputs_cache.past_key_values   ,     position_ids=lowerCamelCase__   ,     )

					_A :				Tuple				=  model(lowerCamelCase__			)

					_A :				Optional[int]				=  np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])			)			)
					self.parent.assertTrue(diff < 1e-3   ,     msg=F"""Max diff is {diff}"""			)




	def 				_lowerCamelCase				(					self   ,     UpperCAmelCase__   ,     UpperCAmelCase__   ,     UpperCAmelCase__			)       ->				Dict:
					_A :				Dict				=  2_0
					_A :				Union[str, Any]				=  model_class_name(lowerCamelCase__			)

					_A	,       _A :				List[Any]				=  (
					    inputs_dict['''input_ids'''],
					    inputs_dict['''attention_mask'''],
					)

					_A :				Optional[int]				=  jnp.concatenate(
					    [
					        attention_mask,
					        jnp.zeros((attention_mask.shape[0], max_length - attention_mask.shape[1])			),
					    ]   ,     axis=-1   ,     )

					_A :				Tuple				=  model.init_cache(input_ids.shape[0]   ,     lowerCamelCase__			)
					_A :				Optional[Any]				=  jnp.broadcast_to(
					    jnp.arange(input_ids.shape[-1] - 1			)[None, :]   ,     (input_ids.shape[0], input_ids.shape[-1] - 1)   ,     )

					_A :				Optional[Any]				=  model(
					    input_ids[:, :-1]   ,     attention_mask=lowerCamelCase__   ,     past_key_values=lowerCamelCase__   ,     position_ids=lowerCamelCase__   ,     )
					_A :				List[Any]				=  jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]   ,     dtype='''i4'''			)
					_A :				List[Any]				=  model(
					    input_ids[:, -1:]   ,     past_key_values=outputs_cache.past_key_values   ,     attention_mask=lowerCamelCase__   ,     position_ids=lowerCamelCase__   ,     )

					_A :				Union[str, Any]				=  model(lowerCamelCase__   ,     attention_mask=lowerCamelCase__			)

					_A :				Union[str, Any]				=  np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])			)			)
					self.parent.assertTrue(diff < 1e-3   ,     msg=F"""Max diff is {diff}"""			)



@require_flax
class  lowerCamelCase__				(						lowerCAmelCase_  ,							unittest.TestCase  ,							lowerCAmelCase_						):


	"""simple docstring"""





	__magic_name__							   =		(FlaxOPTModel, FlaxOPTForCausalLM) if is_flax_available() else ()
	__magic_name__							   =		() if is_flax_available() else ()




	def 				_lowerCamelCase				(					self			)       ->				Optional[int]:
					_A :				Dict				=  FlaxOPTModelTester(self			)




	def 				_lowerCamelCase				(					self			)       ->				Optional[int]:
					_A	,       _A :				List[str]				=  self.model_tester.prepare_config_and_inputs()
					for model_class in self.all_model_classes:
									self.model_tester.check_use_cache_forward(lowerCamelCase__   ,     lowerCamelCase__   ,     lowerCamelCase__			)




	def 				_lowerCamelCase				(					self			)       ->				int:
					_A	,       _A :				Any				=  self.model_tester.prepare_config_and_inputs()
					for model_class in self.all_model_classes:
									self.model_tester.check_use_cache_forward_with_attn_mask(lowerCamelCase__   ,     lowerCamelCase__   ,     lowerCamelCase__			)




	@slow
	def 				_lowerCamelCase				(					self			)       ->				Union[str, Any]:


					for model_class_name in self.all_model_classes:
									_A :				str				=  model_class_name.from_pretrained('''facebook/opt-125m'''			)
									_A :				str				=  np.ones((1, 1)			) * model.config.eos_token_id
									_A :				str				=  model(lowerCamelCase__			)
									self.assertIsNotNone(lowerCamelCase__			)



@require_sentencepiece
@require_flax
class  lowerCamelCase__				(						unittest.TestCase						):


	"""simple docstring"""





	@slow
	def 				_lowerCamelCase				(					self			)       ->				List[Any]:
					_A :				Optional[int]				=  FlaxOPTModel.from_pretrained('''facebook/opt-350m'''			)
					_A :				List[Any]				=  jnp.array([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]			)
					_A :				int				=  model(input_ids=lowerCamelCase__			).last_hidden_state
					_A :				Optional[int]				=  (1, 1_1, 5_1_2)
					self.assertEqual(output.shape   ,     lowerCamelCase__			)
					_A :				int				=  jnp.array(
					    [[-0.2_8_6_7, -1.9_2_5_6, -0.3_0_6_2], [-1.2_7_1_1, -0.1_3_3_7, -0.1_8_9_7], [0.4_1_0_9, 0.1_1_8_7, -1.3_1_4_2]]			)
					self.assertTrue(jnp.allclose(output[:, :3, :3]   ,     lowerCamelCase__   ,     atol=4e-2			)			)



@require_flax
@slow
class  lowerCamelCase__				(						unittest.TestCase						):


	"""simple docstring"""





	def 				_lowerCamelCase				(					self			)       ->				str:
					super().setUp()
					_A :				Dict				=  '''facebook/opt-350m'''




	def 				_lowerCamelCase				(					self			)       ->				Any:
					_A :				Any				=  FlaxOPTForCausalLM.from_pretrained(self.path_model			)
					_A :				str				=  GPTaTokenizer.from_pretrained(self.path_model			)

					_A :				Optional[Any]				=  [
					    '''Today is a beautiful day and I want to''',
					    '''In the city of''',
					    '''Paris is the capital of France and''',
					    '''Computers and mobile phones have taken''',
					]
					# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
					_A :				Optional[int]				=  tokenizer(lowerCamelCase__   ,     return_tensors='''jax'''   ,     padding=lowerCamelCase__   ,     add_special_tokens=lowerCamelCase__			)
					_A :				int				=  model(inputs.input_ids   ,     attention_mask=inputs.attention_mask			)[0].mean(axis=-1			)
					_A :				int				=  jnp.array(
					    [
					        [1.3_8_5_1, -1_3.8_9_2_3, -1_0.5_2_2_9, -1_0.7_5_3_3, -0.2_3_0_9, -1_0.2_3_8_4, -0.5_3_6_5, -9.0_9_4_7, -5.1_6_7_0],
					        [-4.7_0_7_3, -1_0.6_2_7_6, -3.9_4_1_5, -2_1.5_2_4_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2],
					        [0.6_2_4_7, -3.4_2_2_9, -8.9_1_7_9, -1.4_2_9_7, -1_4.1_6_5_0, 1.4_1_4_6, -9.0_2_1_8, -0.2_7_0_3, -0.2_7_0_3],
					        [6.4_7_8_3, -1.9_9_1_3, -1_0.7_9_2_6, -2.3_3_3_6, 1.5_0_9_2, -0.9_9_7_4, -6.8_2_1_3, 1.3_4_7_7, 1.3_4_7_7],
					    ]			)
					self.assertTrue(jnp.allclose(lowerCamelCase__   ,     lowerCamelCase__   ,     atol=4e-2			)			)

					_A :				Optional[Any]				=  jax.jit(lowerCamelCase__			)
					_A :				Optional[int]				=  model(inputs.input_ids   ,     attention_mask=inputs.attention_mask			)[0].mean(axis=-1			)
					self.assertTrue(jnp.allclose(lowerCamelCase__   ,     lowerCamelCase__   ,     atol=4e-2			)			)



@require_flax
@slow
class  lowerCamelCase__				(						unittest.TestCase						):


	"""simple docstring"""





	@property
	def 				_lowerCamelCase				(					self			)       ->				int:
					return [
					    "Today is a beautiful day and I want",
					    "In the city of",
					    "Paris is the capital of France and",
					    "Computers and mobile phones have taken",
					]




	def 				_lowerCamelCase				(					self			)       ->				Union[str, Any]:
					_A :				List[Any]				=  '''facebook/opt-125m'''

					_A :				str				=  [
					    '''Today is a beautiful day and I want to''',
					    '''In the city of New York, the city''',
					    '''Paris is the capital of France and the capital''',
					    '''Computers and mobile phones have taken over the''',
					]

					_A :				List[str]				=  []

					_A :				List[Any]				=  FlaxOPTForCausalLM.from_pretrained(lowerCamelCase__			)
					_A :				int				=  GPTaTokenizer.from_pretrained(lowerCamelCase__			)

					for prompt in self.prompts:
									_A :				Dict				=  tokenizer(lowerCamelCase__   ,     return_tensors='''jax'''			).input_ids

									_A :				str				=  model.generate(lowerCamelCase__   ,     max_length=1_0			)
									_A :				List[Any]				=  generated_ids[0]

									_A :				List[str]				=  tokenizer.batch_decode(lowerCamelCase__   ,     skip_special_tokens=lowerCamelCase__			)
									predicted_outputs += generated_string

					self.assertListEqual(lowerCamelCase__   ,     lowerCamelCase__			)




	def 				_lowerCamelCase				(					self			)       ->				List[Any]:
					_A :				Optional[Any]				=  '''facebook/opt-350m'''

					_A :				Any				=  [
					    '''Today is a beautiful day and I want to''',
					    '''In the city of San Francisco, the city''',
					    '''Paris is the capital of France and the capital''',
					    '''Computers and mobile phones have taken over the''',
					]

					_A :				Optional[int]				=  []
					_A :				Tuple				=  FlaxOPTForCausalLM.from_pretrained(lowerCamelCase__			)
					_A :				Optional[int]				=  GPTaTokenizer.from_pretrained(lowerCamelCase__			)

					for prompt in self.prompts:
									_A :				Tuple				=  tokenizer(lowerCamelCase__   ,     return_tensors='''jax'''			).input_ids

									_A :				List[str]				=  model.generate(lowerCamelCase__   ,     max_length=1_0			)
									_A :				Any				=  generated_ids[0]

									_A :				Optional[int]				=  tokenizer.batch_decode(lowerCamelCase__   ,     skip_special_tokens=lowerCamelCase__			)
									predicted_outputs += generated_string

					self.assertListEqual(lowerCamelCase__   ,     lowerCamelCase__			)




	def 				_lowerCamelCase				(					self			)       ->				Any:
					_A :				Dict				=  '''facebook/opt-125m'''
					_A :				Dict				=  [
					    '''Today is a beautiful day and I want to thank''',
					    '''In the city of Rome Canaver Canaver Canaver Canaver''',
					]
					_A :				Tuple				=  FlaxOPTForCausalLM.from_pretrained(lowerCamelCase__			)
					_A :				Tuple				=  GPTaTokenizer.from_pretrained(lowerCamelCase__			)
					_A :				int				=  tokenizer(
					    [
					        '''Today is a beautiful day and I want to''',
					        '''In the city of''',
					    ]   ,     return_tensors='''jax'''   ,     padding=lowerCamelCase__   ,     )

					_A :				Optional[Any]				=  jax.jit(model.generate			)

					_A :				Dict				=  jit_generate(inputs['''input_ids''']   ,     attention_mask=inputs['''attention_mask''']			).sequences

					_A :				Optional[int]				=  tokenizer.batch_decode(lowerCamelCase__   ,     skip_special_tokens=lowerCamelCase__			)

					self.assertIsNotNone(lowerCamelCase__   ,     lowerCamelCase__			)




	def 				_lowerCamelCase				(					self			)       ->				Optional[Any]:
					_A :				Any				=  '''facebook/opt-350m'''

					_A :				str				=  GPTaTokenizer.from_pretrained(lowerCamelCase__			)
					_A :				Optional[Any]				=  FlaxOPTForCausalLM.from_pretrained(lowerCamelCase__			)

					_A :				str				=  '''left'''

					# use different length sentences to test batching
					_A :				List[Any]				=  [
					    '''Hello, my dog is a little''',
					    '''Today, I''',
					]

					_A :				Union[str, Any]				=  tokenizer(lowerCamelCase__   ,     return_tensors='''jax'''   ,     padding=lowerCamelCase__			)
					_A :				Tuple				=  inputs['''input_ids''']

					_A :				int				=  model.generate(input_ids=lowerCamelCase__   ,     attention_mask=inputs['''attention_mask''']   ,     trace=lowerCamelCase__			)

					_A :				Tuple				=  tokenizer(sentences[0]   ,     return_tensors='''jax'''			).input_ids
					_A :				str				=  model.generate(input_ids=lowerCamelCase__			)

					_A :				Tuple				=  inputs_non_padded.shape[-1] - inputs['''attention_mask'''][-1].sum()
					_A :				Any				=  tokenizer(sentences[1]   ,     return_tensors='''jax'''			).input_ids
					_A :				int				=  model.generate(input_ids=lowerCamelCase__   ,     max_length=model.config.max_length - num_paddings			)

					_A :				Optional[Any]				=  tokenizer.batch_decode(outputs[0]   ,     skip_special_tokens=lowerCamelCase__			)
					_A :				Any				=  tokenizer.decode(output_non_padded[0][0]   ,     skip_special_tokens=lowerCamelCase__			)
					_A :				Union[str, Any]				=  tokenizer.decode(output_padded[0][0]   ,     skip_special_tokens=lowerCamelCase__			)

					_A :				List[Any]				=  [
					    '''Hello, my dog is a little bit of a dork.\nI\'m a little bit''',
					    '''Today, I was in the middle of a conversation with a friend about the''',
					]
					self.assertListEqual(lowerCamelCase__   ,     lowerCamelCase__			)
					self.assertListEqual(lowerCamelCase__   ,     [non_padded_sentence, padded_sentence]			)







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 						UpperCamelCase__					( lowerCAmelCase_				, unittest.TestCase  ):



     '''simple docstring'''

     __snake_case       :      Dict						      =     PreTrainedTokenizerFast
     __snake_case       :      Optional[int]						      =     False
     __snake_case       :      List[Any]						      =     True
     __snake_case       :      str						      =     "tokenizer_file"






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Union[str, Any]			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		False  # because we don't have pretrained_vocab_files_map
           super().setUp()
           SCREAMING_SNAKE_CASE						=		True

           SCREAMING_SNAKE_CASE						=		["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
           SCREAMING_SNAKE_CASE						=		"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

           # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
           SCREAMING_SNAKE_CASE						=		[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

           SCREAMING_SNAKE_CASE						=		PreTrainedTokenizerFast.from_pretrained(model_paths[0]			)
           tokenizer.save_pretrained(self.tmpdirname			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						Tuple:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						List[Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						Union[str, Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						List[Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Any			)	->						Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tmpdirname
           # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                       try:
                             SCREAMING_SNAKE_CASE						=		tempfile.mkdtemp()
                             SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,**lowerCamelCase__			)

                             tokenizer.save_pretrained(self.tmpdirname			)
                             super().test_training_new_tokenizer()
                       finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname			)
                             SCREAMING_SNAKE_CASE						=		tmpdirname_orig






     def 							SCREAMING_SNAKE_CASE__    (				self		:       int			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tmpdirname
           # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                       try:
                             SCREAMING_SNAKE_CASE						=		tempfile.mkdtemp()
                             SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,**lowerCamelCase__			)

                             tokenizer.save_pretrained(self.tmpdirname			)
                             super().test_training_new_tokenizer_with_special_tokens_change()
                       finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname			)
                             SCREAMING_SNAKE_CASE						=		tmpdirname_orig






     def 							SCREAMING_SNAKE_CASE__    (				self		:       int			)	->						Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name			)

           SCREAMING_SNAKE_CASE						=		("""a""" for _ in range(1000			))
           SCREAMING_SNAKE_CASE						=		tokenizer.train_new_from_iterator(text_iterator=lowerCamelCase__		,length=1000		,vocab_size=50			)

           SCREAMING_SNAKE_CASE						=		new_tokenizer.encode("""a🤗"""			)
           self.assertEqual(lowerCamelCase__		,[64, 172, 253, 97, 245]			)

@require_tokenizers
class 						UpperCamelCase__					( unittest.TestCase  ):



     '''simple docstring'''

     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained("""bert-base-cased"""			)
           SCREAMING_SNAKE_CASE						=		json.loads(tokenizer._tokenizer.to_str()			)
           SCREAMING_SNAKE_CASE						=		len(lowerCamelCase__			)

           with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 SCREAMING_SNAKE_CASE						=		["""tokenizer.4.0.0.json"""]
                 tokenizer.save_pretrained(lowerCamelCase__			)
                 json.dump(lowerCamelCase__		,open(os.path.join(lowerCamelCase__		,"""tokenizer.4.0.0.json"""			)		,"""w"""			)			)

                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
                 self.assertEqual(len(lowerCamelCase__			)		,len(lowerCamelCase__			) + 1			)
                 SCREAMING_SNAKE_CASE						=		json.loads(new_tokenizer._tokenizer.to_str()			)
                 self.assertIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(lowerCamelCase__		,"""tokenizer.4.0.0.json"""			)		,os.path.join(lowerCamelCase__		,"""tokenizer.42.0.0.json"""			)			)
                 SCREAMING_SNAKE_CASE						=		["""tokenizer.42.0.0.json"""]
                 tokenizer.save_pretrained(lowerCamelCase__			)
                 SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
                 self.assertEqual(len(lowerCamelCase__			)		,len(lowerCamelCase__			)			)
                 SCREAMING_SNAKE_CASE						=		json.loads(new_tokenizer._tokenizer.to_str()			)
                 self.assertNotIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Dict			)	->						Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		"""hf-internal-testing/test-two-tokenizers"""

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
           self.assertEqual(len(lowerCamelCase__			)		,28997			)
           SCREAMING_SNAKE_CASE						=		json.loads(tokenizer._tokenizer.to_str()			)
           self.assertIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

           # Testing an older version by monkey-patching the version in the module it's used.
           import transformers as old_transformers

           SCREAMING_SNAKE_CASE						=		"""3.0.0"""
           SCREAMING_SNAKE_CASE						=		old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCamelCase__			)
           self.assertEqual(len(lowerCamelCase__			)		,28996			)
           SCREAMING_SNAKE_CASE						=		json.loads(old_tokenizer._tokenizer.to_str()			)
           self.assertNotIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

@require_tokenizers
class 						UpperCamelCase__					( unittest.TestCase  ):



     '''simple docstring'''

     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""			)
           SCREAMING_SNAKE_CASE						=		"""The Matrix is a 1999 science fiction action film."""

           with concurrent.futures.ThreadPoolExecutor() as executor:
                 SCREAMING_SNAKE_CASE						=		[executor.submit(self.fetch		,lowerCamelCase__		,lowerCamelCase__			) for i in range(10			)]
                 SCREAMING_SNAKE_CASE						=		[future.result() for future in futures]
                 self.assertEqual(lowerCamelCase__		,[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10			)]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Optional[int]		,lowerCamelCase__		:       Optional[int]		,lowerCamelCase__		:       str			)	->						Union[str, Any]:




           '''simple docstring'''


           return tokenizer.encode(lowerCamelCase__		,truncation="""longest_first"""		,padding="""longest"""			)


