






"""simple docstring"""



from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel



def 						lowercase    (				__snake_case				: Tuple		,					__snake_case				: str		,					__snake_case				: Union[str, Any]=None		,					__snake_case				: int=None	):


		if attention_mask is None:
				lowercase_ : List[str]							       =   tf.cast(tf.math.not_equal(lowerCAmelCase_		,					config.pad_token_id	)		,					tf.inta	)
		return {"input_ids": input_ids, "attention_mask": attention_mask}





@require_tf
class 					_UpperCAmelCase       :
					SCREAMING_SNAKE_CASE_		:				Dict							    =   OPTConfig
					SCREAMING_SNAKE_CASE_		:				List[Any]							    =   {}
					SCREAMING_SNAKE_CASE_		:				List[str]							    =   "gelu"




					def __init__(				self     :						Union[str, Any]    ,						A     :						int    ,						A     :						List[str]=13    ,						A     :						Optional[int]=7    ,						A     :						Tuple=True    ,						A     :						Optional[Any]=False    ,						A     :						str=99    ,						A     :						Dict=16    ,						A     :						Tuple=2    ,						A     :						int=4    ,						A     :						Tuple=4    ,						A     :						str="gelu"    ,						A     :						Optional[int]=0.1    ,						A     :						Any=0.1    ,						A     :						List[str]=20    ,						A     :						Any=2    ,						A     :						str=1    ,						A     :						Optional[Any]=0    ,						A     :						Optional[Any]=16    ,						A     :						List[str]=16    ,						)  ->  List[str]:
							lowercase_ : int							       =   parent
							lowercase_ : int							       =   batch_size
							lowercase_ : Optional[int]							       =   seq_length
							lowercase_ : Tuple							       =   is_training
							lowercase_ : List[Any]							       =   use_labels
							lowercase_ : Tuple							       =   vocab_size
							lowercase_ : int							       =   hidden_size
							lowercase_ : Optional[Any]							       =   num_hidden_layers
							lowercase_ : List[str]							       =   num_attention_heads
							lowercase_ : str							       =   intermediate_size
							lowercase_ : Any							       =   hidden_act
							lowercase_ : int							       =   hidden_dropout_prob
							lowercase_ : List[str]							       =   attention_probs_dropout_prob
							lowercase_ : List[str]							       =   max_position_embeddings
							lowercase_ : Union[str, Any]							       =   eos_token_id
							lowercase_ : Optional[int]							       =   pad_token_id
							lowercase_ : int							       =   bos_token_id
							lowercase_ : Tuple							       =   embed_dim
							lowercase_ : Dict							       =   word_embed_proj_dim
							lowercase_ : Union[str, Any]							       =   False




					def   A  (				self     :						Optional[Any]   )  ->  Dict:
							lowercase_ : str							       =   ids_tensor([self.batch_size, self.seq_length - 1]    ,						self.vocab_size   )
							lowercase_ : str							       =   tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size   )    ,						1   )
							lowercase_ : str							       =   tf.concat([input_ids, eos_tensor]    ,						axis=1   )

							lowercase_ : Optional[Any]							       =   self.config_cls(
							    vocab_size=self.vocab_size    ,						hidden_size=self.hidden_size    ,						num_hidden_layers=self.num_hidden_layers    ,						num_attention_heads=self.num_attention_heads    ,						ffn_dim=self.intermediate_size    ,						dropout=self.hidden_dropout_prob    ,						attention_dropout=self.attention_probs_dropout_prob    ,						max_position_embeddings=self.max_position_embeddings    ,						eos_token_id=self.eos_token_id    ,						bos_token_id=self.bos_token_id    ,						pad_token_id=self.pad_token_id    ,						embed_dim=self.embed_dim    ,						word_embed_proj_dim=self.word_embed_proj_dim    ,						is_encoder_decoder=UpperCAmelCase__    ,						**self.config_updates    ,						)
							lowercase_ : Union[str, Any]							       =   prepare_opt_inputs_dict(UpperCAmelCase__    ,						UpperCAmelCase__   )
							return config, inputs_dict






					def   A  (				self     :						Optional[Any]    ,						A     :						Union[str, Any]    ,						A     :						List[Any]   )  ->  str:
							lowercase_ : Tuple							       =   TFOPTModel(config=UpperCAmelCase__   )
							lowercase_ : Dict							       =   inputs_dict['''input_ids''']

							lowercase_ : Any							       =   input_ids[:1, :]
							lowercase_ : List[Any]							       =   inputs_dict['''attention_mask'''][:1, :]
							lowercase_ : Any							       =   1

							# first forward pass
							lowercase_ : Optional[int]							       =   model(UpperCAmelCase__    ,						attention_mask=UpperCAmelCase__    ,						use_cache=UpperCAmelCase__   )

							lowercase_       ,					lowercase_ : int							       =   outputs.to_tuple()

							# create hypothetical next token and extent to next_input_ids
							lowercase_ : Any							       =   ids_tensor((self.batch_size, 3)    ,						config.vocab_size   )
							lowercase_ : List[str]							       =   tf.cast(ids_tensor((self.batch_size, 3)    ,						2   )    ,						tf.inta   )

							# append to next input_ids and
							lowercase_ : Optional[int]							       =   tf.concat([input_ids, next_tokens]    ,						axis=-1   )
							lowercase_ : Optional[Any]							       =   tf.concat([attention_mask, next_attn_mask]    ,						axis=-1   )

							lowercase_ : Union[str, Any]							       =   model(UpperCAmelCase__    ,						attention_mask=UpperCAmelCase__   )[0]
							lowercase_ : str							       =   model(UpperCAmelCase__    ,						attention_mask=UpperCAmelCase__    ,						past_key_values=UpperCAmelCase__   )[0]

							self.parent.assertEqual(next_tokens.shape[1]    ,						output_from_past.shape[1]   )

							# select random slice
							lowercase_ : Tuple							       =   int(ids_tensor((1,)    ,						output_from_past.shape[-1]   )   )
							lowercase_ : Optional[int]							       =   output_from_no_past[:, -3:, random_slice_idx]
							lowercase_ : Tuple							       =   output_from_past[:, :, random_slice_idx]

							# test that outputs are equal for slice
							tf.debugging.assert_near(UpperCAmelCase__    ,						UpperCAmelCase__    ,						rtol=1e-3   )





@require_tf
class 					_UpperCAmelCase       ( _A   ,						_A   ,						unittest.TestCase							):
					SCREAMING_SNAKE_CASE_		:				str							    =   (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
					SCREAMING_SNAKE_CASE_		:				Optional[Any]							    =   (TFOPTForCausalLM,) if is_tf_available() else ()
					SCREAMING_SNAKE_CASE_		:				Tuple							    =   (
					    {"feature-extraction": TFOPTModel, "text-generation": TFOPTForCausalLM} if is_tf_available() else {}
					)
					SCREAMING_SNAKE_CASE_		:				Any							    =   False
					SCREAMING_SNAKE_CASE_		:				Union[str, Any]							    =   False
					SCREAMING_SNAKE_CASE_		:				List[str]							    =   False
					SCREAMING_SNAKE_CASE_		:				Tuple							    =   10




					def   A  (				self     :						Union[str, Any]   )  ->  Dict:
							lowercase_ : Any							       =   TFOPTModelTester(self   )
							lowercase_ : Any							       =   ConfigTester(self    ,						config_class=UpperCAmelCase__   )




					def   A  (				self     :						List[str]   )  ->  List[Any]:
							self.config_tester.run_common_tests()




					def   A  (				self     :						Optional[int]   )  ->  Union[str, Any]:
							lowercase_ : List[str]							       =   self.model_tester.prepare_config_and_inputs_for_common()
							self.model_tester.check_decoder_model_past_large_inputs(*UpperCAmelCase__   )






					def   A  (				self     :						Dict   )  ->  str:
							lowercase_       ,					lowercase_ : Dict							       =   self.model_tester.prepare_config_and_inputs_for_common()

							def _get_word_embedding_weight(A     :						Dict    ,						A     :						str   ):
									if hasattr(UpperCAmelCase__    ,						'''weight'''   ):
											return embedding_layer.weight
									else:
											# Here we build the word embeddings weights if not exists.
											# And then we retry to get the attribute once built.
											model.build()
											if hasattr(UpperCAmelCase__    ,						'''weight'''   ):
													return embedding_layer.weight
											else:
													return None

							for model_class in self.all_model_classes:
									for size in [config.vocab_size - 10, config.vocab_size + 10]:
											# build the embeddings
											lowercase_ : List[Any]							       =   model_class(config=UpperCAmelCase__   )
											lowercase_ : Tuple							       =   _get_word_embedding_weight(UpperCAmelCase__    ,						model.get_input_embeddings()   )
											lowercase_ : Union[str, Any]							       =   _get_word_embedding_weight(UpperCAmelCase__    ,						model.get_output_embeddings()   )

											# reshape the embeddings
											model.resize_token_embeddings(UpperCAmelCase__   )
											lowercase_ : Tuple							       =   _get_word_embedding_weight(UpperCAmelCase__    ,						model.get_input_embeddings()   )
											lowercase_ : Optional[int]							       =   _get_word_embedding_weight(UpperCAmelCase__    ,						model.get_output_embeddings()   )

											# check that the resized embeddings size matches the desired size.
											lowercase_ : List[str]							       =   size if size is not None else config.vocab_size

											self.assertEqual(new_input_embeddings.shape[0]    ,						UpperCAmelCase__   )

											# check that weights remain the same after resizing
											lowercase_ : List[str]							       =   True
											for pa, pa in zip(old_input_embeddings.value()    ,						new_input_embeddings.value()   ):
													if tf.math.reduce_sum(tf.math.abs(pa - pa   )   ) > 0:
															lowercase_ : Optional[int]							       =   False
											self.assertTrue(UpperCAmelCase__   )

											if old_output_embeddings is not None and new_output_embeddings is not None:
													self.assertEqual(new_output_embeddings.shape[0]    ,						UpperCAmelCase__   )

													lowercase_ : str							       =   True
													for pa, pa in zip(old_output_embeddings.value()    ,						new_output_embeddings.value()   ):
															if tf.math.reduce_sum(tf.math.abs(pa - pa   )   ) > 0:
																	lowercase_ : Union[str, Any]							       =   False
													self.assertTrue(UpperCAmelCase__   )



def 						lowercase    (				__snake_case				: List[str]	):
		return tf.constant(lowerCAmelCase_		,					dtype=tf.intaa	)





@require_tf
class 					_UpperCAmelCase       ( unittest.TestCase							):
					SCREAMING_SNAKE_CASE_		:				int							    =   99




					def   A  (				self     :						Optional[Any]   )  ->  Optional[int]:
							lowercase_ : List[Any]							       =   tf.ones((4, 1)    ,						dtype=tf.intaa   ) * 2
							lowercase_ : List[Any]							       =   tf.concat([ids_tensor((4, 6)    ,						self.vocab_size - 3   ) + 3, eos_column_vector]    ,						axis=1   )
							lowercase_ : Optional[int]							       =   input_ids.shape[0]
							lowercase_ : int							       =   OPTConfig(
							    vocab_size=self.vocab_size    ,						hidden_size=24    ,						num_hidden_layers=2    ,						num_attention_heads=2    ,						ffn_dim=32    ,						max_position_embeddings=48    ,						eos_token_id=2    ,						pad_token_id=1    ,						bos_token_id=0    ,						)
							return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class 					_UpperCAmelCase       ( unittest.TestCase							):




					@slow
					def   A  (				self     :						List[Any]   )  ->  List[Any]:
							lowercase_ : Optional[int]							       =   TFOPTModel.from_pretrained('''facebook/opt-350m'''   )
							lowercase_ : Optional[int]							       =   _long_tensor([[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]]   )
							lowercase_ : Optional[Any]							       =   tf.not_equal(UpperCAmelCase__    ,						model.config.pad_token_id   )
							with tf.GradientTape():
									lowercase_ : Dict							       =   model(input_ids=UpperCAmelCase__    ,						attention_mask=UpperCAmelCase__   ).last_hidden_state
							lowercase_ : Dict							       =   (1, 11, 5_12)
							self.assertEqual(output.shape    ,						UpperCAmelCase__   )
							lowercase_ : Tuple							       =   tf.constant(
							    [[-0.2873, -1.9218, -0.3033], [-1.2710, -0.1338, -0.1902], [0.4095, 0.1214, -1.3121]]   )
							self.assertTrue(np.allclose(output[:, :3, :3]    ,						UpperCAmelCase__    ,						atol=4e-3   )   )

							lowercase_ : int							       =   tf.function(UpperCAmelCase__    ,						jit_compile=UpperCAmelCase__   )
							lowercase_ : Dict							       =   xla_generate(UpperCAmelCase__    ,						UpperCAmelCase__   )[0]
							self.assertTrue(np.allclose(output[:, :3, :3]    ,						UpperCAmelCase__    ,						atol=4e-2   )   )





@require_tf
@slow
class 					_UpperCAmelCase       ( unittest.TestCase							):




					def   A  (				self     :						Optional[Any]   )  ->  Optional[Any]:
							super().setUp()
							lowercase_ : Union[str, Any]							       =   '''facebook/opt-350m'''






					def   A  (				self     :						Tuple   )  ->  Union[str, Any]:
							lowercase_ : List[Any]							       =   TFOPTForCausalLM.from_pretrained(self.path_model   )
							lowercase_ : str							       =   GPTaTokenizer.from_pretrained(self.path_model   )

							lowercase_ : Tuple							       =   [
							    '''Today is a beautiful day and I want to''',
							    '''In the city of''',
							    '''Paris is the capital of France and''',
							    '''Computers and mobile phones have taken''',
							]
							# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
							lowercase_ : str							       =   tokenizer(UpperCAmelCase__    ,						return_tensors='''tf'''    ,						padding=UpperCAmelCase__    ,						add_special_tokens=UpperCAmelCase__   )
							lowercase_ : Union[str, Any]							       =   tf.math.reduce_mean(model(inputs.input_ids    ,						attention_mask=inputs.attention_mask   )[0]    ,						axis=-1   )
							lowercase_ : Dict							       =   tf.constant(
							    [
							        [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
							        [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
							        [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
							        [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
							    ]   )
							self.assertTrue(np.allclose(UpperCAmelCase__    ,						UpperCAmelCase__    ,						atol=1e-4   )   )

							lowercase_ : Optional[Any]							       =   tf.function(UpperCAmelCase__    ,						jit_compile=UpperCAmelCase__   )
							lowercase_ : Optional[int]							       =   tf.math.reduce_mean(xla_generate(inputs.input_ids    ,						attention_mask=inputs.attention_mask   )[0]    ,						axis=-1   )
							self.assertTrue(np.allclose(UpperCAmelCase__    ,						UpperCAmelCase__    ,						atol=1e-4   )   )





@require_tf
@slow
class 					_UpperCAmelCase       ( unittest.TestCase							):




					@property
					def   A  (				self     :						Dict   )  ->  str:
							return [
							    "Today is a beautiful day and I want",
							    "In the city of",
							    "Paris is the capital of France and",
							    "Computers and mobile phones have taken",
							]




					def   A  (				self     :						Union[str, Any]   )  ->  Any:
							lowercase_ : Optional[Any]							       =   '''facebook/opt-125m'''

							lowercase_ : int							       =   [
							    '''Today is a beautiful day and I want to''',
							    '''In the city of New York, the city''',
							    '''Paris is the capital of France and the capital''',
							    '''Computers and mobile phones have taken over the''',
							]

							lowercase_ : Optional[Any]							       =   []
							lowercase_ : Optional[int]							       =   GPTaTokenizer.from_pretrained(UpperCAmelCase__   )
							lowercase_ : List[str]							       =   TFOPTForCausalLM.from_pretrained(UpperCAmelCase__   )

							for prompt in self.prompts:
									lowercase_ : List[str]							       =   tokenizer(UpperCAmelCase__    ,						return_tensors='''tf'''   ).input_ids

									lowercase_ : Tuple							       =   model.generate(UpperCAmelCase__    ,						max_length=10   )

									lowercase_ : List[Any]							       =   tokenizer.batch_decode(UpperCAmelCase__    ,						skip_special_tokens=UpperCAmelCase__   )
									predicted_outputs += generated_string

							self.assertListEqual(UpperCAmelCase__    ,						UpperCAmelCase__   )




					def   A  (				self     :						int   )  ->  int:
							lowercase_ : Any							       =   '''facebook/opt-350m'''

							lowercase_ : Optional[int]							       =   GPTaTokenizer.from_pretrained(UpperCAmelCase__   )
							lowercase_ : List[Any]							       =   TFOPTForCausalLM.from_pretrained(UpperCAmelCase__   )

							lowercase_ : List[str]							       =   '''left'''

							# use different length sentences to test batching
							lowercase_ : List[str]							       =   [
							    '''Hello, my dog is a little''',
							    '''Today, I''',
							]

							lowercase_ : Optional[int]							       =   tokenizer(UpperCAmelCase__    ,						return_tensors='''tf'''    ,						padding=UpperCAmelCase__   )
							lowercase_ : Union[str, Any]							       =   inputs['''input_ids''']

							lowercase_ : str							       =   model.generate(input_ids=UpperCAmelCase__    ,						attention_mask=inputs['''attention_mask''']   )

							lowercase_ : str							       =   tokenizer(sentences[0]    ,						return_tensors='''tf'''   ).input_ids
							lowercase_ : Optional[int]							       =   model.generate(input_ids=UpperCAmelCase__   )

							lowercase_ : List[Any]							       =   inputs_non_padded.shape[-1] - tf.math.reduce_sum(
							    tf.cast(inputs['''attention_mask'''][-1]    ,						tf.intaa   )   )
							lowercase_ : Any							       =   tokenizer(sentences[1]    ,						return_tensors='''tf'''   ).input_ids
							lowercase_ : Dict							       =   model.generate(input_ids=UpperCAmelCase__    ,						max_length=model.config.max_length - num_paddings   )

							lowercase_ : List[Any]							       =   tokenizer.batch_decode(UpperCAmelCase__    ,						skip_special_tokens=UpperCAmelCase__   )
							lowercase_ : Union[str, Any]							       =   tokenizer.decode(output_non_padded[0]    ,						skip_special_tokens=UpperCAmelCase__   )
							lowercase_ : Union[str, Any]							       =   tokenizer.decode(output_padded[0]    ,						skip_special_tokens=UpperCAmelCase__   )

							lowercase_ : Dict							       =   [
							    '''Hello, my dog is a little bit of a dork.\nI\'m a little bit''',
							    '''Today, I was in the middle of a conversation with a friend about the''',
							]
							self.assertListEqual(UpperCAmelCase__    ,						UpperCAmelCase__   )
							self.assertListEqual(UpperCAmelCase__    ,						[non_padded_sentence, padded_sentence]   )






					def   A  (				self     :						Any   )  ->  Any:
							lowercase_ : List[str]							       =   '''facebook/opt-350m'''

							lowercase_ : List[Any]							       =   [
							    '''Today is a beautiful day and I want to''',
							    '''In the city of San Francisco, the city''',
							    '''Paris is the capital of France and the capital''',
							    '''Computers and mobile phones have taken over the''',
							]

							lowercase_ : Optional[int]							       =   []
							lowercase_ : Any							       =   GPTaTokenizer.from_pretrained(UpperCAmelCase__   )
							lowercase_ : str							       =   TFOPTForCausalLM.from_pretrained(UpperCAmelCase__   )

							for prompt in self.prompts:
									lowercase_ : int							       =   tokenizer(UpperCAmelCase__    ,						return_tensors='''tf'''   ).input_ids

									lowercase_ : Any							       =   model.generate(UpperCAmelCase__    ,						max_length=10   )

									lowercase_ : Union[str, Any]							       =   tokenizer.batch_decode(UpperCAmelCase__    ,						skip_special_tokens=UpperCAmelCase__   )
									predicted_outputs += generated_string

							self.assertListEqual(UpperCAmelCase__    ,						UpperCAmelCase__   )







"""simple docstring"""







import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  UpperCamelCase_	( UpperCamelCase			,  unittest.TestCase):

	"""simple docstring"""





	snake_case__				:  Any    					=			CLIPTokenizer
	snake_case__				:  Dict    					=			CLIPTokenizerFast
	snake_case__				:  List[Any]    					=			True
	snake_case__				:  Optional[Any]    					=			{}
	snake_case__				:  Dict    					=			False




	def   UpperCAmelCase_      ( self			:				Any   )       ->	Any:
						super().setUp()

						# fmt: off
						__SCREAMING_SNAKE_CASE						   =				["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
						# fmt: on
						__SCREAMING_SNAKE_CASE						   =				dict(zip(UpperCAmelCase__			,			range(len(UpperCAmelCase__   )   )   )   )
						__SCREAMING_SNAKE_CASE						   =				["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
						__SCREAMING_SNAKE_CASE						   =				{"unk_token": "<unk>"}

						__SCREAMING_SNAKE_CASE						   =				os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES["vocab_file"]   )
						__SCREAMING_SNAKE_CASE						   =				os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES["merges_file"]   )
						with open(self.vocab_file			,			"w"			,			encoding="utf-8"   ) as fp:
											fp.write(json.dumps(UpperCAmelCase__   ) + "\n"   )
						with open(self.merges_file			,			"w"			,			encoding="utf-8"   ) as fp:
											fp.write("\n".join(UpperCAmelCase__   )   )




	def   UpperCAmelCase_      ( self			:				List[Any]			,			**UpperCAmelCase__			:				Tuple   )       ->	List[Any]:
						kwargs.update(self.special_tokens_map   )
						return CLIPTokenizer.from_pretrained(self.tmpdirname			,			**UpperCAmelCase__   )




	def   UpperCAmelCase_      ( self			:				Any			,			**UpperCAmelCase__			:				Optional[Any]   )       ->	List[str]:
						kwargs.update(self.special_tokens_map   )
						return CLIPTokenizerFast.from_pretrained(self.tmpdirname			,			**UpperCAmelCase__   )




	def   UpperCAmelCase_      ( self			:				List[Any]			,			UpperCAmelCase__			:				int   )       ->	Union[str, Any]:
						__SCREAMING_SNAKE_CASE						   =				"lower newer"
						__SCREAMING_SNAKE_CASE						   =				"lower newer"
						return input_text, output_text




	def   UpperCAmelCase_      ( self			:				int   )       ->	List[str]:
						__SCREAMING_SNAKE_CASE						   =				CLIPTokenizer(self.vocab_file			,			self.merges_file			,			**self.special_tokens_map   )
						__SCREAMING_SNAKE_CASE						   =				"lower newer"
						__SCREAMING_SNAKE_CASE						   =				["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
						__SCREAMING_SNAKE_CASE						   =				tokenizer.tokenize(UpperCAmelCase__   )
						self.assertListEqual(UpperCAmelCase__			,			UpperCAmelCase__   )

						__SCREAMING_SNAKE_CASE						   =				tokens + [tokenizer.unk_token]
						__SCREAMING_SNAKE_CASE						   =				[1_0, 2, 1_6, 9, 3, 2, 1_6, 2_0]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__   )			,			UpperCAmelCase__   )




	@require_ftfy
	def   UpperCAmelCase_      ( self			:				Optional[Any]   )       ->	int:
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
																__SCREAMING_SNAKE_CASE						   =				self.tokenizer_class.from_pretrained(UpperCAmelCase__			,			**UpperCAmelCase__   )
																__SCREAMING_SNAKE_CASE						   =				self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__			,			**UpperCAmelCase__   )

																__SCREAMING_SNAKE_CASE						   =				"A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
																__SCREAMING_SNAKE_CASE						   =				tokenizer_s.tokenize(UpperCAmelCase__   )
																__SCREAMING_SNAKE_CASE						   =				tokenizer_r.tokenize(UpperCAmelCase__   )

																self.assertListEqual(UpperCAmelCase__			,			UpperCAmelCase__   )

																# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																# with Tilde) encoded in 2 different ways
																__SCREAMING_SNAKE_CASE						   =				"xa\u0303y" + " " + "x\xe3y"
																__SCREAMING_SNAKE_CASE						   =				tokenizer_s.tokenize(UpperCAmelCase__   )
																__SCREAMING_SNAKE_CASE						   =				tokenizer_r.tokenize(UpperCAmelCase__   )

																self.assertListEqual(UpperCAmelCase__			,			UpperCAmelCase__   )

																# Test that the tokenization is identical on unicode of space type
																__SCREAMING_SNAKE_CASE						   =				[
																    "\u0009",  # (horizontal tab, '\t')
																    "\u000B",  # (vertical tab)
																    "\u000C",  # (form feed)
																    "\u0020",  # (space, ' ')
																    "\u200E",  # (left-to-right mark):w
																    "\u200F",  # (right-to-left mark)
																]
																for unicode_seq in spaces_unicodes:
																					__SCREAMING_SNAKE_CASE						   =				tokenizer_s.tokenize(UpperCAmelCase__   )
																					__SCREAMING_SNAKE_CASE						   =				tokenizer_r.tokenize(UpperCAmelCase__   )

																					self.assertListEqual(UpperCAmelCase__			,			UpperCAmelCase__   )

																# Test that the tokenization is identical on unicode of line break type
																__SCREAMING_SNAKE_CASE						   =				[
																    "\u000A",  # (line feed, '\n')
																    "\r\n",  # (carriage return and line feed, '\r\n')
																    "\u000D",  # (carriage return, '\r')
																    "\r",  # (carriage return, '\r')
																    "\u000D",  # (carriage return, '\r')
																    "\u2028",  # (line separator)
																    "\u2029",  # (paragraph separator)
																    # "\u0085", # (next line)
																]

																# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																# space (and thus into an empty list).

																for unicode_seq in line_break_unicodes:
																					__SCREAMING_SNAKE_CASE						   =				tokenizer_s.tokenize(UpperCAmelCase__   )
																					__SCREAMING_SNAKE_CASE						   =				tokenizer_r.tokenize(UpperCAmelCase__   )

																					self.assertListEqual(UpperCAmelCase__			,			UpperCAmelCase__   )




	def   UpperCAmelCase_      ( self			:				Tuple   )       ->	Optional[Any]:
						# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
																__SCREAMING_SNAKE_CASE						   =				"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
																__SCREAMING_SNAKE_CASE						   =				F"""{text_of_1_token} {text_of_1_token}"""

																__SCREAMING_SNAKE_CASE						   =				self.rust_tokenizer_class.from_pretrained(
																    UpperCAmelCase__			,			use_fast=UpperCAmelCase__			,			)
																__SCREAMING_SNAKE_CASE						   =				tokenizer_r(UpperCAmelCase__			,			return_offsets_mapping=UpperCAmelCase__			,			add_special_tokens=UpperCAmelCase__   )
																self.assertEqual(encoding.offset_mapping[0]			,			(0, len(UpperCAmelCase__   ))   )
																self.assertEqual(
																    encoding.offset_mapping[1]			,			(len(UpperCAmelCase__   ) + 1, len(UpperCAmelCase__   ) + 1 + len(UpperCAmelCase__   ))			,			)

																__SCREAMING_SNAKE_CASE						   =				F""" {text}"""

																__SCREAMING_SNAKE_CASE						   =				self.rust_tokenizer_class.from_pretrained(
																    UpperCAmelCase__			,			use_fast=UpperCAmelCase__			,			)
																__SCREAMING_SNAKE_CASE						   =				tokenizer_r(UpperCAmelCase__			,			return_offsets_mapping=UpperCAmelCase__			,			add_special_tokens=UpperCAmelCase__   )
																self.assertEqual(encoding.offset_mapping[0]			,			(1, 1 + len(UpperCAmelCase__   ))   )
																self.assertEqual(
																    encoding.offset_mapping[1]			,			(1 + len(UpperCAmelCase__   ) + 1, 1 + len(UpperCAmelCase__   ) + 1 + len(UpperCAmelCase__   ))			,			)




	def   UpperCAmelCase_      ( self			:				str   )       ->	Optional[int]:
						# Test related to the breaking change introduced in transformers v4.17.0
						# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
						with self.assertRaises(UpperCAmelCase__   ) as context:
											self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"   )

						self.assertTrue(
						    context.exception.args[0].startswith(
						        "The `backend_tokenizer` provided does not match the expected format."   )   )




	@require_ftfy
	def   UpperCAmelCase_      ( self			:				Optional[int]   )       ->	int:
						super().test_tokenization_python_rust_equals()




	def   UpperCAmelCase_      ( self			:				Optional[int]   )       ->	Optional[Any]:
						# CLIP always lower cases letters
						pass
