






import warnings
from typing import List, Optional, Union

from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType

class     _SCREAMING_SNAKE_CASE       (			_a      ):
	snake_case__							:     Union[str, Any]     	=  ["""image_processor""", """tokenizer"""]
	snake_case__							:     Optional[Any]     	=  """FlavaImageProcessor"""
	snake_case__							:     List[str]     	=  ("""BertTokenizer""", """BertTokenizerFast""")





	def __init__(		self				:					List[Any] ,  __lowerCamelCase				:					List[str]=None ,  __lowerCamelCase				:					Dict=None ,  **__lowerCamelCase				:					Tuple	):
		UpperCamelCase     :Union[str, Any]          =	None
		if "feature_extractor" in kwargs:
			warnings.warn(
			    """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
			    """ instead.""" ,  __lowerCamelCase ,  )
			UpperCamelCase     :List[Any]          =	kwargs.pop("""feature_extractor"""	)

		UpperCamelCase     :int          =	image_processor if image_processor is not None else feature_extractor
		if image_processor is None:
			raise ValueError("""You need to specify an `image_processor`."""	)
		if tokenizer is None:
			raise ValueError("""You need to specify a `tokenizer`."""	)

		super().__init__(__lowerCamelCase ,  __lowerCamelCase	)
		UpperCamelCase     :Optional[int]          =	self.image_processor





	def __call__(		self				:					Optional[Any] ,  __lowerCamelCase				:					Optional[ImageInput] = None ,  __lowerCamelCase				:					Optional[Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]] = None ,  __lowerCamelCase				:					bool = True ,  __lowerCamelCase				:					Union[bool, str, PaddingStrategy] = False ,  __lowerCamelCase				:					Union[bool, str, TruncationStrategy] = False ,  __lowerCamelCase				:					Optional[int] = None ,  __lowerCamelCase				:					int = 0 ,  __lowerCamelCase				:					Optional[int] = None ,  __lowerCamelCase				:					Optional[bool] = None ,  __lowerCamelCase				:					Optional[bool] = None ,  __lowerCamelCase				:					Optional[bool] = None ,  __lowerCamelCase				:					Optional[bool] = None ,  __lowerCamelCase				:					bool = False ,  __lowerCamelCase				:					bool = False ,  __lowerCamelCase				:					bool = False ,  __lowerCamelCase				:					bool = False ,  __lowerCamelCase				:					bool = True ,  __lowerCamelCase				:					Optional[Union[str, TensorType]] = None ,  **__lowerCamelCase				:					int ,  ):

		if text is None and images is None:
			raise ValueError("""You have to specify either text or images. Both cannot be none."""	)

		if text is not None:
			UpperCamelCase     :List[str]          =	self.tokenizer(
			    text=__lowerCamelCase ,  add_special_tokens=__lowerCamelCase ,  padding=__lowerCamelCase ,  truncation=__lowerCamelCase ,  max_length=__lowerCamelCase ,  stride=__lowerCamelCase ,  pad_to_multiple_of=__lowerCamelCase ,  return_token_type_ids=__lowerCamelCase ,  return_attention_mask=__lowerCamelCase ,  return_overflowing_tokens=__lowerCamelCase ,  return_special_tokens_mask=__lowerCamelCase ,  return_offsets_mapping=__lowerCamelCase ,  return_length=__lowerCamelCase ,  verbose=__lowerCamelCase ,  return_tensors=__lowerCamelCase ,  **__lowerCamelCase ,  )
		if images is not None:
			UpperCamelCase     :Dict          =	self.image_processor(
			    __lowerCamelCase ,  return_image_mask=__lowerCamelCase ,  return_codebook_pixels=__lowerCamelCase ,  return_tensors=__lowerCamelCase ,  **__lowerCamelCase ,  )

		if text is not None and images is not None:
			encoding.update(__lowerCamelCase	)
			return encoding
		elif text is not None:
			return encoding
		else:
			return BatchEncoding(data=dict(**__lowerCamelCase	) ,  tensor_type=__lowerCamelCase	)





	def 						_A (		self				:					Any ,  *__lowerCamelCase				:					int ,  **__lowerCamelCase				:					Optional[int]	):
		return self.tokenizer.batch_decode(*__lowerCamelCase ,  **__lowerCamelCase	)





	def 						_A (		self				:					Union[str, Any] ,  *__lowerCamelCase				:					Dict ,  **__lowerCamelCase				:					List[str]	):
		return self.tokenizer.decode(*__lowerCamelCase ,  **__lowerCamelCase	)





	@property
	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :Optional[Any]          =	self.tokenizer.model_input_names
		UpperCamelCase     :Any          =	self.image_processor.model_input_names
		return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names	)	)





	@property
	def 						_A (		self				:					Optional[Any]	):
		warnings.warn(
		    """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.""" ,  __lowerCamelCase ,  )
		return self.image_processor_class





	@property
	def 						_A (		self				:					List[Any]	):
		warnings.warn(
		    """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.""" ,  __lowerCamelCase ,  )
		return self.image_processor








import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
	import tensorflow as tf

if is_tensorflow_text_available():
	from transformers.models.bert import TFBertTokenizer


UpperCAmelCase_ :						Optional[Any]      =      ['''bert-base-uncased''', '''bert-base-cased''']
UpperCAmelCase_ :						List[str]      =      '''hf-internal-testing/tiny-bert-tf-only'''

if is_tf_available():

	class     _SCREAMING_SNAKE_CASE       (			tf.keras.Model      ):





		def __init__(		self				:					List[str] ,  __lowerCamelCase				:					Union[str, Any]	):
			super().__init__()
			UpperCamelCase     :Any          =	tokenizer
			UpperCamelCase     :List[str]          =	AutoConfig.from_pretrained(__lowerCamelCase	)
			UpperCamelCase     :List[str]          =	TFAutoModel.from_config(__lowerCamelCase	)





		def 						_A (		self				:					Tuple ,  __lowerCamelCase				:					str	):
			UpperCamelCase     :str          =	self.tokenizer(__lowerCamelCase	)
			UpperCamelCase     :Any          =	self.bert(**__lowerCamelCase	)
			return out["pooler_output"]

@require_tf
@require_tensorflow_text
class     _SCREAMING_SNAKE_CASE       (			unittest.TestCase      ):





	def 						_A (		self				:					Dict	):
		super().setUp()

		UpperCamelCase     :int          =	[
		    BertTokenizer.from_pretrained(__lowerCamelCase	) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
		]  # repeat for when fast_bert_tokenizer=false
		UpperCamelCase     :Any          =	[TFBertTokenizer.from_pretrained(__lowerCamelCase	) for checkpoint in TOKENIZER_CHECKPOINTS] + [
		    TFBertTokenizer.from_pretrained(__lowerCamelCase ,  use_fast_bert_tokenizer=__lowerCamelCase	)
		    for checkpoint in TOKENIZER_CHECKPOINTS
		]
		assert len(self.tokenizers	) == len(self.tf_tokenizers	)

		UpperCamelCase     :Any          =	[
		    """This is a straightforward English test sentence.""",
		    """This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.""",
		    """Now we're going to add some Chinese: 一 二 三 一二三""",
		    """And some much more rare Chinese: 齉 堃 齉堃""",
		    """Je vais aussi écrire en français pour tester les accents""",
		    """Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ""",
		]
		UpperCamelCase     :Union[str, Any]          =	list(zip(self.test_sentences ,  self.test_sentences[::-1]	)	)





	def 						_A (		self				:					Optional[int]	):
		for tokenizer, tf_tokenizer in zip(self.tokenizers ,  self.tf_tokenizers	):
			for test_inputs in (self.test_sentences, self.paired_sentences):
				UpperCamelCase     :Any          =	tokenizer(__lowerCamelCase ,  return_tensors="""tf""" ,  padding="""longest"""	)
				UpperCamelCase     :str          =	tf_tokenizer(__lowerCamelCase	)

				for key in python_outputs.keys():
					self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape	)	)
					self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key] ,  tf.intaa	) == tf_outputs[key]	)	)





	@slow
	def 						_A (		self				:					Dict	):
		for tf_tokenizer in self.tf_tokenizers:
			UpperCamelCase     :str          =	tf_tokenizer(self.paired_sentences	)
			UpperCamelCase     :Any          =	tf_tokenizer(
			    text=[sentence[0] for sentence in self.paired_sentences] ,  text_pair=[sentence[1] for sentence in self.paired_sentences] ,  )
			for key in merged_outputs.keys():
				self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key] ,  tf.intaa	) == separated_outputs[key]	)	)





	@slow
	def 						_A (		self				:					List[str]	):
		for tf_tokenizer in self.tf_tokenizers:
			UpperCamelCase     :List[Any]          =	tf.function(__lowerCamelCase	)
			for test_inputs in (self.test_sentences, self.paired_sentences):
				UpperCamelCase     :Any          =	tf.constant(__lowerCamelCase	)
				UpperCamelCase     :List[str]          =	compiled_tokenizer(__lowerCamelCase	)
				UpperCamelCase     :Optional[Any]          =	tf_tokenizer(__lowerCamelCase	)

				for key in eager_outputs.keys():
					self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]	)	)





	@slow
	def 						_A (		self				:					Tuple	):
		for tf_tokenizer in self.tf_tokenizers:
			UpperCamelCase     :List[str]          =	ModelToSave(tokenizer=__lowerCamelCase	)
			UpperCamelCase     :Union[str, Any]          =	tf.convert_to_tensor(self.test_sentences	)
			UpperCamelCase     :Union[str, Any]          =	model(__lowerCamelCase	)  # Build model with some sample inputs
			with TemporaryDirectory() as tempdir:
				UpperCamelCase     :List[str]          =	Path(__lowerCamelCase	) / """saved.model"""
				model.save(__lowerCamelCase	)
				UpperCamelCase     :List[Any]          =	tf.keras.models.load_model(__lowerCamelCase	)
			UpperCamelCase     :Dict          =	loaded_model(__lowerCamelCase	)
			# We may see small differences because the loaded model is compiled, so we need an epsilon for the test
			self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output	)	) ,  1E-5	)
