
'''simple docstring'''







import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer






class 					lowerCAmelCase__  (    tf.keras.layers.Layer	):






					def __init__(  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE = None							,				__SCREAMING_SNAKE_CASE = None							,				__SCREAMING_SNAKE_CASE = None							,				__SCREAMING_SNAKE_CASE = "longest"							,				__SCREAMING_SNAKE_CASE = True							,				__SCREAMING_SNAKE_CASE = 5_12							,				__SCREAMING_SNAKE_CASE = None							,				__SCREAMING_SNAKE_CASE = True							,				__SCREAMING_SNAKE_CASE = True							,				__SCREAMING_SNAKE_CASE = True							,				**__SCREAMING_SNAKE_CASE							,				):


											"""simple docstring"""





											super().__init__()
											if use_fast_bert_tokenizer:
																	lowercase_      :	str						=			FastBertTokenizer(
																	    __snake_case							,				token_out_type=tf.intaa							,				lower_case_nfd_strip_accents=__snake_case							,				**__snake_case						)
											else:
																	lowercase_      :	Optional[Any]						=			tf.lookup.StaticVocabularyTable(
																	    tf.lookup.KeyValueTensorInitializer(
																	        keys=__snake_case							,				key_dtype=tf.string							,				values=tf.range(tf.size(__snake_case							,				out_type=tf.intaa						)							,				dtype=tf.intaa						)							,				value_dtype=tf.intaa							,				)							,				num_oov_buckets=1							,				)
																	lowercase_      :	List[Any]						=			BertTokenizerLayer(
																	    __snake_case							,				token_out_type=tf.intaa							,				lower_case=__snake_case							,				**__snake_case						)

											lowercase_      :	Any						=			vocab_list
											lowercase_      :	Tuple						=			do_lower_case
											lowercase_      :	Tuple						=			cls_token_id or vocab_list.index('''[CLS]'''						)
											lowercase_      :	Tuple						=			sep_token_id or vocab_list.index('''[SEP]'''						)
											lowercase_      :	Optional[Any]						=			pad_token_id or vocab_list.index('''[PAD]'''						)
											lowercase_      :	str						=			ShrinkLongestTrimmer(max_length - 3							,				axis=1						)  # Allow room for special tokens
											lowercase_      :	Tuple						=			max_length
											lowercase_      :	List[str]						=			padding
											lowercase_      :	Any						=			truncation
											lowercase_      :	Optional[int]						=			pad_to_multiple_of
											lowercase_      :	int						=			return_token_type_ids
											lowercase_      :	Any						=			return_attention_mask






					@classmethod
					def       _snake_case       (  cls							,				__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):  # noqa: F821


											"""simple docstring"""





											lowercase_      :	List[str]						=			kwargs.pop('''do_lower_case'''							,				__snake_case						)
											lowercase_      :	List[Any]						=			tokenizer.do_lower_case if do_lower_case is None else do_lower_case
											lowercase_      :	Dict						=			kwargs.pop('''cls_token_id'''							,				__snake_case						)
											lowercase_      :	List[str]						=			tokenizer.cls_token_id if cls_token_id is None else cls_token_id
											lowercase_      :	List[str]						=			kwargs.pop('''sep_token_id'''							,				__snake_case						)
											lowercase_      :	List[Any]						=			tokenizer.sep_token_id if sep_token_id is None else sep_token_id
											lowercase_      :	List[str]						=			kwargs.pop('''pad_token_id'''							,				__snake_case						)
											lowercase_      :	Any						=			tokenizer.pad_token_id if pad_token_id is None else pad_token_id

											lowercase_      :	List[str]						=			tokenizer.get_vocab()
											lowercase_      :	Tuple						=			sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]							,				key=lambda __SCREAMING_SNAKE_CASE						: x[1]						)
											lowercase_      :	List[str]						=			[entry[0] for entry in vocab]
											return cls(
											    vocab_list=__snake_case							,				do_lower_case=__snake_case							,				cls_token_id=__snake_case							,				sep_token_id=__snake_case							,				pad_token_id=__snake_case							,				**__snake_case							,				)






					@classmethod
					def       _snake_case       (  cls							,				__SCREAMING_SNAKE_CASE							,				*__SCREAMING_SNAKE_CASE							,				**__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											try:
																	lowercase_      :	int						=			BertTokenizer.from_pretrained(__snake_case							,				*__snake_case							,				**__snake_case						)
											except:  # noqa: E722
																	from .tokenization_bert_fast import BertTokenizerFast

																	lowercase_      :	Optional[Any]						=			BertTokenizerFast.from_pretrained(__snake_case							,				*__snake_case							,				**__snake_case						)
											return cls.from_tokenizer(__snake_case							,				**__snake_case						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											if self.do_lower_case:
																	lowercase_      :	Optional[Any]						=			case_fold_utfa(__snake_case						)
											lowercase_      :	Optional[int]						=			self.tf_tokenizer.tokenize(__snake_case						)
											return tokens.merge_dims(1							,				-1						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE=None							,				__SCREAMING_SNAKE_CASE=None							,				__SCREAMING_SNAKE_CASE=None							,				__SCREAMING_SNAKE_CASE=None							,				__SCREAMING_SNAKE_CASE=None							,				__SCREAMING_SNAKE_CASE=None							,				__SCREAMING_SNAKE_CASE=None							,				):


											"""simple docstring"""





											if padding is None:
																	lowercase_      :	Optional[Any]						=			self.padding
											if padding not in ("longest", "max_length"):
																	raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''						)
											if max_length is not None and text_pair is not None:
																	# Because we have to instantiate a Trimmer to do it properly
																	raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''						)
											if max_length is None:
																	lowercase_      :	str						=			self.max_length
											if truncation is None:
																	lowercase_      :	Any						=			self.truncation
											if pad_to_multiple_of is None:
																	lowercase_      :	List[str]						=			self.pad_to_multiple_of
											if return_token_type_ids is None:
																	lowercase_      :	Dict						=			self.return_token_type_ids
											if return_attention_mask is None:
																	lowercase_      :	Dict						=			self.return_attention_mask
											if not isinstance(__snake_case							,				tf.Tensor						):
																	lowercase_      :	Union[str, Any]						=			tf.convert_to_tensor(__snake_case						)
											if text_pair is not None and not isinstance(__snake_case							,				tf.Tensor						):
																	lowercase_      :	Optional[Any]						=			tf.convert_to_tensor(__snake_case						)
											if text_pair is not None:
																	if text.shape.rank > 1:
																							raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''						)
																	if text_pair.shape.rank > 1:
																							raise ValueError('''text_pair should not be multidimensional!'''						)
											if text.shape.rank == 2:
																	lowercase_      :	Tuple						=			text[:, 0], text[:, 1]
											lowercase_      :	str						=			self.unpaired_tokenize(__snake_case						)
											if text_pair is None:  # Unpaired text
																	if truncation:
																							lowercase_      :	Any						=			text[:, : max_length - 2]  # Allow room for special tokens
																	lowercase_      :	List[Any]						=			combine_segments(
																	    (text,)							,				start_of_sequence_id=self.cls_token_id							,				end_of_segment_id=self.sep_token_id						)
											else:  # Paired text
																	lowercase_      :	List[str]						=			self.unpaired_tokenize(__snake_case						)
																	if truncation:
																							lowercase_      :	Optional[int]						=			self.paired_trimmer.trim([text, text_pair]						)
																	lowercase_      :	Optional[int]						=			combine_segments(
																	    (text, text_pair)							,				start_of_sequence_id=self.cls_token_id							,				end_of_segment_id=self.sep_token_id						)
											if padding == "longest":
																	lowercase_      :	Union[str, Any]						=			input_ids.bounding_shape(axis=1						)
																	if pad_to_multiple_of is not None:
																							# No ceiling division in tensorflow, so we negate floordiv instead
																							lowercase_      :	Union[str, Any]						=			pad_to_multiple_of * (-tf.math.floordiv(-pad_length							,				__snake_case						))
											else:
																	lowercase_      :	Any						=			max_length

											lowercase_      :	str						=			pad_model_inputs(__snake_case							,				max_seq_length=__snake_case							,				pad_value=self.pad_token_id						)
											lowercase_      :	Any						=			{"input_ids": input_ids}
											if return_attention_mask:
																	lowercase_      :	List[Any]						=			attention_mask
											if return_token_type_ids:
																	lowercase_      :	int						=			pad_model_inputs(
																	    __snake_case							,				max_seq_length=__snake_case							,				pad_value=self.pad_token_id						)
																	lowercase_      :	str						=			token_type_ids
											return output






					def       _snake_case       (  self						):


											"""simple docstring"""





											return {
											    "vocab_list": self.vocab_list,
											    "do_lower_case": self.do_lower_case,
											    "cls_token_id": self.cls_token_id,
											    "sep_token_id": self.sep_token_id,
											    "pad_token_id": self.pad_token_id,
											}





from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding





class 		__lowerCamelCase     (   lowercase		):
							lowerCamelCase__:				Any      =							['''image_processor''', '''tokenizer''']
							lowerCamelCase__:				List[Any]      =							'''EfficientNetImageProcessor'''
							lowerCamelCase__:				str      =							('''BertTokenizer''', '''BertTokenizerFast''')



							def __init__(     self	,				__snake_case	,				__snake_case							)						->     Optional[Any]:






								"""simple docstring"""



								super().__init__(__snake_case	,				__snake_case							)



							def __call__(     self	,				__snake_case=None	,				__snake_case=None	,				__snake_case="max_length"	,				__snake_case=6_4	,				__snake_case=None	,				**__snake_case							)						->     Any:






								"""simple docstring"""



								if text is None and images is None:
									raise ValueError("You have to specify either text or images. Both cannot be none."							)

								if text is not None:
									UpperCAmelCase:					Dict 						=     self.tokenizer(
									    __snake_case	,				padding=__snake_case	,				max_length=__snake_case	,				return_tensors=__snake_case	,				**__snake_case							)

								if images is not None:
									UpperCAmelCase:					List[str] 						=     self.image_processor(__snake_case	,				return_tensors=__snake_case	,				**__snake_case							)

								if text is not None and images is not None:
									UpperCAmelCase:					Optional[int] 						=     image_features.pixel_values
									return encoding
								elif text is not None:
									return encoding
								else:
									return BatchEncoding(data=dict(**__snake_case							)	,				tensor_type=__snake_case							)



							def     A__ (     self	,				*__snake_case	,				**__snake_case							)						->     Any:






								"""simple docstring"""



								return self.tokenizer.batch_decode(*__snake_case	,				**__snake_case							)



							def     A__ (     self	,				*__snake_case	,				**__snake_case							)						->     List[str]:






								"""simple docstring"""



								return self.tokenizer.decode(*__snake_case	,				**__snake_case							)




							@property
							def     A__ (     self							)						->     Optional[int]:






								"""simple docstring"""



								UpperCAmelCase:					Optional[int] 						=     self.tokenizer.model_input_names
								UpperCAmelCase:					int 						=     self.image_processor.model_input_names
								return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names							)							)
