




'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import MobileBertConfig, is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import (
							    TF_MODEL_FOR_PRETRAINING_MAPPING,
							    TFMobileBertForMaskedLM,
							    TFMobileBertForMultipleChoice,
							    TFMobileBertForNextSentencePrediction,
							    TFMobileBertForPreTraining,
							    TFMobileBertForQuestionAnswering,
							    TFMobileBertForSequenceClassification,
							    TFMobileBertForTokenClassification,
							    TFMobileBertModel,
							)






@require_tf
class 						a				(   _lowerCamelCase   ,     _lowerCamelCase   ,     unittest.TestCase     ):
				snake_case_      =		(
				    (
				        TFMobileBertModel,
				        TFMobileBertForMaskedLM,
				        TFMobileBertForNextSentencePrediction,
				        TFMobileBertForPreTraining,
				        TFMobileBertForQuestionAnswering,
				        TFMobileBertForSequenceClassification,
				        TFMobileBertForTokenClassification,
				        TFMobileBertForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				snake_case_      =		(
				    {
				        "feature-extraction": TFMobileBertModel,
				        "fill-mask": TFMobileBertForMaskedLM,
				        "question-answering": TFMobileBertForQuestionAnswering,
				        "text-classification": TFMobileBertForSequenceClassification,
				        "token-classification": TFMobileBertForTokenClassification,
				        "zero-shot": TFMobileBertForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				snake_case_      =		False
				snake_case_      =		False


				def   A_ (		self    :					Dict       ,				lowercase_    :					Tuple       ,				lowercase_    :					Tuple       ,				lowercase_    :					Union[str, Any]=False					):
									snake_case_									=				super()._prepare_for_class(lowercase_       ,				lowercase_       ,				return_labels=lowercase_					)

									if return_labels:
														if model_class in get_values(lowercase_					):
																			snake_case_									=				tf.zeros(self.model_tester.batch_size       ,				dtype=tf.intaa					)

									return inputs_dict






				class 						a				(   _lowerCamelCase     ):


								def __init__(		self    :					Union[str, Any]       ,				lowercase_    :					str       ,				lowercase_    :					int=13       ,				lowercase_    :					Union[str, Any]=7       ,				lowercase_    :					Union[str, Any]=True       ,				lowercase_    :					int=True       ,				lowercase_    :					int=True       ,				lowercase_    :					List[Any]=True       ,				lowercase_    :					List[Any]=99       ,				lowercase_    :					Tuple=32       ,				lowercase_    :					Any=32       ,				lowercase_    :					Tuple=2       ,				lowercase_    :					List[Any]=4       ,				lowercase_    :					List[str]=37       ,				lowercase_    :					List[Any]="gelu"       ,				lowercase_    :					List[str]=0.1       ,				lowercase_    :					Optional[Any]=0.1       ,				lowercase_    :					Optional[Any]=512       ,				lowercase_    :					Union[str, Any]=16       ,				lowercase_    :					Optional[int]=2       ,				lowercase_    :					Tuple=0.02       ,				lowercase_    :					Optional[int]=3       ,				lowercase_    :					Any=4       ,				lowercase_    :					Dict=None       ,				):
													snake_case_									=				parent
													snake_case_									=				batch_size
													snake_case_									=				seq_length
													snake_case_									=				is_training
													snake_case_									=				use_input_mask
													snake_case_									=				use_token_type_ids
													snake_case_									=				use_labels
													snake_case_									=				vocab_size
													snake_case_									=				hidden_size
													snake_case_									=				num_hidden_layers
													snake_case_									=				num_attention_heads
													snake_case_									=				intermediate_size
													snake_case_									=				hidden_act
													snake_case_									=				hidden_dropout_prob
													snake_case_									=				attention_probs_dropout_prob
													snake_case_									=				max_position_embeddings
													snake_case_									=				type_vocab_size
													snake_case_									=				type_sequence_label_size
													snake_case_									=				initializer_range
													snake_case_									=				num_labels
													snake_case_									=				num_choices
													snake_case_									=				scope
													snake_case_									=				embedding_size


								def   A_ (		self    :					Union[str, Any]					):
													snake_case_									=				ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size					)

													snake_case_									=				None
													if self.use_input_mask:
																		snake_case_									=				random_attention_mask([self.batch_size, self.seq_length]					)

													snake_case_									=				None
													if self.use_token_type_ids:
																		snake_case_									=				ids_tensor([self.batch_size, self.seq_length]       ,				self.type_vocab_size					)

													snake_case_									=				None
													snake_case_									=				None
													snake_case_									=				None
													if self.use_labels:
																		snake_case_									=				ids_tensor([self.batch_size]       ,				self.type_sequence_label_size					)
																		snake_case_									=				ids_tensor([self.batch_size, self.seq_length]       ,				self.num_labels					)
																		snake_case_									=				ids_tensor([self.batch_size]       ,				self.num_choices					)

													snake_case_									=				MobileBertConfig(
													    vocab_size=self.vocab_size       ,				hidden_size=self.hidden_size       ,				num_hidden_layers=self.num_hidden_layers       ,				num_attention_heads=self.num_attention_heads       ,				intermediate_size=self.intermediate_size       ,				hidden_act=self.hidden_act       ,				hidden_dropout_prob=self.hidden_dropout_prob       ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,				max_position_embeddings=self.max_position_embeddings       ,				type_vocab_size=self.type_vocab_size       ,				initializer_range=self.initializer_range       ,				embedding_size=self.embedding_size       ,				)

													return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


								def   A_ (		self    :					List[Any]       ,				lowercase_    :					int       ,				lowercase_    :					Tuple       ,				lowercase_    :					str       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Dict       ,				lowercase_    :					List[Any]       ,				lowercase_    :					Optional[int]					):
													snake_case_									=				TFMobileBertModel(config=lowercase_					)
													snake_case_									=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
													snake_case_									=				model(lowercase_					)

													snake_case_									=				[input_ids, input_mask]
													snake_case_									=				model(lowercase_					)

													snake_case_									=				model(lowercase_					)

													self.parent.assertEqual(
													    result.last_hidden_state.shape       ,				(self.batch_size, self.seq_length, self.hidden_size)					)
													self.parent.assertEqual(result.pooler_output.shape       ,				(self.batch_size, self.hidden_size)					)


								def   A_ (		self    :					List[str]       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					str       ,				lowercase_    :					Dict       ,				lowercase_    :					str       ,				lowercase_    :					int       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Optional[int]					):
													snake_case_									=				TFMobileBertForMaskedLM(config=lowercase_					)
													snake_case_									=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
													snake_case_									=				model(lowercase_					)
													self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)					)


								def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					int       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Dict       ,				lowercase_    :					Any       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					List[str]					):
													snake_case_									=				TFMobileBertForNextSentencePrediction(config=lowercase_					)
													snake_case_									=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
													snake_case_									=				model(lowercase_					)
													self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, 2)					)


								def   A_ (		self    :					Dict       ,				lowercase_    :					int       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Any					):
													snake_case_									=				TFMobileBertForPreTraining(config=lowercase_					)
													snake_case_									=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
													snake_case_									=				model(lowercase_					)
													self.parent.assertEqual(
													    result.prediction_logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)					)
													self.parent.assertEqual(result.seq_relationship_logits.shape       ,				(self.batch_size, 2)					)


								def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					str       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					Dict       ,				lowercase_    :					Any       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Tuple					):
													snake_case_									=				self.num_labels
													snake_case_									=				TFMobileBertForSequenceClassification(config=lowercase_					)
													snake_case_									=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
													snake_case_									=				model(lowercase_					)
													self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_labels)					)


								def   A_ (		self    :					List[Any]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Dict       ,				lowercase_    :					int       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					int       ,				lowercase_    :					Tuple       ,				lowercase_    :					Any					):
													snake_case_									=				self.num_choices
													snake_case_									=				TFMobileBertForMultipleChoice(config=lowercase_					)
													snake_case_									=				tf.tile(tf.expand_dims(lowercase_       ,				1					)       ,				(1, self.num_choices, 1)					)
													snake_case_									=				tf.tile(tf.expand_dims(lowercase_       ,				1					)       ,				(1, self.num_choices, 1)					)
													snake_case_									=				tf.tile(tf.expand_dims(lowercase_       ,				1					)       ,				(1, self.num_choices, 1)					)
													snake_case_									=				{
													    '''input_ids''': multiple_choice_inputs_ids,
													    '''attention_mask''': multiple_choice_input_mask,
													    '''token_type_ids''': multiple_choice_token_type_ids,
													}
													snake_case_									=				model(lowercase_					)
													self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_choices)					)


								def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					int       ,				lowercase_    :					Dict       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					int       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Any					):
													snake_case_									=				self.num_labels
													snake_case_									=				TFMobileBertForTokenClassification(config=lowercase_					)
													snake_case_									=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
													snake_case_									=				model(lowercase_					)
													self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.num_labels)					)


								def   A_ (		self    :					List[Any]       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					int       ,				lowercase_    :					Any       ,				lowercase_    :					Any       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					List[Any]       ,				lowercase_    :					Tuple					):
													snake_case_									=				TFMobileBertForQuestionAnswering(config=lowercase_					)
													snake_case_									=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
													snake_case_									=				model(lowercase_					)
													self.parent.assertEqual(result.start_logits.shape       ,				(self.batch_size, self.seq_length)					)
													self.parent.assertEqual(result.end_logits.shape       ,				(self.batch_size, self.seq_length)					)


								def   A_ (		self    :					int					):
													snake_case_									=				self.prepare_config_and_inputs()
													(
													    (
													    snake_case_
													)			,(
													    snake_case_
													)			,(
													    snake_case_
													)			,(
													    snake_case_
													)			,(
													    snake_case_
													)			,(
													    snake_case_
													)			,(
													    snake_case_
													)			,
													)									=				config_and_inputs
													snake_case_									=				{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
													return config, inputs_dict







				def   A_ (		self    :					str					):
									snake_case_									=				TFMobileBertModelTest.TFMobileBertModelTester(self					)
									snake_case_									=				ConfigTester(self       ,				config_class=lowercase_       ,				hidden_size=37					)


				def   A_ (		self    :					int					):
									self.config_tester.run_common_tests()


				def   A_ (		self    :					str					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_model(*lowercase_					)


				def   A_ (		self    :					Dict					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_masked_lm(*lowercase_					)


				def   A_ (		self    :					Any					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_multiple_choice(*lowercase_					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*lowercase_					)


				def   A_ (		self    :					Any					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_pretraining(*lowercase_					)


				def   A_ (		self    :					List[Any]					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_question_answering(*lowercase_					)


				def   A_ (		self    :					Optional[Any]					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_sequence_classification(*lowercase_					)


				def   A_ (		self    :					List[str]					):
									snake_case_									=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_token_classification(*lowercase_					)


				@slow
				def   A_ (		self    :					List[str]					):
									# for model_name in TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									for model_name in ["google/mobilebert-uncased"]:
														snake_case_									=				TFMobileBertModel.from_pretrained(lowercase_					)
														self.assertIsNotNone(lowercase_					)







@require_tf
class 						a				(   unittest.TestCase     ):


				@slow
				def   A_ (		self    :					Optional[Any]					):
									snake_case_									=				TFMobileBertForPreTraining.from_pretrained('''google/mobilebert-uncased'''					)
									snake_case_									=				tf.constant([[0, 1, 2, 3, 4, 5]]					)
									snake_case_									=				model(lowercase_					)[0]

									snake_case_									=				[1, 6, 3_0522]
									self.assertEqual(output.shape       ,				lowercase_					)

									snake_case_									=				tf.constant(
									    [
									        [
									            [-4.591_9547, -9.24_8295, -9.64_5256],
									            [-6.730_6175, -6.44_0284, -6.605_2837],
									            [-7.274_3506, -6.784_7915, -6.02_4673],
									        ]
									    ]					)
									tf.debugging.assert_near(output[:, :3, :3]       ,				lowercase_       ,				atol=1e-4					)







'''simple docstring'''

import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_ta import TaTokenizer
else:
							a  :				Dict									=  None


a  :				List[Any]									=  logging.get_logger(__name__)

a  :				List[Any]									=  {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

a  :				str									=  {
    'vocab_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/spiece.model',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/spiece.model',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/spiece.model',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/spiece.model',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/tokenizer.json',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/tokenizer.json',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/tokenizer.json',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/tokenizer.json',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/tokenizer.json',
    },
}


# TODO(PVP) - this should be removed in Transformers v5
a  :				List[Any]									=  {
    't5-small': 512,
    't5-base': 512,
    't5-large': 512,
    't5-3b': 512,
    't5-11b': 512,
}






class 						a				(   _lowerCamelCase     ):
				snake_case_      =		VOCAB_FILES_NAMES
				snake_case_      =		PRETRAINED_VOCAB_FILES_MAP
				snake_case_      =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				snake_case_      =		["input_ids", "attention_mask"]
				snake_case_      =		TaTokenizer

				snake_case_      =		[]


				def __init__(		self    :					List[Any]       ,				lowercase_    :					int=None       ,				lowercase_    :					Dict=None       ,				lowercase_    :					Dict="</s>"       ,				lowercase_    :					List[Any]="<unk>"       ,				lowercase_    :					int="<pad>"       ,				lowercase_    :					int=100       ,				lowercase_    :					List[Any]=None       ,				**lowercase_    :					List[str]       ,				):
									# Add extra_ids to the special token list
									if extra_ids > 0 and additional_special_tokens is None:
														snake_case_									=				[F"<extra_id_{i}>" for i in range(lowercase_					)]
									elif extra_ids > 0 and additional_special_tokens is not None:
														# Check that we have the right number of extra special tokens
														snake_case_									=				len(set(filter(lambda lowercase_					: bool('''extra_id_''' in str(lowercase_					)					)       ,				lowercase_					)					)					)
														if extra_tokens != extra_ids:
																			raise ValueError(
																			    F"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
																			    ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
																			    ''' tokens'''					)

									super().__init__(
									    lowercase_       ,				tokenizer_file=lowercase_       ,				eos_token=lowercase_       ,				unk_token=lowercase_       ,				pad_token=lowercase_       ,				extra_ids=lowercase_       ,				additional_special_tokens=lowercase_       ,				**lowercase_       ,				)

									snake_case_									=				vocab_file
									snake_case_									=				False if not self.vocab_file else True
									snake_case_									=				extra_ids


				@staticmethod
				def   A_ (		lowercase_    :					Optional[int]       ,				lowercase_    :					List[str]       ,				lowercase_    :					int					):
									if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
														snake_case_									=				TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
														if init_max_model_length is not None and init_max_model_length != max_model_length:
																			return init_max_model_length
														elif init_max_model_length is None:
																			warnings.warn(
																			    '''This tokenizer was incorrectly instantiated with a model max length of'''
																			    F" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
																			    ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
																			    ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
																			    F" {pretrained_model_name_or_path} automatically truncating your input to"
																			    F" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
																			    F" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
																			    ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
																			    ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''       ,				lowercase_       ,				)

									return max_model_length


				def   A_ (		self    :					Optional[Any]       ,				lowercase_    :					str       ,				lowercase_    :					Optional[str] = None					):
									if not self.can_save_slow_tokenizer:
														raise ValueError(
														    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
														    '''tokenizer.'''					)

									if not os.path.isdir(lowercase_					):
														logger.error(F"Vocabulary path ({save_directory}) should be a directory"					)
														return
									snake_case_									=				os.path.join(
									    lowercase_       ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

									if os.path.abspath(self.vocab_file					) != os.path.abspath(lowercase_					):
														copyfile(self.vocab_file       ,				lowercase_					)
														logger.info(F"Copy vocab file to {out_vocab_file}"					)

									return (out_vocab_file,)


				def   A_ (		self    :					Optional[int]       ,				lowercase_    :					List[int]       ,				lowercase_    :					Optional[List[int]] = None					):
									snake_case_									=				token_ids_a + [self.eos_token_id]
									if token_ids_a is None:
														return self.prefix_tokens + token_ids_a
									else:
														snake_case_									=				token_ids_a + [self.eos_token_id]
														return self.prefix_tokens + token_ids_a + token_ids_a


				def   A_ (		self    :					int       ,				lowercase_    :					List[int]       ,				lowercase_    :					Optional[List[int]] = None					):
									snake_case_									=				[self.eos_token_id]

									if token_ids_a is None:
														return len(token_ids_a + eos					) * [0]
									return len(token_ids_a + eos + token_ids_a + eos					) * [0]


				def   A_ (		self    :					Dict					):
									return list(
									    set(filter(lambda lowercase_					: bool(re.search(R'''<extra_id_\d+>'''       ,				lowercase_					)					) is not None       ,				self.additional_special_tokens					)					)					)


				def   A_ (		self    :					Any					):
									return [self.convert_tokens_to_ids(lowercase_					) for token in self.get_sentinel_tokens()]

