






from __future__ import annotations

import unittest

from transformers import MobileBertConfig, is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import (
							    TF_MODEL_FOR_PRETRAINING_MAPPING,
							    TFMobileBertForMaskedLM,
							    TFMobileBertForMultipleChoice,
							    TFMobileBertForNextSentencePrediction,
							    TFMobileBertForPreTraining,
							    TFMobileBertForQuestionAnswering,
							    TFMobileBertForSequenceClassification,
							    TFMobileBertForTokenClassification,
							    TFMobileBertModel,
							)




@require_tf
class 				__A     (			__UpperCamelCase      ,     __UpperCamelCase      ,     unittest.TestCase ):





				"""simple docstring"""







				UpperCamelCase__ :      Optional[int]			 =(
				    (
				        TFMobileBertModel,
				        TFMobileBertForMaskedLM,
				        TFMobileBertForNextSentencePrediction,
				        TFMobileBertForPreTraining,
				        TFMobileBertForQuestionAnswering,
				        TFMobileBertForSequenceClassification,
				        TFMobileBertForTokenClassification,
				        TFMobileBertForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				UpperCamelCase__ :      Union[str, Any]			 =(
				    {
				        """feature-extraction""": TFMobileBertModel,
				        """fill-mask""": TFMobileBertForMaskedLM,
				        """question-answering""": TFMobileBertForQuestionAnswering,
				        """text-classification""": TFMobileBertForSequenceClassification,
				        """token-classification""": TFMobileBertForTokenClassification,
				        """zero-shot""": TFMobileBertForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				UpperCamelCase__ :      Dict			 =False
				UpperCamelCase__ :      Any			 =False


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__=False    ):




									"""simple docstring"""

									__UpperCamelCase      :					Dict       		=super()._prepare_for_class(lowerCamelCase__							,  lowerCamelCase__							,  return_labels=lowerCamelCase__    )

									if return_labels:
														if model_class in get_values(lowerCamelCase__    ):
																			__UpperCamelCase      :					Any       		=tf.zeros(self.model_tester.batch_size							,  dtype=tf.intaa    )

									return inputs_dict




				class 				__A     (			__UpperCamelCase ):





								"""simple docstring"""







								def __init__( self							,  lowerCamelCase__							,  lowerCamelCase__=13							,  lowerCamelCase__=7							,  lowerCamelCase__=True							,  lowerCamelCase__=True							,  lowerCamelCase__=True							,  lowerCamelCase__=True							,  lowerCamelCase__=99							,  lowerCamelCase__=32							,  lowerCamelCase__=32							,  lowerCamelCase__=2							,  lowerCamelCase__=4							,  lowerCamelCase__=37							,  lowerCamelCase__="gelu"							,  lowerCamelCase__=0.1							,  lowerCamelCase__=0.1							,  lowerCamelCase__=512							,  lowerCamelCase__=16							,  lowerCamelCase__=2							,  lowerCamelCase__=0.02							,  lowerCamelCase__=3							,  lowerCamelCase__=4							,  lowerCamelCase__=None							,  ):




													"""simple docstring"""

													__UpperCamelCase      :					List[str]       		=parent
													__UpperCamelCase      :					Any       		=batch_size
													__UpperCamelCase      :					List[str]       		=seq_length
													__UpperCamelCase      :					List[str]       		=is_training
													__UpperCamelCase      :					Optional[Any]       		=use_input_mask
													__UpperCamelCase      :					List[Any]       		=use_token_type_ids
													__UpperCamelCase      :					Union[str, Any]       		=use_labels
													__UpperCamelCase      :					Union[str, Any]       		=vocab_size
													__UpperCamelCase      :					List[Any]       		=hidden_size
													__UpperCamelCase      :					int       		=num_hidden_layers
													__UpperCamelCase      :					Union[str, Any]       		=num_attention_heads
													__UpperCamelCase      :					List[Any]       		=intermediate_size
													__UpperCamelCase      :					Tuple       		=hidden_act
													__UpperCamelCase      :					int       		=hidden_dropout_prob
													__UpperCamelCase      :					Optional[Any]       		=attention_probs_dropout_prob
													__UpperCamelCase      :					Tuple       		=max_position_embeddings
													__UpperCamelCase      :					Optional[int]       		=type_vocab_size
													__UpperCamelCase      :					Any       		=type_sequence_label_size
													__UpperCamelCase      :					str       		=initializer_range
													__UpperCamelCase      :					Dict       		=num_labels
													__UpperCamelCase      :					List[str]       		=num_choices
													__UpperCamelCase      :					Optional[Any]       		=scope
													__UpperCamelCase      :					Dict       		=embedding_size


								def       __lowercase							( self    ):




													"""simple docstring"""

													__UpperCamelCase      :					Union[str, Any]       		=ids_tensor([self.batch_size, self.seq_length]							,  self.vocab_size    )

													__UpperCamelCase      :					int       		=None
													if self.use_input_mask:
																		__UpperCamelCase      :					List[Any]       		=random_attention_mask([self.batch_size, self.seq_length]    )

													__UpperCamelCase      :					List[str]       		=None
													if self.use_token_type_ids:
																		__UpperCamelCase      :					Any       		=ids_tensor([self.batch_size, self.seq_length]							,  self.type_vocab_size    )

													__UpperCamelCase      :					Optional[int]       		=None
													__UpperCamelCase      :					Union[str, Any]       		=None
													__UpperCamelCase      :					Optional[int]       		=None
													if self.use_labels:
																		__UpperCamelCase      :					List[str]       		=ids_tensor([self.batch_size]							,  self.type_sequence_label_size    )
																		__UpperCamelCase      :					Tuple       		=ids_tensor([self.batch_size, self.seq_length]							,  self.num_labels    )
																		__UpperCamelCase      :					Optional[Any]       		=ids_tensor([self.batch_size]							,  self.num_choices    )

													__UpperCamelCase      :					Optional[Any]       		=MobileBertConfig(
													    vocab_size=self.vocab_size							,  hidden_size=self.hidden_size							,  num_hidden_layers=self.num_hidden_layers							,  num_attention_heads=self.num_attention_heads							,  intermediate_size=self.intermediate_size							,  hidden_act=self.hidden_act							,  hidden_dropout_prob=self.hidden_dropout_prob							,  attention_probs_dropout_prob=self.attention_probs_dropout_prob							,  max_position_embeddings=self.max_position_embeddings							,  type_vocab_size=self.type_vocab_size							,  initializer_range=self.initializer_range							,  embedding_size=self.embedding_size							,  )

													return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


								def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




													"""simple docstring"""

													__UpperCamelCase      :					Union[str, Any]       		=TFMobileBertModel(config=lowerCamelCase__    )
													__UpperCamelCase      :					Tuple       		={'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
													__UpperCamelCase      :					Dict       		=model(lowerCamelCase__    )

													__UpperCamelCase      :					List[Any]       		=[input_ids, input_mask]
													__UpperCamelCase      :					Optional[Any]       		=model(lowerCamelCase__    )

													__UpperCamelCase      :					Optional[Any]       		=model(lowerCamelCase__    )

													self.parent.assertEqual(
													    result.last_hidden_state.shape							,  (self.batch_size, self.seq_length, self.hidden_size)    )
													self.parent.assertEqual(result.pooler_output.shape							,  (self.batch_size, self.hidden_size)    )


								def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




													"""simple docstring"""

													__UpperCamelCase      :					List[Any]       		=TFMobileBertForMaskedLM(config=lowerCamelCase__    )
													__UpperCamelCase      :					str       		={'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
													__UpperCamelCase      :					Tuple       		=model(lowerCamelCase__    )
													self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.seq_length, self.vocab_size)    )


								def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




													"""simple docstring"""

													__UpperCamelCase      :					Union[str, Any]       		=TFMobileBertForNextSentencePrediction(config=lowerCamelCase__    )
													__UpperCamelCase      :					Optional[int]       		={'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
													__UpperCamelCase      :					str       		=model(lowerCamelCase__    )
													self.parent.assertEqual(result.logits.shape							,  (self.batch_size, 2)    )


								def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




													"""simple docstring"""

													__UpperCamelCase      :					Union[str, Any]       		=TFMobileBertForPreTraining(config=lowerCamelCase__    )
													__UpperCamelCase      :					Optional[Any]       		={'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
													__UpperCamelCase      :					Dict       		=model(lowerCamelCase__    )
													self.parent.assertEqual(
													    result.prediction_logits.shape							,  (self.batch_size, self.seq_length, self.vocab_size)    )
													self.parent.assertEqual(result.seq_relationship_logits.shape							,  (self.batch_size, 2)    )


								def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




													"""simple docstring"""

													__UpperCamelCase      :					List[Any]       		=self.num_labels
													__UpperCamelCase      :					Tuple       		=TFMobileBertForSequenceClassification(config=lowerCamelCase__    )
													__UpperCamelCase      :					Union[str, Any]       		={'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
													__UpperCamelCase      :					Optional[Any]       		=model(lowerCamelCase__    )
													self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.num_labels)    )


								def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




													"""simple docstring"""

													__UpperCamelCase      :					Union[str, Any]       		=self.num_choices
													__UpperCamelCase      :					List[Any]       		=TFMobileBertForMultipleChoice(config=lowerCamelCase__    )
													__UpperCamelCase      :					Optional[Any]       		=tf.tile(tf.expand_dims(lowerCamelCase__							,  1    )							,  (1, self.num_choices, 1)    )
													__UpperCamelCase      :					Optional[Any]       		=tf.tile(tf.expand_dims(lowerCamelCase__							,  1    )							,  (1, self.num_choices, 1)    )
													__UpperCamelCase      :					Tuple       		=tf.tile(tf.expand_dims(lowerCamelCase__							,  1    )							,  (1, self.num_choices, 1)    )
													__UpperCamelCase      :					int       		={
													    'input_ids': multiple_choice_inputs_ids,
													    'attention_mask': multiple_choice_input_mask,
													    'token_type_ids': multiple_choice_token_type_ids,
													}
													__UpperCamelCase      :					Optional[int]       		=model(lowerCamelCase__    )
													self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.num_choices)    )


								def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




													"""simple docstring"""

													__UpperCamelCase      :					Optional[Any]       		=self.num_labels
													__UpperCamelCase      :					List[str]       		=TFMobileBertForTokenClassification(config=lowerCamelCase__    )
													__UpperCamelCase      :					List[Any]       		={'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
													__UpperCamelCase      :					Optional[Any]       		=model(lowerCamelCase__    )
													self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.seq_length, self.num_labels)    )


								def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




													"""simple docstring"""

													__UpperCamelCase      :					Optional[Any]       		=TFMobileBertForQuestionAnswering(config=lowerCamelCase__    )
													__UpperCamelCase      :					List[str]       		={'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
													__UpperCamelCase      :					List[Any]       		=model(lowerCamelCase__    )
													self.parent.assertEqual(result.start_logits.shape							,  (self.batch_size, self.seq_length)    )
													self.parent.assertEqual(result.end_logits.shape							,  (self.batch_size, self.seq_length)    )







								def       __lowercase							( self    ):




													"""simple docstring"""

													__UpperCamelCase      :					Optional[int]       		=self.prepare_config_and_inputs()
													(
													    __UpperCamelCase							
													)      :					int       		=config_and_inputs
													__UpperCamelCase      :					str       		={'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
													return config, inputs_dict


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					str       		=TFMobileBertModelTest.TFMobileBertModelTester(self    )
									__UpperCamelCase      :					Optional[Any]       		=ConfigTester(self							,  config_class=lowerCamelCase__							,  hidden_size=37    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									self.config_tester.run_common_tests()


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					str       		=self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_model(*lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[int]       		=self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_masked_lm(*lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Tuple       		=self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_multiple_choice(*lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					str       		=self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Dict       		=self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_pretraining(*lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					int       		=self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_question_answering(*lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					int       		=self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_sequence_classification(*lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Dict       		=self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mobilebert_for_token_classification(*lowerCamelCase__    )







				@slow
				def       __lowercase							( self    ):




									"""simple docstring"""

									for model_name in ["google/mobilebert-uncased"]:
														__UpperCamelCase      :					Union[str, Any]       		=TFMobileBertModel.from_pretrained(lowerCamelCase__    )
														self.assertIsNotNone(lowerCamelCase__    )




@require_tf
class 				__A     (			unittest.TestCase ):





				"""simple docstring"""







				@slow
				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					int       		=TFMobileBertForPreTraining.from_pretrained('google/mobilebert-uncased'    )
									__UpperCamelCase      :					List[str]       		=tf.constant([[0, 1, 2, 3, 4, 5]]    )
									__UpperCamelCase      :					List[str]       		=model(lowerCamelCase__    )[0]

									__UpperCamelCase      :					Tuple       		=[1, 6, 30522]
									self.assertEqual(output.shape							,  lowerCamelCase__    )

									__UpperCamelCase      :					List[Any]       		=tf.constant(
									    [
									        [
									            [-4.5_919_547, -9.248_295, -9.645_256],
									            [-6.7_306_175, -6.440_284, -6.6_052_837],
									            [-7.2_743_506, -6.7_847_915, -6.024_673],
									        ]
									    ]    )
									tf.debugging.assert_near(output[:, :3, :3]							,  lowerCamelCase__							,  atol=1E-4    )








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCAmelCase				:       Optional[int]	   =				logging.get_logger(__name__)

_UpperCAmelCase				:       Any	   =				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase				:       int	   =				{
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCAmelCase				:       int	   =				{
    """google/realm-cc-news-pretrained-embedder""": 5_12,
    """google/realm-cc-news-pretrained-encoder""": 5_12,
    """google/realm-cc-news-pretrained-scorer""": 5_12,
    """google/realm-cc-news-pretrained-openqa""": 5_12,
    """google/realm-orqa-nq-openqa""": 5_12,
    """google/realm-orqa-nq-reader""": 5_12,
    """google/realm-orqa-wq-openqa""": 5_12,
    """google/realm-orqa-wq-reader""": 5_12,
}

_UpperCAmelCase				:       Any	   =				{
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}

class    lowerCAmelCase							(					__UpperCamelCase			):
 UpperCAmelCase__													=		VOCAB_FILES_NAMES
 UpperCAmelCase__													=		PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__													=		PRETRAINED_INIT_CONFIGURATION
 UpperCAmelCase__													=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase__													=		RealmTokenizer
 def __init__(      self  :      Optional[int]			,     UpperCAmelCase  :      Tuple=None			,     UpperCAmelCase  :      Any=None			,     UpperCAmelCase  :      List[Any]=True			,     UpperCAmelCase  :      Optional[Any]="[UNK]"			,     UpperCAmelCase  :      Any="[SEP]"			,     UpperCAmelCase  :      Tuple="[PAD]"			,     UpperCAmelCase  :      List[Any]="[CLS]"			,     UpperCAmelCase  :      Union[str, Any]="[MASK]"			,     UpperCAmelCase  :      Optional[Any]=True			,     UpperCAmelCase  :      Any=None			,     **UpperCAmelCase  :      Optional[int]			,     )			->	str:
     super().__init__(
         UpperCAmelCase			,     tokenizer_file=UpperCAmelCase			,     do_lower_case=UpperCAmelCase			,     unk_token=UpperCAmelCase			,     sep_token=UpperCAmelCase			,     pad_token=UpperCAmelCase			,     cls_token=UpperCAmelCase			,     mask_token=UpperCAmelCase			,     tokenize_chinese_chars=UpperCAmelCase			,     strip_accents=UpperCAmelCase			,     **UpperCAmelCase			,     )

     lowerCamelCase__    :      List[Any]					 =		json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
     if (
         normalizer_state.get('lowercase'			,     UpperCAmelCase	) != do_lower_case
         or normalizer_state.get('strip_accents'			,     UpperCAmelCase	) != strip_accents
         or normalizer_state.get('handle_chinese_chars'			,     UpperCAmelCase	) != tokenize_chinese_chars
     ):
         lowerCamelCase__    :      Optional[int]					 =		getattr(UpperCAmelCase			,     normalizer_state.pop('type'	)	)
         lowerCamelCase__    :      Optional[Any]					 =		do_lower_case
         lowerCamelCase__    :      str					 =		strip_accents
         lowerCamelCase__    :      Optional[Any]					 =		tokenize_chinese_chars
         lowerCamelCase__    :      int					 =		normalizer_class(**UpperCAmelCase	)

     lowerCamelCase__    :      str					 =		do_lower_case
 def      A_	(      self  :      Optional[int]			,     UpperCAmelCase  :      int			,     **UpperCAmelCase  :      int	)			->	List[Any]:
     lowerCamelCase__    :      List[Any]					 =		PaddingStrategy.MAX_LENGTH

     lowerCamelCase__    :      Optional[int]					 =		text
     lowerCamelCase__    :      Dict					 =		kwargs.pop('text_pair'			,     UpperCAmelCase	)
     lowerCamelCase__    :      List[Any]					 =		kwargs.pop('return_tensors'			,     UpperCAmelCase	)

     lowerCamelCase__    :      List[Any]					 =		{
         'input_ids': [],
         'attention_mask': [],
         'token_type_ids': [],
     }

     for idx, candidate_text in enumerate(UpperCAmelCase	):
         if batch_text_pair is not None:
             lowerCamelCase__    :      Tuple					 =		batch_text_pair[idx]
         else:
             lowerCamelCase__    :      Dict					 =		None

         lowerCamelCase__    :      Optional[int]					 =		super().__call__(UpperCAmelCase			,     UpperCAmelCase			,     return_tensors=UpperCAmelCase			,     **UpperCAmelCase	)

         lowerCamelCase__    :      Any					 =		encoded_candidates.get('input_ids'	)
         lowerCamelCase__    :      Union[str, Any]					 =		encoded_candidates.get('attention_mask'	)
         lowerCamelCase__    :      Tuple					 =		encoded_candidates.get('token_type_ids'	)

         if encoded_input_ids is not None:
             output_data["input_ids"].append(UpperCAmelCase	)
         if encoded_attention_mask is not None:
             output_data["attention_mask"].append(UpperCAmelCase	)
         if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(UpperCAmelCase	)

     lowerCamelCase__    :      int					 =		{key: item for key, item in output_data.items() if len(UpperCAmelCase	) != 0}

     return BatchEncoding(UpperCAmelCase			,     tensor_type=UpperCAmelCase	)
 def      A_	(      self  :      int			,     UpperCAmelCase  :      Optional[Any]			,     UpperCAmelCase  :      Optional[int]=None	)			->	List[str]:
     lowerCamelCase__    :      Tuple					 =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output
 def      A_	(      self  :      Tuple			,     UpperCAmelCase  :      List[int]			,     UpperCAmelCase  :      Optional[List[int]] = None	)			->	List[int]:
     lowerCamelCase__    :      List[Any]					 =		[self.sep_token_id]
     lowerCamelCase__    :      Optional[int]					 =		[self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep	) * [0]
     return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







 def      A_	(      self  :      Optional[Any]			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      Optional[str] = None	)			->	Tuple[str]:
     lowerCamelCase__    :      int					 =		self._tokenizer.model.save(UpperCAmelCase			,     name=UpperCAmelCase	)
     return tuple(UpperCAmelCase	)





