

from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import (
						    AutoTokenizer,
						    TFAutoModelForCausalLM,
						    TFAutoModelForSeqaSeqLM,
						    TFAutoModelForSpeechSeqaSeq,
						    TFAutoModelForVisionaSeq,
						    TFBartForConditionalGeneration,
						    TFLogitsProcessorList,
						    TFMinLengthLogitsProcessor,
						    tf_top_k_top_p_filtering,
						)

if is_tensorflow_text_available():
						import tensorflow_text as text



@require_tf
class   __lowercase				( unittest.TestCase	):




			"""simple docstring"""







			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       tf.convert_to_tensor(
									    [
									        [
									            8.222_0991,  # 3rd highest value; idx. 0
									            -0.562_0044,
									            5.2322_9752,
									            4.038_6393,
									            -6.879_8378,
									            -0.5478_5802,
									            -3.201_2153,
									            2.9277_7176,
									            1.8817_1953,
									            7.3534_1276,  # 5th highest value; idx. 9
									            8.4320_7833,  # 2nd highest value; idx. 10
									            -9.8571_1836,
									            -5.9620_9236,
									            -1.1303_9161,
									            -7.111_5294,
									            -0.836_9633,
									            -5.318_6408,
									            7.0642_7407,
									            0.8136_9344,
									            -0.8202_3817,
									            -5.917_9796,
									            0.5881_3443,
									            -6.9977_8438,
									            4.7155_1189,
									            -0.1877_1637,
									            7.4402_0759,  # 4th highest value; idx. 25
									            9.3845_0987,  # 1st highest value; idx. 26
									            2.1266_2941,
									            -9.3256_2038,
									            2.3565_2522,
									        ],  # cummulative prob of 5 highest values <= 0.6
									        [
									            0.5842_5518,
									            4.5313_9238,
									            -5.5751_0464,
									            -6.2803_0699,
									            -7.1952_9503,
									            -4.0212_2551,
									            1.3933_7037,
									            -6.0670_7057,
									            1.5948_0517,
									            -9.64_3119,
									            0.0390_7799,
									            0.6723_1762,
									            -8.8820_6726,
									            6.2711_5922,  # 4th highest value; idx. 13
									            2.2852_0723,
									            4.8276_7506,
									            4.3042_1368,
									            8.827_5313,  # 2nd highest value; idx. 17
									            5.4402_9958,  # 5th highest value; idx. 18
									            -4.473_5794,
									            7.3857_9536,  # 3rd highest value; idx. 20
									            -2.9105_1663,
									            2.6194_6077,
									            -2.567_4762,
									            -9.4895_9302,
									            -4.0292_2645,
									            -1.3541_6918,
									            9.6770_2323,  # 1st highest value; idx. 27
									            -5.8947_8553,
									            1.8537_0467,
									        ],  # cummulative prob of 5 highest values <= 0.6
									    ]   ,						dtype=tf.floataa   ,						)

									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       tf.convert_to_tensor(
									    [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]]   ,						dtype=tf.intaa   ,						)  # expected non filtered idx as noted above

									SCREAMING_SNAKE_CASE_: List[str]  		=       tf.convert_to_tensor(
									    [8.22_2099, 7.353_4126, 8.43_2078, 7.440_2075, 9.3_8451, 6.27_1159, 8.82_7531, 5.440_2995, 7.385_7956, 9.67_7023]   ,						dtype=tf.floataa   ,						)  # expected non filtered values as noted above

									SCREAMING_SNAKE_CASE_: List[Any]  		=       tf_top_k_top_p_filtering(lowerCAmelCase__   ,						top_k=10   ,						top_p=0.6   ,						min_tokens_to_keep=4)

									SCREAMING_SNAKE_CASE_: List[str]  		=       output[output != -float("inf")]
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       tf.cast(
									    tf.where(tf.not_equal(lowerCAmelCase__   ,						tf.constant(-float("inf")   ,						dtype=tf.floataa)))   ,						dtype=tf.intaa   ,						)

									tf.debugging.assert_near(lowerCAmelCase__   ,						lowerCAmelCase__   ,						rtol=1E-12)
									tf.debugging.assert_equal(lowerCAmelCase__   ,						lowerCAmelCase__)







@require_tf
class   __lowercase				( unittest.TestCase		,       UpperCAmelCase_	):




			"""simple docstring"""







			if is_tf_available():
						_UpperCAmelCase   :							List[Any]										=     {
						    '''AutoModelForCausalLM''': TFAutoModelForCausalLM,
						    '''AutoModelForSpeechSeq2Seq''': TFAutoModelForSpeechSeqaSeq,
						    '''AutoModelForSeq2SeqLM''': TFAutoModelForSeqaSeqLM,
						    '''AutoModelForVision2Seq''': TFAutoModelForVisionaSeq,
						    '''LogitsProcessorList''': TFLogitsProcessorList,
						    '''MinLengthLogitsProcessor''': TFMinLengthLogitsProcessor,
						    '''create_tensor_fn''': tf.convert_to_tensor,
						    '''floats_tensor''': floats_tensor,
						    '''return_tensors''': '''tf''',
						}



			@slow
			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]):
									# TF-only test: tf.saved_model export
									SCREAMING_SNAKE_CASE_: Any  		=       TFAutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2")
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       2
									SCREAMING_SNAKE_CASE_: int  		=       2



									class   __lowercase				( tf.Module	):




												"""simple docstring"""







												def __init__( self  :     Union[str, Any]   ,						lowerCAmelCase__  :     str):
																		super(lowerCAmelCase__   ,						self).__init__()
																		SCREAMING_SNAKE_CASE_: Any  		=       model




												@tf.function(
												    input_signature=(
												        tf.TensorSpec((None, input_length)   ,						tf.intaa   ,						name="input_ids"),
												        tf.TensorSpec((None, input_length)   ,						tf.intaa   ,						name="attention_mask"),
												    )   ,						jit_compile=lowerCAmelCase__   ,						)
												def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]   ,						lowerCAmelCase__  :     List[str]   ,						lowerCAmelCase__  :     List[str]):
																		SCREAMING_SNAKE_CASE_: Any  		=       self.model.generate(
																		    input_ids=lowerCAmelCase__   ,						attention_mask=lowerCAmelCase__   ,						max_new_tokens=lowerCAmelCase__   ,						return_dict_in_generate=lowerCAmelCase__   ,						)
																		return {"sequences": outputs["sequences"]}







									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       [[2, 0], [102, 103]]
									SCREAMING_SNAKE_CASE_: int  		=       [[1, 0], [1, 1]]
									SCREAMING_SNAKE_CASE_: List[str]  		=       DummyModel(model=lowerCAmelCase__)
									with tempfile.TemporaryDirectory() as tmp_dir:
															tf.saved_model.save(lowerCAmelCase__   ,						lowerCAmelCase__   ,						signatures={"serving_default": dummy_model.serving})
															SCREAMING_SNAKE_CASE_: Optional[Any]  		=       tf.saved_model.load(lowerCAmelCase__).signatures["serving_default"]
															for batch_size in range(1   ,						len(lowerCAmelCase__) + 1):
																					SCREAMING_SNAKE_CASE_: List[Any]  		=       {
																					    "input_ids": tf.constant(dummy_input_ids[:batch_size]),
																					    "attention_mask": tf.constant(dummy_attention_masks[:batch_size]),
																					}
																					SCREAMING_SNAKE_CASE_: List[str]  		=       serving_func(**lowerCAmelCase__)["sequences"]
																					SCREAMING_SNAKE_CASE_: Optional[Any]  		=       test_model.generate(**lowerCAmelCase__   ,						max_new_tokens=lowerCAmelCase__)
																					tf.debugging.assert_equal(lowerCAmelCase__   ,						lowerCAmelCase__)



			@slow
			def        _SCREAMING_SNAKE_CASE			( self  :     List[str]):
									# TF-only test: tf.saved_model export
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       TFAutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2")
									SCREAMING_SNAKE_CASE_: Any  		=       1
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       2



									class   __lowercase				( tf.Module	):




												"""simple docstring"""







												def __init__( self  :     str   ,						lowerCAmelCase__  :     Tuple):
																		super(lowerCAmelCase__   ,						self).__init__()
																		SCREAMING_SNAKE_CASE_: Optional[Any]  		=       model




												@tf.function(
												    input_signature=(
												        tf.TensorSpec((batch_size, None)   ,						tf.intaa   ,						name="input_ids"),
												        tf.TensorSpec((batch_size, None)   ,						tf.intaa   ,						name="attention_mask"),
												    )   ,						jit_compile=lowerCAmelCase__   ,						)
												def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]   ,						lowerCAmelCase__  :     str   ,						lowerCAmelCase__  :     Optional[Any]):
																		SCREAMING_SNAKE_CASE_: Optional[int]  		=       self.model.generate(
																		    input_ids=lowerCAmelCase__   ,						attention_mask=lowerCAmelCase__   ,						max_new_tokens=lowerCAmelCase__   ,						return_dict_in_generate=lowerCAmelCase__   ,						)
																		return {"sequences": outputs["sequences"]}







									SCREAMING_SNAKE_CASE_: List[str]  		=       [[2], [102, 103]]
									SCREAMING_SNAKE_CASE_: Dict  		=       [[1], [1, 1]]
									SCREAMING_SNAKE_CASE_: Dict  		=       DummyModel(model=lowerCAmelCase__)
									with tempfile.TemporaryDirectory() as tmp_dir:
															tf.saved_model.save(lowerCAmelCase__   ,						lowerCAmelCase__   ,						signatures={"serving_default": dummy_model.serving})
															SCREAMING_SNAKE_CASE_: List[str]  		=       tf.saved_model.load(lowerCAmelCase__).signatures["serving_default"]
															for input_row in range(len(lowerCAmelCase__)):
																					SCREAMING_SNAKE_CASE_: Dict  		=       {
																					    "input_ids": tf.constant([dummy_input_ids[input_row]]),
																					    "attention_mask": tf.constant([dummy_attention_masks[input_row]]),
																					}
																					SCREAMING_SNAKE_CASE_: Optional[int]  		=       serving_func(**lowerCAmelCase__)["sequences"]
																					SCREAMING_SNAKE_CASE_: Dict  		=       test_model.generate(**lowerCAmelCase__   ,						max_new_tokens=lowerCAmelCase__)
																					tf.debugging.assert_equal(lowerCAmelCase__   ,						lowerCAmelCase__)



			@slow
			@require_tensorflow_text
			def        _SCREAMING_SNAKE_CASE			( self  :     Any):
									# TF-only test: tf.saved_model export
									with tempfile.TemporaryDirectory() as tmp_dir:
															# file needed to load the TF tokenizer
															hf_hub_download(repo_id="google/flan-t5-small"   ,						filename="spiece.model"   ,						local_dir=lowerCAmelCase__)



															class   __lowercase				( tf.keras.layers.Layer	):




																		"""simple docstring"""







																		def __init__( self  :     List[Any]):
																								super().__init__()
																								SCREAMING_SNAKE_CASE_: int  		=       text.SentencepieceTokenizer(
																								    model=tf.io.gfile.GFile(os.path.join(lowerCAmelCase__   ,						"spiece.model")   ,						"rb").read())
																								SCREAMING_SNAKE_CASE_: int  		=       TFAutoModelForSeqaSeqLM.from_pretrained("hf-internal-testing/tiny-random-t5")




																		def        _SCREAMING_SNAKE_CASE			( self  :     Tuple   ,						lowerCAmelCase__  :     Tuple   ,						*lowerCAmelCase__  :     List[Any]   ,						**lowerCAmelCase__  :     Dict):
																								SCREAMING_SNAKE_CASE_: Tuple  		=       self.tokenizer.tokenize(lowerCAmelCase__)
																								SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Dict  		=       text.pad_model_inputs(
																								    lowerCAmelCase__   ,						max_seq_length=64   ,						pad_value=self.model.config.pad_token_id)
																								SCREAMING_SNAKE_CASE_: Optional[int]  		=       self.model.generate(input_ids=lowerCAmelCase__   ,						attention_mask=lowerCAmelCase__)
																								return self.tokenizer.detokenize(lowerCAmelCase__)







															SCREAMING_SNAKE_CASE_: Optional[Any]  		=       CompleteSentenceTransformer()
															SCREAMING_SNAKE_CASE_: Any  		=       tf.keras.layers.Input(shape=(1,)   ,						dtype=tf.string   ,						name="inputs")
															SCREAMING_SNAKE_CASE_: Any  		=       complete_model(lowerCAmelCase__)
															SCREAMING_SNAKE_CASE_: Optional[Any]  		=       tf.keras.Model(lowerCAmelCase__   ,						lowerCAmelCase__)
															keras_model.save(lowerCAmelCase__)



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									# Has PT equivalent: this test relies on random sampling
									SCREAMING_SNAKE_CASE_: Tuple  		=       {
									    "do_sample": True,
									    "num_beams": 1,
									    "top_p": 0.7,
									    "top_k": 10,
									    "temperature": 0.7,
									}
									SCREAMING_SNAKE_CASE_: List[str]  		=       14

									SCREAMING_SNAKE_CASE_: Any  		=       AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2")
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       "Hello, my dog is cute and"
									SCREAMING_SNAKE_CASE_: Any  		=       tokenizer(lowerCAmelCase__   ,						return_tensors="tf")
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       TFAutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2")

									SCREAMING_SNAKE_CASE_: List[Any]  		=       638
									# forces the generation to happen on CPU, to avoid GPU-related quirks
									with tf.device(":/CPU:0"):
															tf.random.set_seed(0)
															SCREAMING_SNAKE_CASE_: Optional[int]  		=       model.generate(**lowerCAmelCase__   ,						eos_token_id=lowerCAmelCase__   ,						**lowerCAmelCase__)
									self.assertTrue(expectation == len(generated_tokens[0]))

									SCREAMING_SNAKE_CASE_: int  		=       [638, 198]
									with tf.device(":/CPU:0"):
															tf.random.set_seed(0)
															SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       model.generate(**lowerCAmelCase__   ,						eos_token_id=lowerCAmelCase__   ,						**lowerCAmelCase__)
									self.assertTrue(expectation == len(generated_tokens[0]))




			def        _SCREAMING_SNAKE_CASE			( self  :     Tuple):
									# Has PT equivalent: ample use of framework-specific code
									SCREAMING_SNAKE_CASE_: Any  		=       AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bart")
									SCREAMING_SNAKE_CASE_: Tuple  		=       "Hugging Face is a technology company based in New York and Paris."
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       bart_tokenizer(lowerCAmelCase__   ,						return_tensors="tf").input_ids
									SCREAMING_SNAKE_CASE_: Any  		=       TFBartForConditionalGeneration.from_pretrained("hf-internal-testing/tiny-random-bart")
									SCREAMING_SNAKE_CASE_: List[Any]  		=       bart_model.generate(lowerCAmelCase__).numpy()



									class   __lowercase				( UpperCAmelCase_	):




												"""simple docstring"""







												def        _SCREAMING_SNAKE_CASE			( self  :     Tuple   ,						lowerCAmelCase__  :     Union[str, Any]   ,						lowerCAmelCase__  :     Tuple=None   ,						**lowerCAmelCase__  :     List[Any]):
																		return super().call(lowerCAmelCase__   ,						**lowerCAmelCase__)







									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       FakeBart.from_pretrained("hf-internal-testing/tiny-random-bart")
									SCREAMING_SNAKE_CASE_: Dict  		=       bart_model.generate(lowerCAmelCase__   ,						foo="bar").numpy()
									self.assertTrue(np.array_equal(lowerCAmelCase__   ,						lowerCAmelCase__))



									class   __lowercase				( bart_model.model.encoder.__class__	):




												"""simple docstring"""







												def        _SCREAMING_SNAKE_CASE			( self  :     List[str]   ,						lowerCAmelCase__  :     Tuple   ,						**lowerCAmelCase__  :     Tuple):
																		return super().call(lowerCAmelCase__   ,						**lowerCAmelCase__)







									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       FakeEncoder(bart_model.config   ,						bart_model.model.shared)
									SCREAMING_SNAKE_CASE_: Dict  		=       fake_encoder

									# Normal generation still works (the output will be different because the encoder weights are different)
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       bart_model.generate(lowerCAmelCase__).numpy()
									with self.assertRaises(lowerCAmelCase__):
															# FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
															bart_model.generate(lowerCAmelCase__   ,						foo="bar")







'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__SCREAMING_SNAKE_CASE       :     str     =  logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE       :     str     =  {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__SCREAMING_SNAKE_CASE       :     Dict     =  {
    """vocab_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-german-cased""": """https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt""",
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-german-cased""": (
            """https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
    },
}

__SCREAMING_SNAKE_CASE       :     Optional[Any]     =  {
    """distilbert-base-uncased""": 512,
    """distilbert-base-uncased-distilled-squad""": 512,
    """distilbert-base-cased""": 512,
    """distilbert-base-cased-distilled-squad""": 512,
    """distilbert-base-german-cased""": 512,
    """distilbert-base-multilingual-cased""": 512,
}


__SCREAMING_SNAKE_CASE       :     List[Any]     =  {
    """distilbert-base-uncased""": {"""do_lower_case""": True},
    """distilbert-base-uncased-distilled-squad""": {"""do_lower_case""": True},
    """distilbert-base-cased""": {"""do_lower_case""": False},
    """distilbert-base-cased-distilled-squad""": {"""do_lower_case""": False},
    """distilbert-base-german-cased""": {"""do_lower_case""": False},
    """distilbert-base-multilingual-cased""": {"""do_lower_case""": False},
}

class 	lowerCamelCase_	(snake_case__    ):
      '''simple docstring'''

      __UpperCamelCase:  Union[str, Any]						=						VOCAB_FILES_NAMES
      __UpperCamelCase:  str						=						PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase:  Optional[Any]						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase:  Any						=						PRETRAINED_INIT_CONFIGURATION
      __UpperCamelCase:  str						=						["input_ids", "attention_mask"]
      __UpperCamelCase:  List[str]						=						DistilBertTokenizer




      def __init__(       self						:				str						,					A						:				int=None						,					A						:				Tuple=None						,					A						:				Tuple=True						,					A						:				Dict="[UNK]"						,					A						:				List[Any]="[SEP]"						,					A						:				Optional[Any]="[PAD]"						,					A						:				Dict="[CLS]"						,					A						:				Tuple="[MASK]"						,					A						:				str=True						,					A						:				Dict=None						,					**A						:				List[Any]						,					):
             super().__init__(
                 A						,					tokenizer_file=A						,					do_lower_case=A						,					unk_token=A						,					sep_token=A						,					pad_token=A						,					cls_token=A						,					mask_token=A						,					tokenize_chinese_chars=A						,					strip_accents=A						,					**A						,					)

             _UpperCAmelCase     :  str							      =    json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
             if (
                 normalizer_state.get("lowercase"						,					A       ) != do_lower_case
                 or normalizer_state.get("strip_accents"						,					A       ) != strip_accents
                 or normalizer_state.get("handle_chinese_chars"						,					A       ) != tokenize_chinese_chars
             ):
                    _UpperCAmelCase     :  Dict							      =    getattr(A						,					normalizer_state.pop("type"       )       )
                    _UpperCAmelCase     :  int							      =    do_lower_case
                    _UpperCAmelCase     :  Optional[int]							      =    strip_accents
                    _UpperCAmelCase     :  str							      =    tokenize_chinese_chars
                    _UpperCAmelCase     :  List[Any]							      =    normalizer_class(**A       )

             _UpperCAmelCase     :  Dict							      =    do_lower_case




      def      _A	(       self						:				List[Any]						,					A						:				Tuple						,					A						:				Any=None       ):
             _UpperCAmelCase     :  Optional[int]							      =    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                    output += token_ids_a + [self.sep_token_id]

             return output




      def      _A	(       self						:				int						,					A						:				List[int]						,					A						:				Optional[List[int]] = None       ):
             _UpperCAmelCase     :  Any							      =    [self.sep_token_id]
             _UpperCAmelCase     :  str							      =    [self.cls_token_id]
             if token_ids_a is None:
                    return len(cls + token_ids_a + sep       ) * [0]
             return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





      def      _A	(       self						:				Dict						,					A						:				str						,					A						:				Optional[str] = None       ):
             _UpperCAmelCase     :  Any							      =    self._tokenizer.model.save(A						,					name=A       )
             return tuple(A       )

