






import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


__UpperCAmelCase								=						logging.getLogger(__name__)

require_version('pytorch_lightning>=1.0.4')

__UpperCAmelCase								=						{
    'base': AutoModel,
    'sequence-classification': AutoModelForSequenceClassification,
    'question-answering': AutoModelForQuestionAnswering,
    'pretraining': AutoModelForPreTraining,
    'token-classification': AutoModelForTokenClassification,
    'language-modeling': AutoModelWithLMHead,
    'summarization': AutoModelForSeqaSeqLM,
    'translation': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
__UpperCAmelCase								=						{
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
__UpperCAmelCase								=						sorted(arg_to_scheduler.keys())
__UpperCAmelCase								=						'{' + ', '.join(arg_to_scheduler_choices) + '}'



class   lowerCamelCase      (pl.LightningModule			):







		'''simple docstring'''



		def __init__(							self				,		_UpperCamelCase				,		_UpperCamelCase=None				,		_UpperCamelCase="base"				,		_UpperCamelCase=None				,		_UpperCamelCase=None				,		_UpperCamelCase=None				,		**_UpperCamelCase				,		)	->							Tuple:
							super().__init__()
							# TODO: move to self.save_hyperparameters()
							# self.save_hyperparameters()
							# can also expand arguments into trainer signature for easier reading

							self.save_hyperparameters(__UpperCAmelCase   )
							UpperCAmelCase_    :      str            =						0
							UpperCAmelCase_    :      int            =						Path(self.hparams.output_dir   )
							UpperCAmelCase_    :      List[Any]            =						self.hparams.cache_dir if self.hparams.cache_dir else None
							if config is None:
												UpperCAmelCase_    :      str            =						AutoConfig.from_pretrained(
												    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path				,		**({'num_labels': num_labels} if num_labels is not None else {})				,		cache_dir=__UpperCAmelCase				,		**__UpperCAmelCase				,		)
							else:
												UpperCAmelCase_    :      List[Any]            =						config

							UpperCAmelCase_    :      str            =						('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
							for p in extra_model_params:
												if getattr(self.hparams				,		__UpperCAmelCase				,		__UpperCAmelCase   ):
																	assert hasattr(self.config				,		__UpperCAmelCase   ), f"model config doesn't have a `{p}` attribute"
																	setattr(self.config				,		__UpperCAmelCase				,		getattr(self.hparams				,		__UpperCAmelCase   )   )

							if tokenizer is None:
												UpperCAmelCase_    :      List[Any]            =						AutoTokenizer.from_pretrained(
												    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path				,		cache_dir=__UpperCAmelCase				,		)
							else:
												UpperCAmelCase_    :      str            =						tokenizer
							UpperCAmelCase_    :      int            =						MODEL_MODES[mode]
							if model is None:
												UpperCAmelCase_    :      Any            =						self.model_type.from_pretrained(
												    self.hparams.model_name_or_path				,		from_tf=bool('.ckpt' in self.hparams.model_name_or_path   )				,		config=self.config				,		cache_dir=__UpperCAmelCase				,		)
							else:
												UpperCAmelCase_    :      Dict            =						model



		def 					__UpperCAmelCase			(							self				,		*_UpperCamelCase				,		**_UpperCamelCase   )	->							str:
							UpperCAmelCase_    :      List[str]            =						self.model_type.from_pretrained(*__UpperCAmelCase				,		**__UpperCAmelCase   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[Any]:
							UpperCAmelCase_    :      Optional[Any]            =						arg_to_scheduler[self.hparams.lr_scheduler]
							UpperCAmelCase_    :      List[Any]            =						get_schedule_func(
							    self.opt				,		num_warmup_steps=self.hparams.warmup_steps				,		num_training_steps=self.total_steps()   )
							UpperCAmelCase_    :      Optional[int]            =						{'scheduler': scheduler, 'interval': 'step', 'frequency': 1}
							return scheduler



		def 					__UpperCAmelCase			(							self   )	->							List[str]:
							UpperCAmelCase_    :      int            =						self.model
							UpperCAmelCase_    :      Union[str, Any]            =						['bias', 'LayerNorm.weight']
							UpperCAmelCase_    :      Tuple            =						[
							    {
							        'params': [
							            p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay   )
							        ],  # check this named paramters
							        'weight_decay': self.hparams.weight_decay,
							    },
							    {
							        'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay   )],
							        'weight_decay': 0.0,
							    },
							]
							if self.hparams.adafactor:
												UpperCAmelCase_    :      Optional[int]            =						Adafactor(
												    __UpperCAmelCase				,		lr=self.hparams.learning_rate				,		scale_parameter=__UpperCAmelCase				,		relative_step=__UpperCAmelCase   )

							else:
												UpperCAmelCase_    :      Any            =						AdamW(
												    __UpperCAmelCase				,		lr=self.hparams.learning_rate				,		eps=self.hparams.adam_epsilon   )
							UpperCAmelCase_    :      List[Any]            =						optimizer

							UpperCAmelCase_    :      Optional[int]            =						self.get_lr_scheduler()

							return [optimizer], [scheduler]



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Dict:
							return self.validation_step(__UpperCAmelCase				,		__UpperCAmelCase   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Any:
							return self.validation_end(__UpperCAmelCase   )



		def 					__UpperCAmelCase			(							self   )	->							int:
							UpperCAmelCase_    :      Union[str, Any]            =						max(1				,		self.hparams.gpus   )  # TODO: consider num_tpu_cores
							UpperCAmelCase_    :      Tuple            =						self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
							return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Optional[int]:


							if stage == "test":
												UpperCAmelCase_    :      str            =						len(self.test_dataloader().dataset   )
							else:
												UpperCAmelCase_    :      Dict            =						self.get_dataloader('train'				,		self.hparams.train_batch_size				,		shuffle=__UpperCAmelCase   )
												UpperCAmelCase_    :      int            =						len(self.train_dataloader().dataset   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase = False   )	->							str:
							raise NotImplementedError('You must implement this for your task'   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[Any]:
							return self.train_loader



		def 					__UpperCAmelCase			(							self   )	->							List[Any]:
							return self.get_dataloader('dev'				,		self.hparams.eval_batch_size				,		shuffle=__UpperCAmelCase   )



		def 					__UpperCAmelCase			(							self   )	->							Tuple:
							return self.get_dataloader('test'				,		self.hparams.eval_batch_size				,		shuffle=__UpperCAmelCase   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Optional[Any]:
							return os.path.join(
							    self.hparams.data_dir				,		'cached_{}_{}_{}'.format(
							        __UpperCAmelCase				,		list(filter(__UpperCAmelCase				,		self.hparams.model_name_or_path.split('/'   )   )   ).pop()				,		str(self.hparams.max_seq_length   )				,		)				,		)



		@pl.utilities.rank_zero_only
		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							None:
							UpperCAmelCase_    :      Any            =						self.output_dir.joinpath('best_tfmr'   )
							UpperCAmelCase_    :      Union[str, Any]            =						self.step_count
							self.model.save_pretrained(__UpperCAmelCase   )
							self.tokenizer.save_pretrained(__UpperCAmelCase   )



		@staticmethod
		def 					__UpperCAmelCase			(							_UpperCamelCase				,		_UpperCamelCase   )	->							int:
							parser.add_argument(
							    '--model_name_or_path'				,		default=__UpperCAmelCase				,		type=__UpperCAmelCase				,		required=__UpperCAmelCase				,		help='Path to pretrained model or model identifier from huggingface.co/models'				,		)
							parser.add_argument(
							    '--config_name'				,		default=''				,		type=__UpperCAmelCase				,		help='Pretrained config name or path if not the same as model_name'   )
							parser.add_argument(
							    '--tokenizer_name'				,		default=__UpperCAmelCase				,		type=__UpperCAmelCase				,		help='Pretrained tokenizer name or path if not the same as model_name'				,		)
							parser.add_argument(
							    '--cache_dir'				,		default=str(Path(__UpperCAmelCase   ).parent / 'test_run' / 'cache'   )				,		type=__UpperCAmelCase				,		help='Where do you want to store the pre-trained models downloaded from huggingface.co'				,		)
							parser.add_argument(
							    '--encoder_layerdrop'				,		type=__UpperCAmelCase				,		help='Encoder layer dropout probability (Optional). Goes into model.config'				,		)
							parser.add_argument(
							    '--decoder_layerdrop'				,		type=__UpperCAmelCase				,		help='Decoder layer dropout probability (Optional). Goes into model.config'				,		)
							parser.add_argument(
							    '--dropout'				,		type=__UpperCAmelCase				,		help='Dropout probability (Optional). Goes into model.config'				,		)
							parser.add_argument(
							    '--attention_dropout'				,		type=__UpperCAmelCase				,		help='Attention dropout probability (Optional). Goes into model.config'				,		)
							parser.add_argument('--learning_rate'				,		default=5E-5				,		type=__UpperCAmelCase				,		help='The initial learning rate for Adam.'   )
							parser.add_argument(
							    '--lr_scheduler'				,		default='linear'				,		choices=__UpperCAmelCase				,		metavar=__UpperCAmelCase				,		type=__UpperCAmelCase				,		help='Learning rate scheduler'				,		)
							parser.add_argument('--weight_decay'				,		default=0.0				,		type=__UpperCAmelCase				,		help='Weight decay if we apply some.'   )
							parser.add_argument('--adam_epsilon'				,		default=1E-8				,		type=__UpperCAmelCase				,		help='Epsilon for Adam optimizer.'   )
							parser.add_argument('--warmup_steps'				,		default=0				,		type=__UpperCAmelCase				,		help='Linear warmup over warmup_steps.'   )
							parser.add_argument('--num_workers'				,		default=4				,		type=__UpperCAmelCase				,		help='kwarg passed to DataLoader'   )
							parser.add_argument('--num_train_epochs'				,		dest='max_epochs'				,		default=3				,		type=__UpperCAmelCase   )
							parser.add_argument('--train_batch_size'				,		default=3_2				,		type=__UpperCAmelCase   )
							parser.add_argument('--eval_batch_size'				,		default=3_2				,		type=__UpperCAmelCase   )
							parser.add_argument('--adafactor'				,		action='store_true'   )





class   lowerCamelCase      (pl.Callback			):







		'''simple docstring'''



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase   )	->							int:


							if (
							    trainer.is_global_zero and trainer.global_rank == 0
							):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
												pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.





class   lowerCamelCase      (pl.Callback			):







		'''simple docstring'''



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Union[str, Any]:


							for name, param in pl_module.model.rag.named_parameters():
												if param.grad is None:
																	print(__UpperCAmelCase   )





class   lowerCamelCase      (pl.Callback			):







		'''simple docstring'''



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase   )	->							int:
							UpperCAmelCase_    :      Tuple            =						trainer.lr_schedulers[0]['scheduler']
							UpperCAmelCase_    :      Optional[Any]            =						{f"lr_group_{i}": lr for i, lr in enumerate(lr_scheduler.get_lr()   )}
							pl_module.logger.log_metrics(__UpperCAmelCase   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Union[str, Any]:
							rank_zero_info('***** Validation results *****'   )
							UpperCAmelCase_    :      Dict            =						trainer.callback_metrics
							# Log results
							for key in sorted(__UpperCAmelCase   ):
												if key not in ["log", "progress_bar"]:
																	rank_zero_info('{} = {}\n'.format(__UpperCAmelCase				,		str(metrics[key]   )   )   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase   )	->							Optional[int]:
							rank_zero_info('***** Test results *****'   )
							UpperCAmelCase_    :      Optional[int]            =						trainer.callback_metrics
							# Log and save results to file
							UpperCAmelCase_    :      Optional[int]            =						os.path.join(pl_module.hparams.output_dir				,		'test_results.txt'   )
							with open(__UpperCAmelCase				,		'w'   ) as writer:
												for key in sorted(__UpperCAmelCase   ):
																	if key not in ["log", "progress_bar"]:
																						rank_zero_info('{} = {}\n'.format(__UpperCAmelCase				,		str(metrics[key]   )   )   )
																						writer.write('{} = {}\n'.format(__UpperCAmelCase				,		str(metrics[key]   )   )   )





def      lowercase__      (     __snake_case    :							Union[str, Any]     ,       __snake_case    :							int  ):


					'''simple docstring'''


					parser.add_argument(
					    '--output_dir'     ,       default=str(Path(__snake_case  ).parent / 'test_run' / 'model_checkpoints'  )     ,       type=__snake_case     ,       help='The output directory where the model predictions and checkpoints will be written.'     ,       )
					parser.add_argument(
					    '--fp16'     ,       action='store_true'     ,       help='Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'     ,       )

					parser.add_argument(
					    '--fp16_opt_level'     ,       type=__snake_case     ,       default='O2'     ,       help=(
					        'For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'
					        'See details at https://nvidia.github.io/apex/amp.html'
					    )     ,       )
					parser.add_argument('--n_tpu_cores'     ,       dest='tpu_cores'     ,       type=__snake_case  )
					parser.add_argument('--max_grad_norm'     ,       dest='gradient_clip_val'     ,       default=1.0     ,       type=__snake_case     ,       help='Max gradient norm'  )
					parser.add_argument('--do_train'     ,       action='store_true'     ,       help='Whether to run training.'  )
					parser.add_argument('--do_predict'     ,       action='store_true'     ,       help='Whether to run predictions on the test set.'  )
					parser.add_argument(
					    '--gradient_accumulation_steps'     ,       dest='accumulate_grad_batches'     ,       type=__snake_case     ,       default=1     ,       help='Number of updates steps to accumulate before performing a backward/update pass.'     ,       )
					parser.add_argument('--seed'     ,       type=__snake_case     ,       default=42     ,       help='random seed for initialization'  )
					parser.add_argument(
					    '--data_dir'     ,       default=str(Path(__snake_case  ).parent / 'test_run' / 'dummy-train-data'  )     ,       type=__snake_case     ,       help='The input data dir. Should contain the training files for the CoNLL-2003 NER task.'     ,       )






def      lowercase__      (     __snake_case    :							BaseTransformer     ,       __snake_case    :							argparse.Namespace     ,       __snake_case    :							Tuple=None     ,       __snake_case    :							Any=True     ,       __snake_case    :							List[str]=[]     ,       __snake_case    :							List[Any]=None     ,       __snake_case    :							Union[str, Any]=None     ,       **__snake_case    :							Optional[Any]     ,       ):


					'''simple docstring'''


					pl.seed_everything(args.seed  )

					# init model
					UpperCAmelCase_    :      Optional[Any]            =						Path(model.hparams.output_dir  )
					odir.mkdir(exist_ok=__snake_case  )

					# add custom checkpoints
					if checkpoint_callback is None:
										UpperCAmelCase_    :      List[Any]            =						pl.callbacks.ModelCheckpoint(
										    filepath=args.output_dir     ,       prefix='checkpoint'     ,       monitor='val_loss'     ,       mode='min'     ,       save_top_k=1  )
					if early_stopping_callback:
										extra_callbacks.append(__snake_case  )
					if logging_callback is None:
										UpperCAmelCase_    :      int            =						LoggingCallback()

					UpperCAmelCase_    :      Optional[int]            =						{}

					if args.fpaa:
										UpperCAmelCase_    :      Tuple            =						16

					if args.gpus > 1:
										UpperCAmelCase_    :      Union[str, Any]            =						'auto'
										UpperCAmelCase_    :      str            =						'ddp'

					UpperCAmelCase_    :      List[str]            =						args.accumulate_grad_batches
					UpperCAmelCase_    :      str            =						None
					UpperCAmelCase_    :      List[str]            =						'auto'

					UpperCAmelCase_    :      List[str]            =						pl.Trainer.from_argparse_args(
					    __snake_case     ,       weights_summary=__snake_case     ,       callbacks=[logging_callback] + extra_callbacks + [InitCallback()] + [checkpoint_callback]     ,       logger=__snake_case     ,       val_check_interval=1     ,       num_sanity_val_steps=2     ,       **__snake_case     ,       )

					if args.do_train:
										trainer.fit(__snake_case  )

					else:
										print('RAG modeling tests with new set functions successfuly executed!'  )
					return trainer














import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


UpperCAmelCase__				   =		{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__				   =		{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase__				   =		{
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

UpperCAmelCase__				   =		{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      __snake_case       =  VOCAB_FILES_NAMES
      __snake_case       =  PRETRAINED_VOCAB_FILES_MAP
      __snake_case       =  PRETRAINED_INIT_CONFIGURATION
      __snake_case       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __snake_case       =  ElectraTokenizer







      def __init__(      self      :						Dict ,					__UpperCAmelCase      :						int=None ,					__UpperCAmelCase      :						str=None ,					__UpperCAmelCase      :						Optional[int]=True ,					__UpperCAmelCase      :						str="[UNK]" ,					__UpperCAmelCase      :						Any="[SEP]" ,					__UpperCAmelCase      :						str="[PAD]" ,					__UpperCAmelCase      :						Optional[Any]="[CLS]" ,					__UpperCAmelCase      :						Union[str, Any]="[MASK]" ,					__UpperCAmelCase      :						List[str]=True ,					__UpperCAmelCase      :						Tuple=None ,					**__UpperCAmelCase      :						Optional[int] ,					)    ->str:




       """simple docstring"""


       super().__init__(
           __UpperCAmelCase ,					tokenizer_file=__UpperCAmelCase ,					do_lower_case=__UpperCAmelCase ,					unk_token=__UpperCAmelCase ,					sep_token=__UpperCAmelCase ,					pad_token=__UpperCAmelCase ,					cls_token=__UpperCAmelCase ,					mask_token=__UpperCAmelCase ,					tokenize_chinese_chars=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase ,					**__UpperCAmelCase ,					)

       a		  =      json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
       if (
           normalizer_state.get('''lowercase''' ,					__UpperCAmelCase  ) != do_lower_case
           or normalizer_state.get('''strip_accents''' ,					__UpperCAmelCase  ) != strip_accents
           or normalizer_state.get('''handle_chinese_chars''' ,					__UpperCAmelCase  ) != tokenize_chinese_chars
       ):
        a		  =      getattr(__UpperCAmelCase ,					normalizer_state.pop('''type'''  )  )
        a		  =      do_lower_case
        a		  =      strip_accents
        a		  =      tokenize_chinese_chars
        a		  =      normalizer_class(**__UpperCAmelCase  )

       a		  =      do_lower_case







      def       __lowerCAmelCase					(      self      :						List[Any] ,					__UpperCAmelCase      :						Optional[int] ,					__UpperCAmelCase      :						Tuple=None  )    ->str:




       """simple docstring"""


       a		  =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

       return output







      def       __lowerCAmelCase					(      self      :						Optional[Any] ,					__UpperCAmelCase      :						List[int] ,					__UpperCAmelCase      :						Optional[List[int]] = None  )    ->List[int]:




       """simple docstring"""


       a		  =      [self.sep_token_id]
       a		  =      [self.cls_token_id]
       if token_ids_a is None:
        return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







      def       __lowerCAmelCase					(      self      :						List[str] ,					__UpperCAmelCase      :						str ,					__UpperCAmelCase      :						Optional[str] = None  )    ->Tuple[str]:




       """simple docstring"""


       a		  =      self._tokenizer.model.save(__UpperCAmelCase ,					name=__UpperCAmelCase  )
       return tuple(__UpperCAmelCase  )


