




'''simple docstring'''
import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


A				: Dict					      =			logging.getLogger(__name__)

require_version("""pytorch_lightning>=1.0.4""")

A				: Any					      =			{
    '''base''': AutoModel,
    '''sequence-classification''': AutoModelForSequenceClassification,
    '''question-answering''': AutoModelForQuestionAnswering,
    '''pretraining''': AutoModelForPreTraining,
    '''token-classification''': AutoModelForTokenClassification,
    '''language-modeling''': AutoModelWithLMHead,
    '''summarization''': AutoModelForSeqaSeqLM,
    '''translation''': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
A				: Union[str, Any]					      =			{
    '''linear''': get_linear_schedule_with_warmup,
    '''cosine''': get_cosine_schedule_with_warmup,
    '''cosine_w_restarts''': get_cosine_with_hard_restarts_schedule_with_warmup,
    '''polynomial''': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
A				: Dict					      =			sorted(arg_to_scheduler.keys())
A				: Any					      =			'''{''' + ''', '''.join(arg_to_scheduler_choices) + '''}'''






class 			SCREAMING_SNAKE_CASE(							pl.LightningModule	):

       def __init__(       self   ,						lowerCamelCase__   ,						lowerCamelCase__=None   ,						lowerCamelCase__="base"   ,						lowerCamelCase__=None   ,						lowerCamelCase__=None   ,						lowerCamelCase__=None   ,						**lowerCamelCase__   ,						)       ->   Tuple:

           """simple docstring"""



           super().__init__()
           # TODO: move to self.save_hyperparameters()
           # self.save_hyperparameters()
           # can also expand arguments into trainer signature for easier reading

           self.save_hyperparameters(__UpperCamelCase	)
           __lowercase     							=		0
           __lowercase     							=		Path(self.hparams.output_dir	)
           __lowercase     							=		self.hparams.cache_dir if self.hparams.cache_dir else None
           if config is None:
               __lowercase     							=		AutoConfig.from_pretrained(
                   self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path   ,						**({"""num_labels""": num_labels} if num_labels is not None else {})   ,						cache_dir=__UpperCamelCase   ,						**__UpperCamelCase   ,						)
           else:
               __lowercase     							=		config

           __lowercase     							=		('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
           for p in extra_model_params:
               if getattr(self.hparams   ,						__UpperCamelCase   ,						__UpperCamelCase	):
                   assert hasattr(self.config   ,						__UpperCamelCase	), F'model config doesn\'t have a `{p}` attribute'
                   setattr(self.config   ,						__UpperCamelCase   ,						getattr(self.hparams   ,						__UpperCamelCase	)	)

           if tokenizer is None:
               __lowercase     							=		AutoTokenizer.from_pretrained(
                   self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path   ,						cache_dir=__UpperCamelCase   ,						)
           else:
               __lowercase     							=		tokenizer
           __lowercase     							=		MODEL_MODES[mode]
           if model is None:
               __lowercase     							=		self.model_type.from_pretrained(
                   self.hparams.model_name_or_path   ,						from_tf=bool(""".ckpt""" in self.hparams.model_name_or_path	)   ,						config=self.config   ,						cache_dir=__UpperCamelCase   ,						)
           else:
               __lowercase     							=		model

       def        snake_case__     (       self   ,						*lowerCamelCase__   ,						**lowerCamelCase__	)       ->   Tuple:

           """simple docstring"""



           __lowercase     							=		self.model_type.from_pretrained(*__UpperCamelCase   ,						**__UpperCamelCase	)

       def        snake_case__     (       self	)       ->   Tuple:

           """simple docstring"""



           __lowercase     							=		arg_to_scheduler[self.hparams.lr_scheduler]
           __lowercase     							=		get_schedule_func(
               self.opt   ,						num_warmup_steps=self.hparams.warmup_steps   ,						num_training_steps=self.total_steps()	)
           __lowercase     							=		{'scheduler': scheduler, 'interval': 'step', 'frequency': 1}
           return scheduler

       def        snake_case__     (       self	)       ->   Any:

           """simple docstring"""



           __lowercase     							=		self.model
           __lowercase     							=		['bias', 'LayerNorm.weight']
           __lowercase     							=		[
               {
                   'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay	)],
                   'weight_decay': self.hparams.weight_decay,
               },
               {
                   'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay	)],
                   'weight_decay': 0.0,
               },
           ]
           if self.hparams.adafactor:
               __lowercase     							=		Adafactor(
                   __UpperCamelCase   ,						lr=self.hparams.learning_rate   ,						scale_parameter=__UpperCamelCase   ,						relative_step=__UpperCamelCase	)

           else:
               __lowercase     							=		AdamW(
                   __UpperCamelCase   ,						lr=self.hparams.learning_rate   ,						eps=self.hparams.adam_epsilon	)
           __lowercase     							=		optimizer

           __lowercase     							=		self.get_lr_scheduler()

           return [optimizer], [scheduler]

       def        snake_case__     (       self   ,						lowerCamelCase__   ,						lowerCamelCase__	)       ->   Optional[int]:

           """simple docstring"""



           return self.validation_step(__UpperCamelCase   ,						__UpperCamelCase	)

       def        snake_case__     (       self   ,						lowerCamelCase__	)       ->   Tuple:

           """simple docstring"""



           return self.validation_end(__UpperCamelCase	)

       def        snake_case__     (       self	)       ->   int:

           """simple docstring"""



           __lowercase     							=		max(1   ,						self.hparams.gpus	)  # TODO: consider num_tpu_cores
           __lowercase     							=		self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
           return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs

       def        snake_case__     (       self   ,						lowerCamelCase__	)       ->   int:

           """simple docstring"""



           if mode == "test":
               __lowercase     							=		len(self.test_dataloader().dataset	)
           else:
               __lowercase     							=		self.get_dataloader("""train"""   ,						self.hparams.train_batch_size   ,						shuffle=__UpperCamelCase	)
               __lowercase     							=		len(self.train_dataloader().dataset	)

       def        snake_case__     (       self   ,						lowerCamelCase__   ,						lowerCamelCase__   ,						lowerCamelCase__ = False	)       ->   Union[str, Any]:

           """simple docstring"""



           raise NotImplementedError("""You must implement this for your task"""	)

       def        snake_case__     (       self	)       ->   Tuple:

           """simple docstring"""



           return self.train_loader

       def        snake_case__     (       self	)       ->   Dict:

           """simple docstring"""



           return self.get_dataloader("""dev"""   ,						self.hparams.eval_batch_size   ,						shuffle=__UpperCamelCase	)

       def        snake_case__     (       self	)       ->   List[str]:

           """simple docstring"""



           return self.get_dataloader("""test"""   ,						self.hparams.eval_batch_size   ,						shuffle=__UpperCamelCase	)

       def        snake_case__     (       self   ,						lowerCamelCase__	)       ->   int:

           """simple docstring"""



           return os.path.join(
               self.hparams.data_dir   ,						"""cached_{}_{}_{}""".format(
                   __UpperCamelCase   ,						list(filter(__UpperCamelCase   ,						self.hparams.model_name_or_path.split("""/"""	)	)	).pop()   ,						str(self.hparams.max_seq_length	)   ,						)   ,						)

       @pl.utilities.rank_zero_only
       def        snake_case__     (       self   ,						lowerCamelCase__	)       ->   None:

           """simple docstring"""



           __lowercase     							=		self.output_dir.joinpath("""best_tfmr"""	)
           __lowercase     							=		self.step_count
           self.model.save_pretrained(__UpperCamelCase	)
           self.tokenizer.save_pretrained(__UpperCamelCase	)




       @staticmethod
       def        snake_case__     (       lowerCamelCase__   ,						lowerCamelCase__	)       ->   Any:

           """simple docstring"""



           parser.add_argument(
               """--model_name_or_path"""   ,						default=__UpperCamelCase   ,						type=__UpperCamelCase   ,						required=__UpperCamelCase   ,						help="""Path to pretrained model or model identifier from huggingface.co/models"""   ,						)
           parser.add_argument(
               """--config_name"""   ,						default=""""""   ,						type=__UpperCamelCase   ,						help="""Pretrained config name or path if not the same as model_name"""	)
           parser.add_argument(
               """--tokenizer_name"""   ,						default=__UpperCamelCase   ,						type=__UpperCamelCase   ,						help="""Pretrained tokenizer name or path if not the same as model_name"""   ,						)
           parser.add_argument(
               """--cache_dir"""   ,						default=""""""   ,						type=__UpperCamelCase   ,						help="""Where do you want to store the pre-trained models downloaded from huggingface.co"""   ,						)
           parser.add_argument(
               """--encoder_layerdrop"""   ,						type=__UpperCamelCase   ,						help="""Encoder layer dropout probability (Optional). Goes into model.config"""   ,						)
           parser.add_argument(
               """--decoder_layerdrop"""   ,						type=__UpperCamelCase   ,						help="""Decoder layer dropout probability (Optional). Goes into model.config"""   ,						)
           parser.add_argument(
               """--dropout"""   ,						type=__UpperCamelCase   ,						help="""Dropout probability (Optional). Goes into model.config"""   ,						)
           parser.add_argument(
               """--attention_dropout"""   ,						type=__UpperCamelCase   ,						help="""Attention dropout probability (Optional). Goes into model.config"""   ,						)
           parser.add_argument("""--learning_rate"""   ,						default=5E-5   ,						type=__UpperCamelCase   ,						help="""The initial learning rate for Adam."""	)
           parser.add_argument(
               """--lr_scheduler"""   ,						default="""linear"""   ,						choices=__UpperCamelCase   ,						metavar=__UpperCamelCase   ,						type=__UpperCamelCase   ,						help="""Learning rate scheduler"""   ,						)
           parser.add_argument("""--weight_decay"""   ,						default=0.0   ,						type=__UpperCamelCase   ,						help="""Weight decay if we apply some."""	)
           parser.add_argument("""--adam_epsilon"""   ,						default=1E-8   ,						type=__UpperCamelCase   ,						help="""Epsilon for Adam optimizer."""	)
           parser.add_argument("""--warmup_steps"""   ,						default=0   ,						type=__UpperCamelCase   ,						help="""Linear warmup over warmup_steps."""	)
           parser.add_argument("""--num_workers"""   ,						default=4   ,						type=__UpperCamelCase   ,						help="""kwarg passed to DataLoader"""	)
           parser.add_argument("""--num_train_epochs"""   ,						dest="""max_epochs"""   ,						default=3   ,						type=__UpperCamelCase	)
           parser.add_argument("""--train_batch_size"""   ,						default=32   ,						type=__UpperCamelCase	)
           parser.add_argument("""--eval_batch_size"""   ,						default=32   ,						type=__UpperCamelCase	)
           parser.add_argument("""--adafactor"""   ,						action="""store_true"""	)






class 			SCREAMING_SNAKE_CASE(							pl.Callback	):

       def        snake_case__     (       self   ,						lowerCamelCase__   ,						lowerCamelCase__	)       ->   List[Any]:

           """simple docstring"""



           __lowercase     							=		trainer.lr_schedulers[0]['scheduler']
           __lowercase     							=		{F'lr_group_{i}': lr for i, lr in enumerate(lr_scheduler.get_lr()	)}
           pl_module.logger.log_metrics(__UpperCamelCase	)

       def        snake_case__     (       self   ,						lowerCamelCase__   ,						lowerCamelCase__	)       ->   Optional[int]:

           """simple docstring"""



           rank_zero_info("""***** Validation results *****"""	)
           __lowercase     							=		trainer.callback_metrics
           # Log results
           for key in sorted(__UpperCamelCase	):
               if key not in ["log", "progress_bar"]:
                   rank_zero_info("""{} = {}\n""".format(__UpperCamelCase   ,						str(metrics[key]	)	)	)




       def        snake_case__     (       self   ,						lowerCamelCase__   ,						lowerCamelCase__	)       ->   List[Any]:

           """simple docstring"""



           rank_zero_info("""***** Test results *****"""	)
           __lowercase     							=		trainer.callback_metrics
           # Log and save results to file
           __lowercase     							=		os.path.join(pl_module.hparams.output_dir   ,						"""test_results.txt"""	)
           with open(__UpperCamelCase   ,						"""w"""	) as writer:
               for key in sorted(__UpperCamelCase	):
                   if key not in ["log", "progress_bar"]:
                       rank_zero_info("""{} = {}\n""".format(__UpperCamelCase   ,						str(metrics[key]	)	)	)
                       writer.write("""{} = {}\n""".format(__UpperCamelCase   ,						str(metrics[key]	)	)	)

def 							snake_case_    (  a__						:      str       ,a__						:      Optional[int]     ):



    """simple docstring"""




    parser.add_argument(
        """--output_dir"""       ,default=A__       ,type=A__       ,required=A__       ,help="""The output directory where the model predictions and checkpoints will be written."""       ,)
    parser.add_argument(
        """--fp16"""       ,action="""store_true"""       ,help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""       ,)

    parser.add_argument(
        """--fp16_opt_level"""       ,type=A__       ,default="""O2"""       ,help=(
            """For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\']."""
            """See details at https://nvidia.github.io/apex/amp.html"""
        )       ,)
    parser.add_argument("""--n_tpu_cores"""       ,dest="""tpu_cores"""       ,type=A__     )
    parser.add_argument("""--max_grad_norm"""       ,dest="""gradient_clip_val"""       ,default=1.0       ,type=A__       ,help="""Max gradient norm"""     )
    parser.add_argument("""--do_train"""       ,action="""store_true"""       ,help="""Whether to run training."""     )
    parser.add_argument("""--do_predict"""       ,action="""store_true"""       ,help="""Whether to run predictions on the test set."""     )
    parser.add_argument(
        """--gradient_accumulation_steps"""       ,dest="""accumulate_grad_batches"""       ,type=A__       ,default=1       ,help="""Number of updates steps to accumulate before performing a backward/update pass."""       ,)
    parser.add_argument("""--seed"""       ,type=A__       ,default=42       ,help="""random seed for initialization"""     )
    parser.add_argument(
        """--data_dir"""       ,default=A__       ,type=A__       ,required=A__       ,help="""The input data dir. Should contain the training files for the CoNLL-2003 NER task."""       ,)



def 							snake_case_    (  a__						:      int       ,a__						:      List[str]       ,a__						:      Optional[int]=None       ,a__						:      List[Any]=True       ,a__						:      Union[str, Any]=[]       ,a__						:      Optional[int]=None       ,a__						:      Optional[Any]=None       ,**a__						:      Optional[int]       ,):



    """simple docstring"""




    pl.seed_everything(args.seed     )

    # init model
    __lowercase     							=		Path(model.hparams.output_dir     )
    odir.mkdir(exist_ok=A__     )

    # add custom checkpoints
    if checkpoint_callback is None:
        __lowercase     							=		pl.callbacks.ModelCheckpoint(
            filepath=args.output_dir       ,prefix="""checkpoint"""       ,monitor="""val_loss"""       ,mode="""min"""       ,save_top_k=1     )
    if early_stopping_callback:
        extra_callbacks.append(A__     )
    if logging_callback is None:
        __lowercase     							=		LoggingCallback()

    __lowercase     							=		{}

    # TODO: remove with PyTorch 1.6 since pl uses native amp
    if args.fpaa:
        __lowercase     							=		16
        __lowercase     							=		args.fpaa_opt_level

    if args.gpus > 1:
        __lowercase     							=		'ddp'

    __lowercase     							=		args.accumulate_grad_batches
    __lowercase     							=		extra_train_kwargs.get("""accelerator"""       ,A__     )
    __lowercase     							=		extra_train_kwargs.get("""profiler"""       ,A__     )

    __lowercase     							=		pl.Trainer.from_argparse_args(
        A__       ,weights_summary=A__       ,callbacks=[logging_callback] + extra_callbacks       ,logger=A__       ,checkpoint_callback=A__       ,**A__       ,)

    if args.do_train:
        trainer.fit(A__     )

    return trainer





from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


lowerCAmelCase__	:  List[Any]						=  logging.get_logger(__name__)

lowerCAmelCase__	:  str						=  {
    '''google/bigbird-pegasus-large-arxiv''': (
        '''https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'''
    ),
    '''google/bigbird-pegasus-large-pubmed''': (
        '''https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'''
    ),
    '''google/bigbird-pegasus-large-bigpatent''': (
        '''https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'''
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}





class 					__snake_case    (     _lowerCamelCase      ):
					__lowerCamelCase			   =  """bigbird_pegasus"""
					__lowerCamelCase			   =  ["""past_key_values"""]
					__lowerCamelCase			   =  {
					    """num_attention_heads""": """encoder_attention_heads""",
					    """hidden_size""": """d_model""",
					    """attention_probs_dropout_prob""": """attention_dropout""",
					}




					def __init__(  self       ,			__UpperCamelCase=96103       ,			__UpperCamelCase=4096       ,			__UpperCamelCase=16       ,			__UpperCamelCase=4096       ,			__UpperCamelCase=16       ,			__UpperCamelCase=16       ,			__UpperCamelCase=4096       ,			__UpperCamelCase=16       ,			__UpperCamelCase=0.0       ,			__UpperCamelCase=0.0       ,			__UpperCamelCase=True       ,			__UpperCamelCase=True       ,			__UpperCamelCase="gelu_new"       ,			__UpperCamelCase=1024       ,			__UpperCamelCase=0.1       ,			__UpperCamelCase=0.0       ,			__UpperCamelCase=0.0       ,			__UpperCamelCase=0.0_2       ,			__UpperCamelCase=2       ,			__UpperCamelCase=0.0       ,			__UpperCamelCase=True       ,			__UpperCamelCase=0       ,			__UpperCamelCase=2       ,			__UpperCamelCase=1       ,			__UpperCamelCase="block_sparse"       ,			__UpperCamelCase=64       ,			__UpperCamelCase=3       ,			__UpperCamelCase=False       ,			**__UpperCamelCase       ,			)	->    Tuple:



									'''simple docstring'''







									snake_case__   :      Union[str, Any]   				= vocab_size
									snake_case__   :      Optional[Any]   				= max_position_embeddings
									snake_case__   :      Tuple   				= d_model
									snake_case__   :      str   				= encoder_ffn_dim
									snake_case__   :      List[str]   				= encoder_layers
									snake_case__   :      Any   				= encoder_attention_heads
									snake_case__   :      Any   				= decoder_ffn_dim
									snake_case__   :      List[Any]   				= decoder_layers
									snake_case__   :      Dict   				= decoder_attention_heads
									snake_case__   :      List[Any]   				= dropout
									snake_case__   :      List[Any]   				= attention_dropout
									snake_case__   :      List[Any]   				= activation_dropout
									snake_case__   :      str   				= activation_function
									snake_case__   :      Optional[Any]   				= init_std
									snake_case__   :      Optional[int]   				= encoder_layerdrop
									snake_case__   :      List[Any]   				= decoder_layerdrop
									snake_case__   :      Tuple   				= classifier_dropout
									snake_case__   :      Optional[Any]   				= use_cache
									snake_case__   :      List[Any]   				= encoder_layers
									snake_case__   :      Union[str, Any]   				= scale_embedding  # scale factor will be sqrt(d_model) if True

									# extra config
									snake_case__   :      Optional[Any]   				= attention_type
									snake_case__   :      Optional[int]   				= block_size
									snake_case__   :      Optional[Any]   				= num_random_blocks
									snake_case__   :      List[str]   				= use_bias

									super().__init__(
									    pad_token_id=__UpperCamelCase       ,			bos_token_id=__UpperCamelCase       ,			eos_token_id=__UpperCamelCase       ,			is_encoder_decoder=__UpperCamelCase       ,			decoder_start_token_id=__UpperCamelCase       ,			**__UpperCamelCase       ,			)





class 					__snake_case    (     _lowerCamelCase      ):




					@property
					def      __a     (  self						)	->    Mapping[str, Mapping[int, str]]:



									'''simple docstring'''







									if self.task in ["default", "seq2seq-lm"]:
													snake_case__   :      str   				= OrderedDict(
													    [
													        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
													        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
													    ]						)

													if self.use_past:
																	snake_case__   :      Optional[int]   				= {0: 'batch'}
																	snake_case__   :      str   				= {0: 'batch', 1: 'past_decoder_sequence + sequence'}
													else:
																	snake_case__   :      Any   				= {0: 'batch', 1: 'decoder_sequence'}
																	snake_case__   :      List[str]   				= {0: 'batch', 1: 'decoder_sequence'}

													if self.use_past:
																	self.fill_with_past_key_values_(__UpperCamelCase       ,			direction='inputs'						)
									elif self.task == "causal-lm":
													# TODO: figure this case out.
													snake_case__   :      Any   				= OrderedDict(
													    [
													        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
													        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
													    ]						)
													if self.use_past:
																	snake_case__				,    snake_case__   :      Optional[Any]   				= self.num_layers
																	for i in range(__UpperCamelCase						):
																					snake_case__   :      str   				= {0: 'batch', 2: 'past_sequence + sequence'}
																					snake_case__   :      Optional[int]   				= {0: 'batch', 2: 'past_sequence + sequence'}
									else:
													snake_case__   :      Union[str, Any]   				= OrderedDict(
													    [
													        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
													        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
													        ('decoder_input_ids', {0: 'batch', 1: 'decoder_sequence'}),
													        ('decoder_attention_mask', {0: 'batch', 1: 'decoder_sequence'}),
													    ]						)

									return common_inputs




					@property
					def      __a     (  self						)	->    Mapping[str, Mapping[int, str]]:



									'''simple docstring'''







									if self.task in ["default", "seq2seq-lm"]:
													snake_case__   :      int   				= super().outputs
									else:
													snake_case__   :      Dict   				= super(__UpperCamelCase       ,			self						).outputs
													if self.use_past:
																	snake_case__				,    snake_case__   :      List[Any]   				= self.num_layers
																	for i in range(__UpperCamelCase						):
																					snake_case__   :      List[Any]   				= {0: 'batch', 2: 'past_sequence + sequence'}
																					snake_case__   :      List[Any]   				= {0: 'batch', 2: 'past_sequence + sequence'}
									return common_outputs




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase = -1       ,			__UpperCamelCase = -1       ,			__UpperCamelCase = False       ,			__UpperCamelCase = None       ,			)	->    Mapping[str, Any]:



									'''simple docstring'''







									snake_case__   :      List[str]   				= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    __UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase						)

									# Generate decoder inputs
									snake_case__   :      Optional[Any]   				= seq_length if not self.use_past else 1
									snake_case__   :      int   				= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    __UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase						)
									snake_case__   :      str   				= {F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
									snake_case__   :      Any   				= dict(**__UpperCamelCase       ,			**__UpperCamelCase						)

									if self.use_past:
													if not is_torch_available():
																	raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'						)
													else:
																	import torch
													snake_case__				,    snake_case__   :      int   				= common_inputs['input_ids'].shape
													snake_case__   :      int   				= common_inputs['decoder_input_ids'].shape[1]
													snake_case__				,    snake_case__   :      List[Any]   				= self.num_attention_heads
													snake_case__   :      Any   				= (
													    batch,
													    num_encoder_attention_heads,
													    encoder_seq_length,
													    self._config.hidden_size // num_encoder_attention_heads,
													)
													snake_case__   :      Optional[int]   				= decoder_seq_length + 3
													snake_case__   :      Union[str, Any]   				= (
													    batch,
													    num_decoder_attention_heads,
													    decoder_past_length,
													    self._config.hidden_size // num_decoder_attention_heads,
													)

													snake_case__   :      Union[str, Any]   				= torch.cat(
													    [common_inputs['decoder_attention_mask'], torch.ones(__UpperCamelCase       ,			__UpperCamelCase						)]       ,			dim=1						)

													snake_case__   :      Optional[Any]   				= []
													# If the number of encoder and decoder layers are present in the model configuration, both are considered
													snake_case__				,    snake_case__   :      str   				= self.num_layers
													snake_case__   :      str   				= min(__UpperCamelCase       ,			__UpperCamelCase						)
													snake_case__   :      Optional[int]   				= max(__UpperCamelCase       ,			__UpperCamelCase						) - min_num_layers
													snake_case__   :      int   				= 'encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

													for _ in range(__UpperCamelCase						):
																	common_inputs["past_key_values"].append(
																	    (
																	        torch.zeros(__UpperCamelCase						),
																	        torch.zeros(__UpperCamelCase						),
																	        torch.zeros(__UpperCamelCase						),
																	        torch.zeros(__UpperCamelCase						),
																	    )						)
													# TODO: test this.
													snake_case__   :      Dict   				= encoder_shape if remaining_side_name == 'encoder' else decoder_shape
													for _ in range(__UpperCamelCase       ,			__UpperCamelCase						):
																	common_inputs["past_key_values"].append((torch.zeros(__UpperCamelCase						), torch.zeros(__UpperCamelCase						))						)
									return common_inputs




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase = -1       ,			__UpperCamelCase = -1       ,			__UpperCamelCase = False       ,			__UpperCamelCase = None       ,			)	->    Mapping[str, Any]:



									'''simple docstring'''







									snake_case__   :      List[str]   				= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    __UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase						)

									if self.use_past:
													if not is_torch_available():
																	raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'						)
													else:
																	import torch
													snake_case__				,    snake_case__   :      Optional[int]   				= common_inputs['input_ids'].shape
													# Not using the same length for past_key_values
													snake_case__   :      str   				= seqlen + 2
													snake_case__				,    snake_case__   :      List[str]   				= self.num_layers
													snake_case__				,    snake_case__   :      str   				= self.num_attention_heads
													snake_case__   :      Tuple   				= (
													    batch,
													    num_encoder_attention_heads,
													    past_key_values_length,
													    self._config.hidden_size // num_encoder_attention_heads,
													)

													snake_case__   :      Dict   				= common_inputs['attention_mask'].dtype
													snake_case__   :      Optional[Any]   				= torch.cat(
													    [common_inputs['attention_mask'], torch.ones(__UpperCamelCase       ,			__UpperCamelCase       ,			dtype=__UpperCamelCase						)]       ,			dim=1						)
													snake_case__   :      int   				= [
													    (torch.zeros(__UpperCamelCase						), torch.zeros(__UpperCamelCase						)) for _ in range(__UpperCamelCase						)
													]
									return common_inputs




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase = -1       ,			__UpperCamelCase = -1       ,			__UpperCamelCase = False       ,			__UpperCamelCase = None       ,			)	->    Mapping[str, Any]:



									'''simple docstring'''







									snake_case__   :      Optional[int]   				= compute_effective_axis_dimension(
									    __UpperCamelCase       ,			fixed_dimension=OnnxConfig.default_fixed_batch       ,			num_token_to_add=0						)

									# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
									snake_case__   :      str   				= tokenizer.num_special_tokens_to_add(__UpperCamelCase						)
									snake_case__   :      List[str]   				= compute_effective_axis_dimension(
									    __UpperCamelCase       ,			fixed_dimension=OnnxConfig.default_fixed_sequence       ,			num_token_to_add=__UpperCamelCase						)

									# Generate dummy inputs according to compute batch and sequence
									snake_case__   :      Optional[Any]   				= [' '.join([tokenizer.unk_token]						) * seq_length] * batch_size
									snake_case__   :      Optional[Any]   				= dict(tokenizer(__UpperCamelCase       ,			return_tensors=__UpperCamelCase						)						)
									return common_inputs




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase = -1       ,			__UpperCamelCase = -1       ,			__UpperCamelCase = False       ,			__UpperCamelCase = None       ,			)	->    Mapping[str, Any]:



									'''simple docstring'''







									if self.task in ["default", "seq2seq-lm"]:
													snake_case__   :      Dict   				= self._generate_dummy_inputs_for_default_and_seqaseq_lm(
													    __UpperCamelCase       ,			batch_size=__UpperCamelCase       ,			seq_length=__UpperCamelCase       ,			is_pair=__UpperCamelCase       ,			framework=__UpperCamelCase						)

									elif self.task == "causal-lm":
													snake_case__   :      Union[str, Any]   				= self._generate_dummy_inputs_for_causal_lm(
													    __UpperCamelCase       ,			batch_size=__UpperCamelCase       ,			seq_length=__UpperCamelCase       ,			is_pair=__UpperCamelCase       ,			framework=__UpperCamelCase						)
									else:
													snake_case__   :      Any   				= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
													    __UpperCamelCase       ,			batch_size=__UpperCamelCase       ,			seq_length=__UpperCamelCase       ,			is_pair=__UpperCamelCase       ,			framework=__UpperCamelCase						)

									return common_inputs




					def      __a     (  self       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase						)	->    Dict:



									'''simple docstring'''







									if self.task in ["default", "seq2seq-lm"]:
													snake_case__   :      Optional[int]   				= super()._flatten_past_key_values_(__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase						)
									else:
													snake_case__   :      Optional[Any]   				= super(__UpperCamelCase       ,			self						)._flatten_past_key_values_(
													    __UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase       ,			__UpperCamelCase						)


