

"""simple docstring"""





import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


_a					 =				logging.getLogger(__name__)

require_version('pytorch_lightning>=1.0.4')

_a					 =				{
    'base': AutoModel,
    'sequence-classification': AutoModelForSequenceClassification,
    'question-answering': AutoModelForQuestionAnswering,
    'pretraining': AutoModelForPreTraining,
    'token-classification': AutoModelForTokenClassification,
    'language-modeling': AutoModelWithLMHead,
    'summarization': AutoModelForSeqaSeqLM,
    'translation': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
_a					 =				{
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
_a					 =				sorted(arg_to_scheduler.keys())
_a					 =				'{' + ', '.join(arg_to_scheduler_choices) + '}'




class    _lowerCAmelCase    (      pl.LightningModule  ):







 """simple docstring"""







 def __init__(     self				:     Dict,   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     List[str]=None,   UpperCAmelCase__				:     str="base",   UpperCAmelCase__				:     Any=None,   UpperCAmelCase__				:     Optional[Any]=None,   UpperCAmelCase__				:     List[str]=None,   **UpperCAmelCase__				:     Any,   ):
    super().__init__()
    # TODO: move to self.save_hyperparameters()
    # self.save_hyperparameters()
    # can also expand arguments into trainer signature for easier reading

    self.save_hyperparameters(_SCREAMING_SNAKE_CASE      )
    __lowercase								=		0
    __lowercase								=		Path(self.hparams.output_dir      )
    __lowercase								=		self.hparams.cache_dir if self.hparams.cache_dir else None
    if config is None:
       __lowercase								=		AutoConfig.from_pretrained(
           self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path,   **({"num_labels": num_labels} if num_labels is not None else {}),   cache_dir=_SCREAMING_SNAKE_CASE,   **_SCREAMING_SNAKE_CASE,   )
    else:
       __lowercase								=		config

    __lowercase								=		("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
    for p in extra_model_params:
       if getattr(self.hparams,   _SCREAMING_SNAKE_CASE,   _SCREAMING_SNAKE_CASE      ):
          assert hasattr(self.config,   _SCREAMING_SNAKE_CASE      ), F"""model config doesn\'t have a `{p}` attribute"""
          setattr(self.config,   _SCREAMING_SNAKE_CASE,   getattr(self.hparams,   _SCREAMING_SNAKE_CASE      )      )

    if tokenizer is None:
       __lowercase								=		AutoTokenizer.from_pretrained(
           self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path,   cache_dir=_SCREAMING_SNAKE_CASE,   )
    else:
       __lowercase								=		tokenizer
    __lowercase								=		MODEL_MODES[mode]
    if model is None:
       __lowercase								=		self.model_type.from_pretrained(
           self.hparams.model_name_or_path,   from_tf=bool(".ckpt" in self.hparams.model_name_or_path      ),   config=self.config,   cache_dir=_SCREAMING_SNAKE_CASE,   )
    else:
       __lowercase								=		model


 def    _lowercase					(     self				:     Union[str, Any],   *UpperCAmelCase__				:     str,   **UpperCAmelCase__				:     Tuple      ):
    __lowercase								=		self.model_type.from_pretrained(*_SCREAMING_SNAKE_CASE,   **_SCREAMING_SNAKE_CASE      )


 def    _lowercase					(     self				:     Optional[int]      ):
    __lowercase								=		arg_to_scheduler[self.hparams.lr_scheduler]
    __lowercase								=		get_schedule_func(
        self.opt,   num_warmup_steps=self.hparams.warmup_steps,   num_training_steps=self.total_steps()      )
    __lowercase								=		{"scheduler": scheduler, "interval": "step", "frequency": 1}
    return scheduler


 def    _lowercase					(     self				:     Any      ):
    __lowercase								=		self.model
    __lowercase								=		["bias", "LayerNorm.weight"]
    __lowercase								=		[
        {
            "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay      )],
            "weight_decay": self.hparams.weight_decay,
        },
        {
            "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay      )],
            "weight_decay": 0.0,
        },
    ]
    if self.hparams.adafactor:
       __lowercase								=		Adafactor(
           _SCREAMING_SNAKE_CASE,   lr=self.hparams.learning_rate,   scale_parameter=_SCREAMING_SNAKE_CASE,   relative_step=_SCREAMING_SNAKE_CASE      )

    else:
       __lowercase								=		AdamW(
           _SCREAMING_SNAKE_CASE,   lr=self.hparams.learning_rate,   eps=self.hparams.adam_epsilon      )
    __lowercase								=		optimizer

    __lowercase								=		self.get_lr_scheduler()

    return [optimizer], [scheduler]


 def    _lowercase					(     self				:     Any,   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     Any      ):
    return self.validation_step(_SCREAMING_SNAKE_CASE,   _SCREAMING_SNAKE_CASE      )


 def    _lowercase					(     self				:     Optional[int],   UpperCAmelCase__				:     Optional[int]      ):
    return self.validation_end(_SCREAMING_SNAKE_CASE      )


 def    _lowercase					(     self				:     Optional[int]      ):
    __lowercase								=		max(1,   self.hparams.gpus      )  # TODO: consider num_tpu_cores
    __lowercase								=		self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
    return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs


 def    _lowercase					(     self				:     Any,   UpperCAmelCase__				:     Any      ):
    if mode == "test":
       __lowercase								=		len(self.test_dataloader().dataset      )
    else:
       __lowercase								=		self.get_dataloader("train",   self.hparams.train_batch_size,   shuffle=_SCREAMING_SNAKE_CASE      )
       __lowercase								=		len(self.train_dataloader().dataset      )


 def    _lowercase					(     self				:     str,   UpperCAmelCase__				:     int,   UpperCAmelCase__				:     Tuple,   UpperCAmelCase__				:     List[str] = False      ):
    raise NotImplementedError("You must implement this for your task"      )


 def    _lowercase					(     self				:     int      ):
    return self.train_loader


 def    _lowercase					(     self				:     List[str]      ):
    return self.get_dataloader("dev",   self.hparams.eval_batch_size,   shuffle=_SCREAMING_SNAKE_CASE      )


 def    _lowercase					(     self				:     Dict      ):
    return self.get_dataloader("test",   self.hparams.eval_batch_size,   shuffle=_SCREAMING_SNAKE_CASE      )


 def    _lowercase					(     self				:     Union[str, Any],   UpperCAmelCase__				:     Optional[int]      ):
    return os.path.join(
        self.hparams.data_dir,   "cached_{}_{}_{}".format(
            _SCREAMING_SNAKE_CASE,   list(filter(_SCREAMING_SNAKE_CASE,   self.hparams.model_name_or_path.split("/"      )      )      ).pop(),   str(self.hparams.max_seq_length      ),   ),   )


 @pl.utilities.rank_zero_only
 def    _lowercase					(     self				:     str,   UpperCAmelCase__				:     Union[str, Any]      ):
    __lowercase								=		self.output_dir.joinpath("best_tfmr"      )
    __lowercase								=		self.step_count
    self.model.save_pretrained(_SCREAMING_SNAKE_CASE      )
    self.tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE      )



 @staticmethod
 def    _lowercase					(     UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     Union[str, Any]      ):
    parser.add_argument(
        "--model_name_or_path",   default=_SCREAMING_SNAKE_CASE,   type=_SCREAMING_SNAKE_CASE,   required=_SCREAMING_SNAKE_CASE,   help="Path to pretrained model or model identifier from huggingface.co/models",   )
    parser.add_argument(
        "--config_name",   default="",   type=_SCREAMING_SNAKE_CASE,   help="Pretrained config name or path if not the same as model_name"      )
    parser.add_argument(
        "--tokenizer_name",   default=_SCREAMING_SNAKE_CASE,   type=_SCREAMING_SNAKE_CASE,   help="Pretrained tokenizer name or path if not the same as model_name",   )
    parser.add_argument(
        "--cache_dir",   default="",   type=_SCREAMING_SNAKE_CASE,   help="Where do you want to store the pre-trained models downloaded from huggingface.co",   )
    parser.add_argument(
        "--encoder_layerdrop",   type=_SCREAMING_SNAKE_CASE,   help="Encoder layer dropout probability (Optional). Goes into model.config",   )
    parser.add_argument(
        "--decoder_layerdrop",   type=_SCREAMING_SNAKE_CASE,   help="Decoder layer dropout probability (Optional). Goes into model.config",   )
    parser.add_argument(
        "--dropout",   type=_SCREAMING_SNAKE_CASE,   help="Dropout probability (Optional). Goes into model.config",   )
    parser.add_argument(
        "--attention_dropout",   type=_SCREAMING_SNAKE_CASE,   help="Attention dropout probability (Optional). Goes into model.config",   )
    parser.add_argument("--learning_rate",   default=5E-5,   type=_SCREAMING_SNAKE_CASE,   help="The initial learning rate for Adam."      )
    parser.add_argument(
        "--lr_scheduler",   default="linear",   choices=_SCREAMING_SNAKE_CASE,   metavar=_SCREAMING_SNAKE_CASE,   type=_SCREAMING_SNAKE_CASE,   help="Learning rate scheduler",   )
    parser.add_argument("--weight_decay",   default=0.0,   type=_SCREAMING_SNAKE_CASE,   help="Weight decay if we apply some."      )
    parser.add_argument("--adam_epsilon",   default=1E-8,   type=_SCREAMING_SNAKE_CASE,   help="Epsilon for Adam optimizer."      )
    parser.add_argument("--warmup_steps",   default=0,   type=_SCREAMING_SNAKE_CASE,   help="Linear warmup over warmup_steps."      )
    parser.add_argument("--num_workers",   default=4,   type=_SCREAMING_SNAKE_CASE,   help="kwarg passed to DataLoader"      )
    parser.add_argument("--num_train_epochs",   dest="max_epochs",   default=3,   type=_SCREAMING_SNAKE_CASE      )
    parser.add_argument("--train_batch_size",   default=3_2,   type=_SCREAMING_SNAKE_CASE      )
    parser.add_argument("--eval_batch_size",   default=3_2,   type=_SCREAMING_SNAKE_CASE      )
    parser.add_argument("--adafactor",   action="store_true"      )







class    _lowerCAmelCase    (      pl.Callback  ):







 """simple docstring"""







 def    _lowercase					(     self				:     str,   UpperCAmelCase__				:     Any,   UpperCAmelCase__				:     Any      ):
    __lowercase								=		trainer.lr_schedulers[0]["scheduler"]
    __lowercase								=		{F"""lr_group_{i}""": lr for i, lr in enumerate(lr_scheduler.get_lr()      )}
    pl_module.logger.log_metrics(_SCREAMING_SNAKE_CASE      )


 def    _lowercase					(     self				:     List[Any],   UpperCAmelCase__				:     Dict,   UpperCAmelCase__				:     List[Any]      ):
    rank_zero_info("***** Validation results *****"      )
    __lowercase								=		trainer.callback_metrics
    # Log results
    for key in sorted(_SCREAMING_SNAKE_CASE      ):
       if key not in ["log", "progress_bar"]:
          rank_zero_info("{} = {}\n".format(_SCREAMING_SNAKE_CASE,   str(metrics[key]      )      )      )



 def    _lowercase					(     self				:     List[Any],   UpperCAmelCase__				:     str,   UpperCAmelCase__				:     Union[str, Any]      ):
    rank_zero_info("***** Test results *****"      )
    __lowercase								=		trainer.callback_metrics
    # Log and save results to file
    __lowercase								=		os.path.join(pl_module.hparams.output_dir,   "test_results.txt"      )
    with open(_SCREAMING_SNAKE_CASE,   "w"      ) as writer:
       for key in sorted(_SCREAMING_SNAKE_CASE      ):
          if key not in ["log", "progress_bar"]:
             rank_zero_info("{} = {}\n".format(_SCREAMING_SNAKE_CASE,   str(metrics[key]      )      )      )
             writer.write("{} = {}\n".format(_SCREAMING_SNAKE_CASE,   str(metrics[key]      )      )      )







def     _A    (    UpperCamelCase_	:			Dict,     UpperCamelCase_	:			Any)      ->		None:
   '''simple docstring'''


   parser.add_argument(
       "--output_dir",     default=_A,     type=_A,     required=_A,     help="The output directory where the model predictions and checkpoints will be written.",     )
   parser.add_argument(
       "--fp16",     action="store_true",     help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",     )

   parser.add_argument(
       "--fp16_opt_level",     type=_A,     default="O2",     help=(
           "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
           "See details at https://nvidia.github.io/apex/amp.html"
       ),     )
   parser.add_argument("--n_tpu_cores",     dest="tpu_cores",     type=_A)
   parser.add_argument("--max_grad_norm",     dest="gradient_clip_val",     default=1.0,     type=_A,     help="Max gradient norm")
   parser.add_argument("--do_train",     action="store_true",     help="Whether to run training.")
   parser.add_argument("--do_predict",     action="store_true",     help="Whether to run predictions on the test set.")
   parser.add_argument(
       "--gradient_accumulation_steps",     dest="accumulate_grad_batches",     type=_A,     default=1,     help="Number of updates steps to accumulate before performing a backward/update pass.",     )
   parser.add_argument("--seed",     type=_A,     default=42,     help="random seed for initialization")
   parser.add_argument(
       "--data_dir",     default=_A,     type=_A,     required=_A,     help="The input data dir. Should contain the training files for the CoNLL-2003 NER task.",     )




def     _A    (    UpperCamelCase_	:			BaseTransformer,     UpperCamelCase_	:			argparse.Namespace,     UpperCamelCase_	:			List[Any]=None,     UpperCamelCase_	:			Any=True,     UpperCamelCase_	:			str=[],     UpperCamelCase_	:			Optional[Any]=None,     UpperCamelCase_	:			Optional[int]=None,     **UpperCamelCase_	:			Union[str, Any],     )      ->		Dict:
   '''simple docstring'''


   pl.seed_everything(args.seed)

   # init model
   __lowercase								=		Path(model.hparams.output_dir)
   odir.mkdir(exist_ok=_A)

   # add custom checkpoints
   if checkpoint_callback is None:
      __lowercase								=		pl.callbacks.ModelCheckpoint(
          filepath=args.output_dir,     prefix="checkpoint",     monitor="val_loss",     mode="min",     save_top_k=1)
   if early_stopping_callback:
      extra_callbacks.append(_A)
   if logging_callback is None:
      __lowercase								=		LoggingCallback()

   __lowercase								=		{}

   # TODO: remove with PyTorch 1.6 since pl uses native amp
   if args.fpaa:
      __lowercase								=		16
      __lowercase								=		args.fpaa_opt_level

   if args.gpus > 1:
      __lowercase								=		"ddp"

   __lowercase								=		args.accumulate_grad_batches
   __lowercase								=		extra_train_kwargs.get("accelerator",     _A)
   __lowercase								=		extra_train_kwargs.get("profiler",     _A)

   __lowercase								=		pl.Trainer.from_argparse_args(
       _A,     weights_summary=_A,     callbacks=[logging_callback] + extra_callbacks,     logger=_A,     checkpoint_callback=_A,     **_A,     )

   if args.do_train:
      trainer.fit(_A)

   return trainer







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A	:   List[Any]	=					logging.get_logger(__name__)

__A	:   Tuple	=					{
    'google/electra-small-generator': 'https://huggingface.co/google/electra-small-generator/resolve/main/config.json',
    'google/electra-base-generator': 'https://huggingface.co/google/electra-base-generator/resolve/main/config.json',
    'google/electra-large-generator': 'https://huggingface.co/google/electra-large-generator/resolve/main/config.json',
    'google/electra-small-discriminator': (
        'https://huggingface.co/google/electra-small-discriminator/resolve/main/config.json'
    ),
    'google/electra-base-discriminator': (
        'https://huggingface.co/google/electra-base-discriminator/resolve/main/config.json'
    ),
    'google/electra-large-discriminator': (
        'https://huggingface.co/google/electra-large-discriminator/resolve/main/config.json'
    ),
}


class 			_SCREAMING_SNAKE_CASE      (  lowerCAmelCase__):
						_UpperCamelCase:Tuple	=		"electra"







						def __init__(			self     ,						_SCREAMING_SNAKE_CASE=3_0522     ,						_SCREAMING_SNAKE_CASE=128     ,						_SCREAMING_SNAKE_CASE=256     ,						_SCREAMING_SNAKE_CASE=12     ,						_SCREAMING_SNAKE_CASE=4     ,						_SCREAMING_SNAKE_CASE=1024     ,						_SCREAMING_SNAKE_CASE="gelu"     ,						_SCREAMING_SNAKE_CASE=0.1     ,						_SCREAMING_SNAKE_CASE=0.1     ,						_SCREAMING_SNAKE_CASE=512     ,						_SCREAMING_SNAKE_CASE=2     ,						_SCREAMING_SNAKE_CASE=0.0_2     ,						_SCREAMING_SNAKE_CASE=1E-12     ,						_SCREAMING_SNAKE_CASE="first"     ,						_SCREAMING_SNAKE_CASE=True     ,						_SCREAMING_SNAKE_CASE="gelu"     ,						_SCREAMING_SNAKE_CASE=0.1     ,						_SCREAMING_SNAKE_CASE=0     ,						_SCREAMING_SNAKE_CASE="absolute"     ,						_SCREAMING_SNAKE_CASE=True     ,						_SCREAMING_SNAKE_CASE=None     ,						**_SCREAMING_SNAKE_CASE     ,						)->							Optional[int]:
													super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE     ,						**_SCREAMING_SNAKE_CASE							)

													lowerCamelCase_								=vocab_size
													lowerCamelCase_								=embedding_size
													lowerCamelCase_								=hidden_size
													lowerCamelCase_								=num_hidden_layers
													lowerCamelCase_								=num_attention_heads
													lowerCamelCase_								=intermediate_size
													lowerCamelCase_								=hidden_act
													lowerCamelCase_								=hidden_dropout_prob
													lowerCamelCase_								=attention_probs_dropout_prob
													lowerCamelCase_								=max_position_embeddings
													lowerCamelCase_								=type_vocab_size
													lowerCamelCase_								=initializer_range
													lowerCamelCase_								=layer_norm_eps

													lowerCamelCase_								=summary_type
													lowerCamelCase_								=summary_use_proj
													lowerCamelCase_								=summary_activation
													lowerCamelCase_								=summary_last_dropout
													lowerCamelCase_								=position_embedding_type
													lowerCamelCase_								=use_cache
													lowerCamelCase_								=classifier_dropout


class 			_SCREAMING_SNAKE_CASE      (  lowerCAmelCase__):







						@property
						def        _snake_case				(			self							)->							Mapping[str, Mapping[int, str]]:
													if self.task == "multiple-choice":
																				lowerCamelCase_								={0: """batch""", 1: """choice""", 2: """sequence"""}
													else:
																				lowerCamelCase_								={0: """batch""", 1: """sequence"""}
													return OrderedDict(
													    [
													        ("""input_ids""", dynamic_axis),
													        ("""attention_mask""", dynamic_axis),
													        ("""token_type_ids""", dynamic_axis),
													    ]							)
