




"""simple docstring"""


import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


A_						=				logging.getLogger(__name__)

require_version('''pytorch_lightning>=1.0.4''')

A_						=				{
    '''base''': AutoModel,
    '''sequence-classification''': AutoModelForSequenceClassification,
    '''question-answering''': AutoModelForQuestionAnswering,
    '''pretraining''': AutoModelForPreTraining,
    '''token-classification''': AutoModelForTokenClassification,
    '''language-modeling''': AutoModelWithLMHead,
    '''summarization''': AutoModelForSeqaSeqLM,
    '''translation''': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
A_						=				{
    '''linear''': get_linear_schedule_with_warmup,
    '''cosine''': get_cosine_schedule_with_warmup,
    '''cosine_w_restarts''': get_cosine_with_hard_restarts_schedule_with_warmup,
    '''polynomial''': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
A_						=				sorted(arg_to_scheduler.keys())
A_						=				'''{''' + ''', '''.join(arg_to_scheduler_choices) + '''}'''






class 					lowercase(     pl.LightningModule ):







 '''simple docstring'''





 def __init__(				self:		List[str],      a_:		argparse.Namespace,      a_:		Tuple=None,      a_:		Dict="base",      a_:		Tuple=None,      a_:		Union[str, Any]=None,      a_:		int=None,      **a_:		int,      ):




   '''simple docstring'''



   super().__init__()
   # TODO: move to self.save_hyperparameters()
   # self.save_hyperparameters()
   # can also expand arguments into trainer signature for easier reading

   self.save_hyperparameters(a_		)
   _snake_case							:		Any								=  0
   _snake_case							:		str								=  Path(self.hparams.output_dir		)
   _snake_case							:		List[Any]								=  self.hparams.cache_dir if self.hparams.cache_dir else None
   if config is None:
     _snake_case							:		Optional[int]								=  AutoConfig.from_pretrained(
         self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path,      **({"""num_labels""": num_labels} if num_labels is not None else {}),      cache_dir=a_,      **a_,      )
   else:
     _snake_case							:		PretrainedConfig								=  config

   _snake_case							:		Dict								=  ("""encoder_layerdrop""", """decoder_layerdrop""", """dropout""", """attention_dropout""")
   for p in extra_model_params:
     if getattr(self.hparams,      a_,      a_		):
       assert hasattr(self.config,      a_		), f"model config doesn't have a `{p}` attribute"
       setattr(self.config,      a_,      getattr(self.hparams,      a_		)		)

   if tokenizer is None:
     _snake_case							:		Union[str, Any]								=  AutoTokenizer.from_pretrained(
         self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path,      cache_dir=a_,      )
   else:
     _snake_case							:		PreTrainedTokenizer								=  tokenizer
   _snake_case							:		Union[str, Any]								=  MODEL_MODES[mode]
   if model is None:
     _snake_case							:		Dict								=  self.model_type.from_pretrained(
         self.hparams.model_name_or_path,      from_tf=bool(""".ckpt""" in self.hparams.model_name_or_path		),      config=self.config,      cache_dir=a_,      )
   else:
     _snake_case							:		int								=  model


 def  UpperCamelCase_   (				self:		Dict,      *a_:		Dict,      **a_:		List[str]		):




   '''simple docstring'''



   _snake_case							:		List[Any]								=  self.model_type.from_pretrained(*a_,      **a_		)


 def  UpperCamelCase_   (				self:		Union[str, Any]		):




   '''simple docstring'''



   _snake_case							:		Dict								=  arg_to_scheduler[self.hparams.lr_scheduler]
   _snake_case							:		Optional[int]								=  get_schedule_func(
       self.opt,      num_warmup_steps=self.hparams.warmup_steps,      num_training_steps=self.total_steps()		)
   _snake_case							:		Dict								=  {"""scheduler""": scheduler, """interval""": """step""", """frequency""": 1}
   return scheduler


 def  UpperCamelCase_   (				self:		Tuple		):




   '''simple docstring'''



   _snake_case							:		Any								=  self.model
   _snake_case							:		List[Any]								=  ["""bias""", """LayerNorm.weight"""]
   _snake_case							:		Any								=  [
       {
           """params""": [
               p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay		)
           ],  # check this named paramters
           """weight_decay""": self.hparams.weight_decay,
       },
       {
           """params""": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay		)],
           """weight_decay""": 0.0,
       },
   ]
   if self.hparams.adafactor:
     _snake_case							:		Optional[int]								=  Adafactor(
         a_,      lr=self.hparams.learning_rate,      scale_parameter=a_,      relative_step=a_		)

   else:
     _snake_case							:		Optional[int]								=  AdamW(
         a_,      lr=self.hparams.learning_rate,      eps=self.hparams.adam_epsilon		)
   _snake_case							:		Optional[Any]								=  optimizer

   _snake_case							:		Tuple								=  self.get_lr_scheduler()

   return [optimizer], [scheduler]


 def  UpperCamelCase_   (				self:		int,      a_:		int,      a_:		Optional[Any]		):




   '''simple docstring'''



   return self.validation_step(a_,      a_		)


 def  UpperCamelCase_   (				self:		Optional[Any],      a_:		Union[str, Any]		):




   '''simple docstring'''



   return self.validation_end(a_		)


 def  UpperCamelCase_   (				self:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		List[Any]								=  max(1,      self.hparams.gpus		)  # TODO: consider num_tpu_cores
   _snake_case							:		Optional[Any]								=  self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
   return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs


 def  UpperCamelCase_   (				self:		Optional[Any],      a_:		Optional[Any]		):




   '''simple docstring'''



   if stage == "test":
     _snake_case							:		Union[str, Any]								=  len(self.test_dataloader().dataset		)
   else:
     _snake_case							:		Union[str, Any]								=  self.get_dataloader("""train""",      self.hparams.train_batch_size,      shuffle=a_		)
     _snake_case							:		Union[str, Any]								=  len(self.train_dataloader().dataset		)


 def  UpperCamelCase_   (				self:		List[Any],      a_:		str,      a_:		int,      a_:		bool = False		):




   '''simple docstring'''



   raise NotImplementedError("""You must implement this for your task"""		)


 def  UpperCamelCase_   (				self:		str		):




   '''simple docstring'''



   return self.train_loader


 def  UpperCamelCase_   (				self:		Tuple		):




   '''simple docstring'''



   return self.get_dataloader("""dev""",      self.hparams.eval_batch_size,      shuffle=a_		)


 def  UpperCamelCase_   (				self:		Tuple		):




   '''simple docstring'''



   return self.get_dataloader("""test""",      self.hparams.eval_batch_size,      shuffle=a_		)


 def  UpperCamelCase_   (				self:		Dict,      a_:		int		):




   '''simple docstring'''



   return os.path.join(
       self.hparams.data_dir,      """cached_{}_{}_{}""".format(
           a_,      list(filter(a_,      self.hparams.model_name_or_path.split("""/"""		)		)		).pop(),      str(self.hparams.max_seq_length		),      ),      )


 @pl.utilities.rank_zero_only
 def  UpperCamelCase_   (				self:		str,      a_:		Dict[str, Any]		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  self.output_dir.joinpath("""best_tfmr"""		)
   _snake_case							:		Optional[int]								=  self.step_count
   self.model.save_pretrained(a_		)
   self.tokenizer.save_pretrained(a_		)


 @staticmethod
 def  UpperCamelCase_   (				a_:		Dict,      a_:		Dict		):




   '''simple docstring'''



   parser.add_argument(
       """--model_name_or_path""",      default=a_,      type=a_,      required=a_,      help="""Path to pretrained model or model identifier from huggingface.co/models""",      )
   parser.add_argument(
       """--config_name""",      default="""""",      type=a_,      help="""Pretrained config name or path if not the same as model_name"""		)
   parser.add_argument(
       """--tokenizer_name""",      default=a_,      type=a_,      help="""Pretrained tokenizer name or path if not the same as model_name""",      )
   parser.add_argument(
       """--cache_dir""",      default=str(Path(a_		).parent / """test_run""" / """cache"""		),      type=a_,      help="""Where do you want to store the pre-trained models downloaded from huggingface.co""",      )
   parser.add_argument(
       """--encoder_layerdrop""",      type=a_,      help="""Encoder layer dropout probability (Optional). Goes into model.config""",      )
   parser.add_argument(
       """--decoder_layerdrop""",      type=a_,      help="""Decoder layer dropout probability (Optional). Goes into model.config""",      )
   parser.add_argument(
       """--dropout""",      type=a_,      help="""Dropout probability (Optional). Goes into model.config""",      )
   parser.add_argument(
       """--attention_dropout""",      type=a_,      help="""Attention dropout probability (Optional). Goes into model.config""",      )
   parser.add_argument("""--learning_rate""",      default=5E-5,      type=a_,      help="""The initial learning rate for Adam."""		)
   parser.add_argument(
       """--lr_scheduler""",      default="""linear""",      choices=a_,      metavar=a_,      type=a_,      help="""Learning rate scheduler""",      )
   parser.add_argument("""--weight_decay""",      default=0.0,      type=a_,      help="""Weight decay if we apply some."""		)
   parser.add_argument("""--adam_epsilon""",      default=1E-8,      type=a_,      help="""Epsilon for Adam optimizer."""		)
   parser.add_argument("""--warmup_steps""",      default=0,      type=a_,      help="""Linear warmup over warmup_steps."""		)
   parser.add_argument("""--num_workers""",      default=4,      type=a_,      help="""kwarg passed to DataLoader"""		)
   parser.add_argument("""--num_train_epochs""",      dest="""max_epochs""",      default=3,      type=a_		)
   parser.add_argument("""--train_batch_size""",      default=32,      type=a_		)
   parser.add_argument("""--eval_batch_size""",      default=32,      type=a_		)
   parser.add_argument("""--adafactor""",      action="""store_true"""		)







class 					lowercase(     pl.Callback ):







 '''simple docstring'''





 def  UpperCamelCase_   (				self:		Optional[int],      a_:		Optional[int],      a_:		Any		):




   '''simple docstring'''



   if (
       trainer.is_global_zero and trainer.global_rank == 0
   ):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
     pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.







class 					lowercase(     pl.Callback ):







 '''simple docstring'''





 def  UpperCamelCase_   (				self:		str,      a_:		Optional[Any],      a_:		List[str]		):




   '''simple docstring'''



   for name, param in pl_module.model.rag.named_parameters():
     if param.grad is None:
       print(a_		)







class 					lowercase(     pl.Callback ):







 '''simple docstring'''





 def  UpperCamelCase_   (				self:		List[Any],      a_:		Union[str, Any],      a_:		str		):




   '''simple docstring'''



   _snake_case							:		Any								=  trainer.lr_schedulers[0]["""scheduler"""]
   _snake_case							:		List[str]								=  {f"lr_group_{i}": lr for i, lr in enumerate(lr_scheduler.get_lr()		)}
   pl_module.logger.log_metrics(a_		)


 def  UpperCamelCase_   (				self:		Dict,      a_:		pl.Trainer,      a_:		pl.LightningModule		):




   '''simple docstring'''



   rank_zero_info("""***** Validation results *****"""		)
   _snake_case							:		List[str]								=  trainer.callback_metrics
   # Log results
   for key in sorted(a_		):
     if key not in ["log", "progress_bar"]:
       rank_zero_info("""{} = {}\n""".format(a_,      str(metrics[key]		)		)		)


 def  UpperCamelCase_   (				self:		Any,      a_:		pl.Trainer,      a_:		pl.LightningModule		):




   '''simple docstring'''



   rank_zero_info("""***** Test results *****"""		)
   _snake_case							:		Union[str, Any]								=  trainer.callback_metrics
   # Log and save results to file
   _snake_case							:		str								=  os.path.join(pl_module.hparams.output_dir,      """test_results.txt"""		)
   with open(a_,      """w"""		) as writer:
     for key in sorted(a_		):
       if key not in ["log", "progress_bar"]:
         rank_zero_info("""{} = {}\n""".format(a_,      str(metrics[key]		)		)		)
         writer.write("""{} = {}\n""".format(a_,      str(metrics[key]		)		)		)







def 					UpperCAmelCase__       (snake_case__							:					Tuple		,   snake_case__							:					Optional[int]    ):
  """simple docstring"""

  parser.add_argument(
      """--output_dir"""		,   default=str(Path(snake_case__    ).parent / """test_run""" / """model_checkpoints"""    )		,   type=snake_case__		,   help="""The output directory where the model predictions and checkpoints will be written."""		,   )
  parser.add_argument(
      """--fp16"""		,   action="""store_true"""		,   help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""		,   )

  parser.add_argument(
      """--fp16_opt_level"""		,   type=snake_case__		,   default="""O2"""		,   help=(
          """For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."""
          """See details at https://nvidia.github.io/apex/amp.html"""
      )		,   )
  parser.add_argument("""--n_tpu_cores"""		,   dest="""tpu_cores"""		,   type=snake_case__    )
  parser.add_argument("""--max_grad_norm"""		,   dest="""gradient_clip_val"""		,   default=1.0		,   type=snake_case__		,   help="""Max gradient norm"""    )
  parser.add_argument("""--do_train"""		,   action="""store_true"""		,   help="""Whether to run training."""    )
  parser.add_argument("""--do_predict"""		,   action="""store_true"""		,   help="""Whether to run predictions on the test set."""    )
  parser.add_argument(
      """--gradient_accumulation_steps"""		,   dest="""accumulate_grad_batches"""		,   type=snake_case__		,   default=1		,   help="""Number of updates steps to accumulate before performing a backward/update pass."""		,   )
  parser.add_argument("""--seed"""		,   type=snake_case__		,   default=42		,   help="""random seed for initialization"""    )
  parser.add_argument(
      """--data_dir"""		,   default=str(Path(snake_case__    ).parent / """test_run""" / """dummy-train-data"""    )		,   type=snake_case__		,   help="""The input data dir. Should contain the training files for the CoNLL-2003 NER task."""		,   )






def 					UpperCAmelCase__       (snake_case__							:					BaseTransformer		,   snake_case__							:					argparse.Namespace		,   snake_case__							:					int=None		,   snake_case__							:					str=True		,   snake_case__							:					Optional[Any]=[]		,   snake_case__							:					List[str]=None		,   snake_case__							:					Any=None		,   **snake_case__							:					Tuple		,   ):
  """simple docstring"""

  pl.seed_everything(args.seed    )

  # init model
  _snake_case							:		str								=  Path(model.hparams.output_dir    )
  odir.mkdir(exist_ok=snake_case__    )

  # add custom checkpoints
  if checkpoint_callback is None:
    _snake_case							:		Optional[Any]								=  pl.callbacks.ModelCheckpoint(
        filepath=args.output_dir		,   prefix="""checkpoint"""		,   monitor="""val_loss"""		,   mode="""min"""		,   save_top_k=1    )
  if early_stopping_callback:
    extra_callbacks.append(snake_case__    )
  if logging_callback is None:
    _snake_case							:		Union[str, Any]								=  LoggingCallback()

  _snake_case							:		List[str]								=  {}

  if args.fpaa:
    _snake_case							:		Union[str, Any]								=  16

  if args.gpus > 1:
    _snake_case							:		Optional[Any]								=  """auto"""
    _snake_case							:		Union[str, Any]								=  """ddp"""

  _snake_case							:		List[str]								=  args.accumulate_grad_batches
  _snake_case							:		Dict								=  None
  _snake_case							:		str								=  """auto"""

  _snake_case							:		Dict								=  pl.Trainer.from_argparse_args(
      snake_case__		,   weights_summary=snake_case__		,   callbacks=[logging_callback] + extra_callbacks + [InitCallback()] + [checkpoint_callback]		,   logger=snake_case__		,   val_check_interval=1		,   num_sanity_val_steps=2		,   **snake_case__		,   )

  if args.do_train:
    trainer.fit(snake_case__    )

  else:
    print("""RAG modeling tests with new set functions successfuly executed!"""    )
  return trainer






'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


__lowercase	:						str       =	{
    'configuration_data2vec_audio': ['DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP', 'Data2VecAudioConfig'],
    'configuration_data2vec_text': [
        'DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'Data2VecTextConfig',
        'Data2VecTextOnnxConfig',
    ],
    'configuration_data2vec_vision': [
        'DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'Data2VecVisionConfig',
        'Data2VecVisionOnnxConfig',
    ],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowercase	:						str       =	[
	    'DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'Data2VecAudioForAudioFrameClassification',
	    'Data2VecAudioForCTC',
	    'Data2VecAudioForSequenceClassification',
	    'Data2VecAudioForXVector',
	    'Data2VecAudioModel',
	    'Data2VecAudioPreTrainedModel',
	]
	__lowercase	:						Tuple       =	[
	    'DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'Data2VecTextForCausalLM',
	    'Data2VecTextForMaskedLM',
	    'Data2VecTextForMultipleChoice',
	    'Data2VecTextForQuestionAnswering',
	    'Data2VecTextForSequenceClassification',
	    'Data2VecTextForTokenClassification',
	    'Data2VecTextModel',
	    'Data2VecTextPreTrainedModel',
	]
	__lowercase	:						Dict       =	[
	    'DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'Data2VecVisionForImageClassification',
	    'Data2VecVisionForMaskedImageModeling',
	    'Data2VecVisionForSemanticSegmentation',
	    'Data2VecVisionModel',
	    'Data2VecVisionPreTrainedModel',
	]

if is_tf_available():
	__lowercase	:						Optional[Any]       =	[
	    'TFData2VecVisionForImageClassification',
	    'TFData2VecVisionForSemanticSegmentation',
	    'TFData2VecVisionModel',
	    'TFData2VecVisionPreTrainedModel',
	]

if TYPE_CHECKING:
	from .configuration_dataavec_audio import DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP, DataaVecAudioConfig
	from .configuration_dataavec_text import (
	    DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    DataaVecTextConfig,
	    DataaVecTextOnnxConfig,
	)
	from .configuration_dataavec_vision import (
	    DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    DataaVecVisionConfig,
	    DataaVecVisionOnnxConfig,
	)

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_dataavec_audio import (
		    DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST,
		    DataaVecAudioForAudioFrameClassification,
		    DataaVecAudioForCTC,
		    DataaVecAudioForSequenceClassification,
		    DataaVecAudioForXVector,
		    DataaVecAudioModel,
		    DataaVecAudioPreTrainedModel,
		)
		from .modeling_dataavec_text import (
		    DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    DataaVecTextForCausalLM,
		    DataaVecTextForMaskedLM,
		    DataaVecTextForMultipleChoice,
		    DataaVecTextForQuestionAnswering,
		    DataaVecTextForSequenceClassification,
		    DataaVecTextForTokenClassification,
		    DataaVecTextModel,
		    DataaVecTextPreTrainedModel,
		)
		from .modeling_dataavec_vision import (
		    DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST,
		    DataaVecVisionForImageClassification,
		    DataaVecVisionForMaskedImageModeling,
		    DataaVecVisionForSemanticSegmentation,
		    DataaVecVisionModel,
		    DataaVecVisionPreTrainedModel,
		)
	if is_tf_available():
		from .modeling_tf_dataavec_vision import (
		    TFDataaVecVisionForImageClassification,
		    TFDataaVecVisionForSemanticSegmentation,
		    TFDataaVecVisionModel,
		    TFDataaVecVisionPreTrainedModel,
		)

else:
	import sys

	__lowercase	:						int       =	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



