

import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


__A     						=      logging.getLogger(__name__)

require_version('''pytorch_lightning>=1.0.4''')

__A     						=      {
    '''base''': AutoModel,
    '''sequence-classification''': AutoModelForSequenceClassification,
    '''question-answering''': AutoModelForQuestionAnswering,
    '''pretraining''': AutoModelForPreTraining,
    '''token-classification''': AutoModelForTokenClassification,
    '''language-modeling''': AutoModelWithLMHead,
    '''summarization''': AutoModelForSeqaSeqLM,
    '''translation''': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
__A     						=      {
    '''linear''': get_linear_schedule_with_warmup,
    '''cosine''': get_cosine_schedule_with_warmup,
    '''cosine_w_restarts''': get_cosine_with_hard_restarts_schedule_with_warmup,
    '''polynomial''': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
__A     						=      sorted(arg_to_scheduler.keys())
__A     						=      '''{''' + ''', '''.join(arg_to_scheduler_choices) + '''}'''



class     lowercase_       (    pl.LightningModule	):



      def __init__(     self :							Tuple     ,   A__ :							List[Any]     ,   A__ :							List[Any]=None     ,   A__ :							Union[str, Any]="base"     ,   A__ :							List[str]=None     ,   A__ :							int=None     ,   A__ :							Any=None     ,   **A__ :							Tuple     ,   )       ->	List[Any]:
        super().__init__()
        # TODO: move to self.save_hyperparameters()
        # self.save_hyperparameters()
        # can also expand arguments into trainer signature for easier reading

        self.save_hyperparameters(__A						)
        _snake_case		=				0
        _snake_case		=				Path(self.hparams.output_dir						)
        _snake_case		=				self.hparams.cache_dir if self.hparams.cache_dir else None
        if config is None:
          _snake_case		=				AutoConfig.from_pretrained(
              self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path     ,   **({'''num_labels''': num_labels} if num_labels is not None else {})     ,   cache_dir=__A     ,   **__A     ,   )
        else:
          _snake_case		=				config

        _snake_case		=				("""encoder_layerdrop""", """decoder_layerdrop""", """dropout""", """attention_dropout""")
        for p in extra_model_params:
          if getattr(self.hparams     ,   __A     ,   __A						):
            assert hasattr(self.config     ,   __A						), f"""model config doesn't have a `{p}` attribute"""
            setattr(self.config     ,   __A     ,   getattr(self.hparams     ,   __A						)						)

        if tokenizer is None:
          _snake_case		=				AutoTokenizer.from_pretrained(
              self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path     ,   cache_dir=__A     ,   )
        else:
          _snake_case		=				tokenizer
        _snake_case		=				MODEL_MODES[mode]
        if model is None:
          _snake_case		=				self.model_type.from_pretrained(
              self.hparams.model_name_or_path     ,   from_tf=bool('''.ckpt''' in self.hparams.model_name_or_path						)     ,   config=self.config     ,   cache_dir=__A     ,   )
        else:
          _snake_case		=				model



      def 		UpperCamelCase_						(     self :							Tuple     ,   *A__ :							List[str]     ,   **A__ :							List[Any]						)       ->	List[Any]:
        _snake_case		=				self.model_type.from_pretrained(*__A     ,   **__A						)



      def 		UpperCamelCase_						(     self :							Dict						)       ->	List[Any]:
        _snake_case		=				arg_to_scheduler[self.hparams.lr_scheduler]
        _snake_case		=				get_schedule_func(
            self.opt     ,   num_warmup_steps=self.hparams.warmup_steps     ,   num_training_steps=self.total_steps()						)
        _snake_case		=				{"""scheduler""": scheduler, """interval""": """step""", """frequency""": 1}
        return scheduler



      def 		UpperCamelCase_						(     self :							int						)       ->	str:
        _snake_case		=				self.model
        _snake_case		=				["""bias""", """LayerNorm.weight"""]
        _snake_case		=				[
            {
                """params""": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay						)],
                """weight_decay""": self.hparams.weight_decay,
            },
            {
                """params""": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay						)],
                """weight_decay""": 0.0,
            },
        ]
        if self.hparams.adafactor:
          _snake_case		=				Adafactor(
              __A     ,   lr=self.hparams.learning_rate     ,   scale_parameter=__A     ,   relative_step=__A						)

        else:
          _snake_case		=				AdamW(
              __A     ,   lr=self.hparams.learning_rate     ,   eps=self.hparams.adam_epsilon						)
        _snake_case		=				optimizer

        _snake_case		=				self.get_lr_scheduler()

        return [optimizer], [scheduler]



      def 		UpperCamelCase_						(     self :							Union[str, Any]     ,   A__ :							str     ,   A__ :							Optional[int]						)       ->	Any:
        return self.validation_step(__A     ,   __A						)



      def 		UpperCamelCase_						(     self :							Optional[int]     ,   A__ :							str						)       ->	Union[str, Any]:
        return self.validation_end(__A						)



      def 		UpperCamelCase_						(     self :							Any						)       ->	int:
        _snake_case		=				max(1     ,   self.hparams.gpus						)  # TODO: consider num_tpu_cores
        _snake_case		=				self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
        return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs



      def 		UpperCamelCase_						(     self :							Optional[Any]     ,   A__ :							int						)       ->	Tuple:
        if mode == "test":
          _snake_case		=				len(self.test_dataloader().dataset						)
        else:
          _snake_case		=				self.get_dataloader('''train'''     ,   self.hparams.train_batch_size     ,   shuffle=__A						)
          _snake_case		=				len(self.train_dataloader().dataset						)



      def 		UpperCamelCase_						(     self :							Tuple     ,   A__ :							Optional[int]     ,   A__ :							Optional[int]     ,   A__ :							int = False						)       ->	Union[str, Any]:
        raise NotImplementedError('''You must implement this for your task'''						)



      def 		UpperCamelCase_						(     self :							Tuple						)       ->	Optional[int]:
        return self.train_loader



      def 		UpperCamelCase_						(     self :							Optional[int]						)       ->	List[str]:
        return self.get_dataloader('''dev'''     ,   self.hparams.eval_batch_size     ,   shuffle=__A						)



      def 		UpperCamelCase_						(     self :							Dict						)       ->	Any:
        return self.get_dataloader('''test'''     ,   self.hparams.eval_batch_size     ,   shuffle=__A						)



      def 		UpperCamelCase_						(     self :							Dict     ,   A__ :							Optional[int]						)       ->	Optional[int]:
        return os.path.join(
            self.hparams.data_dir     ,   '''cached_{}_{}_{}'''.format(
                __A     ,   list(filter(__A     ,   self.hparams.model_name_or_path.split('''/'''						)						)						).pop()     ,   str(self.hparams.max_seq_length						)     ,   )     ,   )



      @pl.utilities.rank_zero_only
      def 		UpperCamelCase_						(     self :							Tuple     ,   A__ :							Optional[int]						)       ->	None:
        _snake_case		=				self.output_dir.joinpath('''best_tfmr'''						)
        _snake_case		=				self.step_count
        self.model.save_pretrained(__A						)
        self.tokenizer.save_pretrained(__A						)







      @staticmethod
      def 		UpperCamelCase_						(     A__ :							Optional[int]     ,   A__ :							int						)       ->	Dict:
        parser.add_argument(
            '''--model_name_or_path'''     ,   default=__A     ,   type=__A     ,   required=__A     ,   help='''Path to pretrained model or model identifier from huggingface.co/models'''     ,   )
        parser.add_argument(
            '''--config_name'''     ,   default=''''''     ,   type=__A     ,   help='''Pretrained config name or path if not the same as model_name'''						)
        parser.add_argument(
            '''--tokenizer_name'''     ,   default=__A     ,   type=__A     ,   help='''Pretrained tokenizer name or path if not the same as model_name'''     ,   )
        parser.add_argument(
            '''--cache_dir'''     ,   default=''''''     ,   type=__A     ,   help='''Where do you want to store the pre-trained models downloaded from huggingface.co'''     ,   )
        parser.add_argument(
            '''--encoder_layerdrop'''     ,   type=__A     ,   help='''Encoder layer dropout probability (Optional). Goes into model.config'''     ,   )
        parser.add_argument(
            '''--decoder_layerdrop'''     ,   type=__A     ,   help='''Decoder layer dropout probability (Optional). Goes into model.config'''     ,   )
        parser.add_argument(
            '''--dropout'''     ,   type=__A     ,   help='''Dropout probability (Optional). Goes into model.config'''     ,   )
        parser.add_argument(
            '''--attention_dropout'''     ,   type=__A     ,   help='''Attention dropout probability (Optional). Goes into model.config'''     ,   )
        parser.add_argument('''--learning_rate'''     ,   default=5e-5     ,   type=__A     ,   help='''The initial learning rate for Adam.'''						)
        parser.add_argument(
            '''--lr_scheduler'''     ,   default='''linear'''     ,   choices=__A     ,   metavar=__A     ,   type=__A     ,   help='''Learning rate scheduler'''     ,   )
        parser.add_argument('''--weight_decay'''     ,   default=0.0     ,   type=__A     ,   help='''Weight decay if we apply some.'''						)
        parser.add_argument('''--adam_epsilon'''     ,   default=1e-8     ,   type=__A     ,   help='''Epsilon for Adam optimizer.'''						)
        parser.add_argument('''--warmup_steps'''     ,   default=0     ,   type=__A     ,   help='''Linear warmup over warmup_steps.'''						)
        parser.add_argument('''--num_workers'''     ,   default=4     ,   type=__A     ,   help='''kwarg passed to DataLoader'''						)
        parser.add_argument('''--num_train_epochs'''     ,   dest='''max_epochs'''     ,   default=3     ,   type=__A						)
        parser.add_argument('''--train_batch_size'''     ,   default=32     ,   type=__A						)
        parser.add_argument('''--eval_batch_size'''     ,   default=32     ,   type=__A						)
        parser.add_argument('''--adafactor'''     ,   action='''store_true'''						)



class     lowercase_       (    pl.Callback	):



      def 		UpperCamelCase_						(     self :							List[Any]     ,   A__ :							Tuple     ,   A__ :							Any						)       ->	Any:
        _snake_case		=				trainer.lr_schedulers[0]["""scheduler"""]
        _snake_case		=				{f"""lr_group_{i}""": lr for i, lr in enumerate(lr_scheduler.get_lr()						)}
        pl_module.logger.log_metrics(__A						)



      def 		UpperCamelCase_						(     self :							str     ,   A__ :							Tuple     ,   A__ :							Dict						)       ->	int:
        rank_zero_info('''***** Validation results *****'''						)
        _snake_case		=				trainer.callback_metrics
        # Log results
        for key in sorted(__A						):
          if key not in ["log", "progress_bar"]:
            rank_zero_info('''{} = {}\n'''.format(__A     ,   str(metrics[key]						)						)						)







      def 		UpperCamelCase_						(     self :							List[Any]     ,   A__ :							List[Any]     ,   A__ :							Optional[Any]						)       ->	Tuple:
        rank_zero_info('''***** Test results *****'''						)
        _snake_case		=				trainer.callback_metrics
        # Log and save results to file
        _snake_case		=				os.path.join(pl_module.hparams.output_dir     ,   '''test_results.txt'''						)
        with open(__A     ,   '''w'''						) as writer:
          for key in sorted(__A						):
            if key not in ["log", "progress_bar"]:
              rank_zero_info('''{} = {}\n'''.format(__A     ,   str(metrics[key]						)						)						)
              writer.write('''{} = {}\n'''.format(__A     ,   str(metrics[key]						)						)						)



def  snake_case_(_UpperCamelCase					,			_UpperCamelCase	)				->      None:



  """simple docstring"""






  parser.add_argument(
      '''--output_dir'''					,			default=lowercase__					,			type=lowercase__					,			required=lowercase__					,			help='''The output directory where the model predictions and checkpoints will be written.'''					,			)
  parser.add_argument(
      '''--fp16'''					,			action='''store_true'''					,			help='''Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'''					,			)

  parser.add_argument(
      '''--fp16_opt_level'''					,			type=lowercase__					,			default='''O2'''					,			help=(
          '''For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'''
          '''See details at https://nvidia.github.io/apex/amp.html'''
      )					,			)
  parser.add_argument('''--n_tpu_cores'''					,			dest='''tpu_cores'''					,			type=lowercase__	)
  parser.add_argument('''--max_grad_norm'''					,			dest='''gradient_clip_val'''					,			default=1.0					,			type=lowercase__					,			help='''Max gradient norm'''	)
  parser.add_argument('''--do_train'''					,			action='''store_true'''					,			help='''Whether to run training.'''	)
  parser.add_argument('''--do_predict'''					,			action='''store_true'''					,			help='''Whether to run predictions on the test set.'''	)
  parser.add_argument(
      '''--gradient_accumulation_steps'''					,			dest='''accumulate_grad_batches'''					,			type=lowercase__					,			default=1					,			help='''Number of updates steps to accumulate before performing a backward/update pass.'''					,			)
  parser.add_argument('''--seed'''					,			type=lowercase__					,			default=42					,			help='''random seed for initialization'''	)
  parser.add_argument(
      '''--data_dir'''					,			default=lowercase__					,			type=lowercase__					,			required=lowercase__					,			help='''The input data dir. Should contain the training files for the CoNLL-2003 NER task.'''					,			)



def  snake_case_(_UpperCamelCase					,			_UpperCamelCase					,			_UpperCamelCase=None					,			_UpperCamelCase=True					,			_UpperCamelCase=[]					,			_UpperCamelCase=None					,			_UpperCamelCase=None					,			**_UpperCamelCase					,			)				->      Tuple:



  """simple docstring"""






  pl.seed_everything(args.seed	)

  # init model
  _snake_case		=				Path(model.hparams.output_dir	)
  odir.mkdir(exist_ok=lowercase__	)

  # add custom checkpoints
  if checkpoint_callback is None:
    _snake_case		=				pl.callbacks.ModelCheckpoint(
        filepath=args.output_dir					,			prefix='''checkpoint'''					,			monitor='''val_loss'''					,			mode='''min'''					,			save_top_k=1	)
  if early_stopping_callback:
    extra_callbacks.append(lowercase__	)
  if logging_callback is None:
    _snake_case		=				LoggingCallback()

  _snake_case		=				{}

  # TODO: remove with PyTorch 1.6 since pl uses native amp
  if args.fpaa:
    _snake_case		=				16
    _snake_case		=				args.fpaa_opt_level

  if args.gpus > 1:
    _snake_case		=				"""ddp"""

  _snake_case		=				args.accumulate_grad_batches
  _snake_case		=				extra_train_kwargs.get('''accelerator'''					,			lowercase__	)
  _snake_case		=				extra_train_kwargs.get('''profiler'''					,			lowercase__	)

  _snake_case		=				pl.Trainer.from_argparse_args(
      lowercase__					,			weights_summary=lowercase__					,			callbacks=[logging_callback] + extra_callbacks					,			logger=lowercase__					,			checkpoint_callback=lowercase__					,			**lowercase__					,			)

  if args.do_train:
    trainer.fit(lowercase__	)

  return trainer



"""simple docstring"""






from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy




class      _SCREAMING_SNAKE_CASE   (     A__				):
			UpperCAmelCase_    :Union[str, Any]												=		"MarkupLMFeatureExtractor"
			UpperCAmelCase_    :Optional[Any]												=		("MarkupLMTokenizer", "MarkupLMTokenizerFast")
			UpperCAmelCase_    :List[Any]												=		True






			def __call__(   self			,  __A=None			,  __A=None			,  __A=None			,  __A=None			,  __A=None			,  __A = True			,  __A = False			,  __A = None			,  __A = None			,  __A = 0			,  __A = None			,  __A = None			,  __A = None			,  __A = False			,  __A = False			,  __A = False			,  __A = False			,  __A = True			,  __A = None			,  **__A			,  )     ->	BatchEncoding:
						# first, create nodes and xpaths
						if self.parse_html:
									if html_strings is None:
												raise ValueError("""Make sure to pass HTML strings in case `parse_html` is set to `True`"""			)

									if nodes is not None or xpaths is not None or node_labels is not None:
												raise ValueError(
												    """Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"""			)

									lowerCAmelCase_       :Union[str, Any]     	=    self.feature_extractor(__A			)
									lowerCAmelCase_       :Any     	=    features["""nodes"""]
									lowerCAmelCase_       :Union[str, Any]     	=    features["""xpaths"""]
						else:
									if html_strings is not None:
												raise ValueError("""You have passed HTML strings but `parse_html` is set to `False`."""			)
									if nodes is None or xpaths is None:
												raise ValueError("""Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"""			)

        # # second, apply the tokenizer
						if questions is not None and self.parse_html:
									if isinstance(__A			,  __A			):
												lowerCAmelCase_       :Dict     	=    [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

						lowerCAmelCase_       :Optional[Any]     	=    self.tokenizer(
						    text=questions if questions is not None else nodes			,  text_pair=nodes if questions is not None else None			,  xpaths=__A			,  node_labels=__A			,  add_special_tokens=__A			,  padding=__A			,  truncation=__A			,  max_length=__A			,  stride=__A			,  pad_to_multiple_of=__A			,  return_token_type_ids=__A			,  return_attention_mask=__A			,  return_overflowing_tokens=__A			,  return_special_tokens_mask=__A			,  return_offsets_mapping=__A			,  return_length=__A			,  verbose=__A			,  return_tensors=__A			,  **__A			,  )

						return encoded_inputs






			def 					__lowerCAmelCase					(   self			,  *__A			,  **__A			)     ->	Tuple:
						return self.tokenizer.batch_decode(*__A			,  **__A			)






			def 					__lowerCAmelCase					(   self			,  *__A			,  **__A			)     ->	str:
						return self.tokenizer.decode(*__A			,  **__A			)






			@property
			def 					__lowerCAmelCase					(   self			)     ->	Any:
						lowerCAmelCase_       :Any     	=    self.tokenizer.model_input_names
						return tokenizer_input_names






