"""simple docstring"""

import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


__lowercase    =				logging.getLogger(__name__)

require_version('''pytorch_lightning>=1.0.4''')

__lowercase    =				{
    '''base''': AutoModel,
    '''sequence-classification''': AutoModelForSequenceClassification,
    '''question-answering''': AutoModelForQuestionAnswering,
    '''pretraining''': AutoModelForPreTraining,
    '''token-classification''': AutoModelForTokenClassification,
    '''language-modeling''': AutoModelWithLMHead,
    '''summarization''': AutoModelForSeqaSeqLM,
    '''translation''': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
__lowercase    =				{
    '''linear''': get_linear_schedule_with_warmup,
    '''cosine''': get_cosine_schedule_with_warmup,
    '''cosine_w_restarts''': get_cosine_with_hard_restarts_schedule_with_warmup,
    '''polynomial''': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
__lowercase    =				sorted(arg_to_scheduler.keys())
__lowercase    =				'''{''' + ''', '''.join(arg_to_scheduler_choices) + '''}'''






class 		_lowercase (						pl.LightningModule      ):



						"""simple docstring"""




						def __init__(				self							:    int     , UpperCamelCase__							:    argparse.Namespace     , UpperCamelCase__							:    Dict=None     , UpperCamelCase__							:    int="base"     , UpperCamelCase__							:    str=None     , UpperCamelCase__							:    List[str]=None     , UpperCamelCase__							:    List[str]=None     , **UpperCamelCase__							:    Tuple     , )				->				Dict:
													'''simple docstring'''



													super().__init__()
													# TODO: move to self.save_hyperparameters()
													# self.save_hyperparameters()
													# can also expand arguments into trainer signature for easier reading

													self.save_hyperparameters(UpperCamelCase__				)
													__UpperCamelCase							=0
													__UpperCamelCase							=Path(self.hparams.output_dir				)
													__UpperCamelCase							=self.hparams.cache_dir if self.hparams.cache_dir else None
													if config is None:
																				__UpperCamelCase							=AutoConfig.from_pretrained(
																				    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path     , **({'''num_labels''': num_labels} if num_labels is not None else {})     , cache_dir=UpperCamelCase__     , **UpperCamelCase__     , )
													else:
																				__UpperCamelCase							=config

													__UpperCamelCase							=('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
													for p in extra_model_params:
																				if getattr(self.hparams     , UpperCamelCase__     , UpperCamelCase__				):
																											assert hasattr(self.config     , UpperCamelCase__				), f"""model config doesn't have a `{p}` attribute"""
																											setattr(self.config     , UpperCamelCase__     , getattr(self.hparams     , UpperCamelCase__				)				)

													if tokenizer is None:
																				__UpperCamelCase							=AutoTokenizer.from_pretrained(
																				    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path     , cache_dir=UpperCamelCase__     , )
													else:
																				__UpperCamelCase							=tokenizer
													__UpperCamelCase							=MODEL_MODES[mode]
													if model is None:
																				__UpperCamelCase							=self.model_type.from_pretrained(
																				    self.hparams.model_name_or_path     , from_tf=bool('''.ckpt''' in self.hparams.model_name_or_path				)     , config=self.config     , cache_dir=UpperCamelCase__     , )
													else:
																				__UpperCamelCase							=model





						def   UpperCAmelCase_ (				self							:    Optional[int]     , *UpperCamelCase__							:    Any     , **UpperCamelCase__							:    Optional[Any]				)				->				List[Any]:
													'''simple docstring'''



													__UpperCamelCase							=self.model_type.from_pretrained(*UpperCamelCase__     , **UpperCamelCase__				)





						def   UpperCAmelCase_ (				self							:    Dict				)				->				List[Any]:
													'''simple docstring'''



													__UpperCamelCase							=arg_to_scheduler[self.hparams.lr_scheduler]
													__UpperCamelCase							=get_schedule_func(
													    self.opt     , num_warmup_steps=self.hparams.warmup_steps     , num_training_steps=self.total_steps()				)
													__UpperCamelCase							={'''scheduler''': scheduler, '''interval''': '''step''', '''frequency''': 1}
													return scheduler





						def   UpperCAmelCase_ (				self							:    Union[str, Any]				)				->				Tuple:
													'''simple docstring'''



													__UpperCamelCase							=self.model
													__UpperCamelCase							=['''bias''', '''LayerNorm.weight''']
													__UpperCamelCase							=[
													    {
													        '''params''': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay				)],
													        '''weight_decay''': self.hparams.weight_decay,
													    },
													    {
													        '''params''': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay				)],
													        '''weight_decay''': 0.0,
													    },
													]
													if self.hparams.adafactor:
																				__UpperCamelCase							=Adafactor(
																				    UpperCamelCase__     , lr=self.hparams.learning_rate     , scale_parameter=UpperCamelCase__     , relative_step=UpperCamelCase__				)

													else:
																				__UpperCamelCase							=AdamW(
																				    UpperCamelCase__     , lr=self.hparams.learning_rate     , eps=self.hparams.adam_epsilon				)
													__UpperCamelCase							=optimizer

													__UpperCamelCase							=self.get_lr_scheduler()

													return [optimizer], [scheduler]





						def   UpperCAmelCase_ (				self							:    Any     , UpperCamelCase__							:    str     , UpperCamelCase__							:    List[Any]				)				->				str:
													'''simple docstring'''



													return self.validation_step(UpperCamelCase__     , UpperCamelCase__				)





						def   UpperCAmelCase_ (				self							:    int     , UpperCamelCase__							:    int				)				->				str:
													'''simple docstring'''



													return self.validation_end(UpperCamelCase__				)





						def   UpperCAmelCase_ (				self							:    Dict				)				->				int:
													'''simple docstring'''



													__UpperCamelCase							=max(1     , self.hparams.gpus				)  # TODO: consider num_tpu_cores
													__UpperCamelCase							=self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
													return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs





						def   UpperCAmelCase_ (				self							:    int     , UpperCamelCase__							:    Union[str, Any]				)				->				Optional[Any]:
													'''simple docstring'''



													if mode == "test":
																				__UpperCamelCase							=len(self.test_dataloader().dataset				)
													else:
																				__UpperCamelCase							=self.get_dataloader('''train'''     , self.hparams.train_batch_size     , shuffle=UpperCamelCase__				)
																				__UpperCamelCase							=len(self.train_dataloader().dataset				)





						def   UpperCAmelCase_ (				self							:    Tuple     , UpperCamelCase__							:    str     , UpperCamelCase__							:    int     , UpperCamelCase__							:    bool = False				)				->				str:
													'''simple docstring'''



													raise NotImplementedError('''You must implement this for your task'''				)





						def   UpperCAmelCase_ (				self							:    Dict				)				->				Tuple:
													'''simple docstring'''



													return self.train_loader





						def   UpperCAmelCase_ (				self							:    Union[str, Any]				)				->				List[Any]:
													'''simple docstring'''



													return self.get_dataloader('''dev'''     , self.hparams.eval_batch_size     , shuffle=UpperCamelCase__				)





						def   UpperCAmelCase_ (				self							:    List[str]				)				->				Optional[int]:
													'''simple docstring'''



													return self.get_dataloader('''test'''     , self.hparams.eval_batch_size     , shuffle=UpperCamelCase__				)





						def   UpperCAmelCase_ (				self							:    Optional[int]     , UpperCamelCase__							:    Dict				)				->				Dict:
													'''simple docstring'''



													return os.path.join(
													    self.hparams.data_dir     , '''cached_{}_{}_{}'''.format(
													        UpperCamelCase__     , list(filter(UpperCamelCase__     , self.hparams.model_name_or_path.split('''/'''				)				)				).pop()     , str(self.hparams.max_seq_length				)     , )     , )





						@pl.utilities.rank_zero_only
						def   UpperCAmelCase_ (				self							:    List[Any]     , UpperCamelCase__							:    Dict[str, Any]				)				->				None:
													'''simple docstring'''



													__UpperCamelCase							=self.output_dir.joinpath('''best_tfmr'''				)
													__UpperCamelCase							=self.step_count
													self.model.save_pretrained(UpperCamelCase__				)
													self.tokenizer.save_pretrained(UpperCamelCase__				)





						@staticmethod
						def   UpperCAmelCase_ (				UpperCamelCase__							:    Tuple     , UpperCamelCase__							:    Optional[Any]				)				->				Dict:
													'''simple docstring'''



													parser.add_argument(
													    '''--model_name_or_path'''     , default=UpperCamelCase__     , type=UpperCamelCase__     , required=UpperCamelCase__     , help='''Path to pretrained model or model identifier from huggingface.co/models'''     , )
													parser.add_argument(
													    '''--config_name'''     , default=''''''     , type=UpperCamelCase__     , help='''Pretrained config name or path if not the same as model_name'''				)
													parser.add_argument(
													    '''--tokenizer_name'''     , default=UpperCamelCase__     , type=UpperCamelCase__     , help='''Pretrained tokenizer name or path if not the same as model_name'''     , )
													parser.add_argument(
													    '''--cache_dir'''     , default=''''''     , type=UpperCamelCase__     , help='''Where do you want to store the pre-trained models downloaded from huggingface.co'''     , )
													parser.add_argument(
													    '''--encoder_layerdrop'''     , type=UpperCamelCase__     , help='''Encoder layer dropout probability (Optional). Goes into model.config'''     , )
													parser.add_argument(
													    '''--decoder_layerdrop'''     , type=UpperCamelCase__     , help='''Decoder layer dropout probability (Optional). Goes into model.config'''     , )
													parser.add_argument(
													    '''--dropout'''     , type=UpperCamelCase__     , help='''Dropout probability (Optional). Goes into model.config'''     , )
													parser.add_argument(
													    '''--attention_dropout'''     , type=UpperCamelCase__     , help='''Attention dropout probability (Optional). Goes into model.config'''     , )
													parser.add_argument('''--learning_rate'''     , default=5E-5     , type=UpperCamelCase__     , help='''The initial learning rate for Adam.'''				)
													parser.add_argument(
													    '''--lr_scheduler'''     , default='''linear'''     , choices=UpperCamelCase__     , metavar=UpperCamelCase__     , type=UpperCamelCase__     , help='''Learning rate scheduler'''     , )
													parser.add_argument('''--weight_decay'''     , default=0.0     , type=UpperCamelCase__     , help='''Weight decay if we apply some.'''				)
													parser.add_argument('''--adam_epsilon'''     , default=1E-8     , type=UpperCamelCase__     , help='''Epsilon for Adam optimizer.'''				)
													parser.add_argument('''--warmup_steps'''     , default=0     , type=UpperCamelCase__     , help='''Linear warmup over warmup_steps.'''				)
													parser.add_argument('''--num_workers'''     , default=4     , type=UpperCamelCase__     , help='''kwarg passed to DataLoader'''				)
													parser.add_argument('''--num_train_epochs'''     , dest='''max_epochs'''     , default=3     , type=UpperCamelCase__				)
													parser.add_argument('''--train_batch_size'''     , default=32     , type=UpperCamelCase__				)
													parser.add_argument('''--eval_batch_size'''     , default=32     , type=UpperCamelCase__				)
													parser.add_argument('''--adafactor'''     , action='''store_true'''				)






class 		_lowercase (						pl.Callback      ):



						"""simple docstring"""




						def   UpperCAmelCase_ (				self							:    Union[str, Any]     , UpperCamelCase__							:    Optional[int]     , UpperCamelCase__							:    Tuple				)				->				Optional[int]:
													'''simple docstring'''



													__UpperCamelCase							=trainer.lr_schedulers[0]['''scheduler''']
													__UpperCamelCase							={f"""lr_group_{i}""": lr for i, lr in enumerate(lr_scheduler.get_lr()				)}
													pl_module.logger.log_metrics(UpperCamelCase__				)





						def   UpperCAmelCase_ (				self							:    Any     , UpperCamelCase__							:    pl.Trainer     , UpperCamelCase__							:    pl.LightningModule				)				->				Any:
													'''simple docstring'''



													rank_zero_info('''***** Validation results *****'''				)
													__UpperCamelCase							=trainer.callback_metrics
													# Log results
													for key in sorted(UpperCamelCase__				):
																				if key not in ["log", "progress_bar"]:
																											rank_zero_info('''{} = {}\n'''.format(UpperCamelCase__     , str(metrics[key]				)				)				)





						def   UpperCAmelCase_ (				self							:    Any     , UpperCamelCase__							:    pl.Trainer     , UpperCamelCase__							:    pl.LightningModule				)				->				int:
													'''simple docstring'''



													rank_zero_info('''***** Test results *****'''				)
													__UpperCamelCase							=trainer.callback_metrics
													# Log and save results to file
													__UpperCamelCase							=os.path.join(pl_module.hparams.output_dir     , '''test_results.txt'''				)
													with open(UpperCamelCase__     , '''w'''				) as writer:
																				for key in sorted(UpperCamelCase__				):
																											if key not in ["log", "progress_bar"]:
																																		rank_zero_info('''{} = {}\n'''.format(UpperCamelCase__     , str(metrics[key]				)				)				)
																																		writer.write('''{} = {}\n'''.format(UpperCamelCase__     , str(metrics[key]				)				)				)






def    lowerCAmelCase			(__UpperCamelCase							:    List[str]							,					__UpperCamelCase							:    str			):
							"""simple docstring"""


							parser.add_argument(
							    '''--output_dir'''							,					default=__UpperCamelCase							,					type=__UpperCamelCase							,					required=__UpperCamelCase							,					help='''The output directory where the model predictions and checkpoints will be written.'''							,					)
							parser.add_argument(
							    '''--fp16'''							,					action='''store_true'''							,					help='''Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'''							,					)

							parser.add_argument(
							    '''--fp16_opt_level'''							,					type=__UpperCamelCase							,					default='''O2'''							,					help=(
							        '''For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'''
							        '''See details at https://nvidia.github.io/apex/amp.html'''
							    )							,					)
							parser.add_argument('''--n_tpu_cores'''							,					dest='''tpu_cores'''							,					type=__UpperCamelCase			)
							parser.add_argument('''--max_grad_norm'''							,					dest='''gradient_clip_val'''							,					default=1.0							,					type=__UpperCamelCase							,					help='''Max gradient norm'''			)
							parser.add_argument('''--do_train'''							,					action='''store_true'''							,					help='''Whether to run training.'''			)
							parser.add_argument('''--do_predict'''							,					action='''store_true'''							,					help='''Whether to run predictions on the test set.'''			)
							parser.add_argument(
							    '''--gradient_accumulation_steps'''							,					dest='''accumulate_grad_batches'''							,					type=__UpperCamelCase							,					default=1							,					help='''Number of updates steps to accumulate before performing a backward/update pass.'''							,					)
							parser.add_argument('''--seed'''							,					type=__UpperCamelCase							,					default=4_2							,					help='''random seed for initialization'''			)
							parser.add_argument(
							    '''--data_dir'''							,					default=__UpperCamelCase							,					type=__UpperCamelCase							,					required=__UpperCamelCase							,					help='''The input data dir. Should contain the training files for the CoNLL-2003 NER task.'''							,					)






def    lowerCAmelCase			(__UpperCamelCase							:    BaseTransformer							,					__UpperCamelCase							:    argparse.Namespace							,					__UpperCamelCase							:    int=None							,					__UpperCamelCase							:    Optional[Any]=True							,					__UpperCamelCase							:    Optional[int]=[]							,					__UpperCamelCase							:    List[Any]=None							,					__UpperCamelCase							:    Tuple=None							,					**__UpperCamelCase							:    List[str]							,					):
							"""simple docstring"""


							pl.seed_everything(args.seed			)

							# init model
							__UpperCamelCase							=Path(model.hparams.output_dir			)
							odir.mkdir(exist_ok=__UpperCamelCase			)

							# add custom checkpoints
							if checkpoint_callback is None:
														__UpperCamelCase							=pl.callbacks.ModelCheckpoint(
														    filepath=args.output_dir							,					prefix='''checkpoint'''							,					monitor='''val_loss'''							,					mode='''min'''							,					save_top_k=1			)
							if early_stopping_callback:
														extra_callbacks.append(__UpperCamelCase			)
							if logging_callback is None:
														__UpperCamelCase							=LoggingCallback()

							__UpperCamelCase							={}

							# TODO: remove with PyTorch 1.6 since pl uses native amp
							if args.fpaa:
														__UpperCamelCase							=1_6
														__UpperCamelCase							=args.fpaa_opt_level

							if args.gpus > 1:
														__UpperCamelCase							='''ddp'''

							__UpperCamelCase							=args.accumulate_grad_batches
							__UpperCamelCase							=extra_train_kwargs.get('''accelerator'''							,					__UpperCamelCase			)
							__UpperCamelCase							=extra_train_kwargs.get('''profiler'''							,					__UpperCamelCase			)

							__UpperCamelCase							=pl.Trainer.from_argparse_args(
							    __UpperCamelCase							,					weights_summary=__UpperCamelCase							,					callbacks=[logging_callback] + extra_callbacks							,					logger=__UpperCamelCase							,					checkpoint_callback=__UpperCamelCase							,					**__UpperCamelCase							,					)

							if args.do_train:
														trainer.fit(__UpperCamelCase			)

							return trainer




"""simple docstring"""

def    lowerCAmelCase			(__UpperCamelCase							:    int							,					__UpperCamelCase							:    int			):
							"""simple docstring"""


							return (input_a, input_a).count(0			) % 2


def    lowerCAmelCase			():
							"""simple docstring"""


							assert xor_gate(0							,					0			) == 0
							assert xor_gate(0							,					1			) == 1
							assert xor_gate(1							,					0			) == 1
							assert xor_gate(1							,					1			) == 0


if __name__ == "__main__":
				print(xor_gate(0, 0))
				print(xor_gate(0, 1))



