
'''simple docstring'''





import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


a__		: Optional[Any]  							=							logging.get_logger(__name__)


class 			UpperCamelCase__	(     nn.Module):






 def __init__(      self	:Optional[Any]      )					->      Optional[Any]:







       '''simple docstring'''



       super().__init__()
       if version.parse(torch.__version__      ) < version.parse('1.12.0'      ):
             raise ImportError(
                 F'You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '
                 'PytorchGELUTanh. Please upgrade torch.'      )






 def 						lowercase_    (      self	:Optional[Any]						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return nn.functional.gelu(_A						,					approximate='tanh'      )


class 			UpperCamelCase__	(     nn.Module):






 def 						lowercase_    (      self	:Optional[Any]						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi      ) * (input + 0.044_715 * torch.pow(_A						,					3.0      ))      ))


class 			UpperCamelCase__	(     nn.Module):






 def __init__(      self	:Optional[int]						,					_A	:bool = False      )					->      Tuple:







       '''simple docstring'''



       super().__init__()
       if use_gelu_python:
             __A													=      self._gelu_python
       else:
             __A													=      nn.functional.gelu






 def 						lowercase_    (      self	:int						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0      )      ))






 def 						lowercase_    (      self	:Optional[int]						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return self.act(_A      )


class 			UpperCamelCase__	(     nn.Module):






 def 						lowercase_    (      self	:Tuple						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return 0.5 * input * (1.0 + torch.tanh(input * 0.7_978_845_608 * (1.0 + 0.044_715 * input * input)      ))


class 			UpperCamelCase__	(     nn.Module):






 def 						lowercase_    (      self	:Optional[int]						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return input * torch.sigmoid(1.702 * input      )


class 			UpperCamelCase__	(     nn.Module):






 def __init__(      self	:List[str]						,					_A	:float						,					_A	:float      )					->      List[Any]:







       '''simple docstring'''



       if min > max:
             raise ValueError(F'min should be < max (got min: {min}, max: {max})'      )

       super().__init__()
       __A													=      min
       __A													=      max






 def 						lowercase_    (      self	:Dict						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return torch.clip(gelu(_A      )						,					self.min						,					self.max      )


class 			UpperCamelCase__	(     nn.Module):






 def __init__(      self	:Dict      )					->      Optional[int]:







       '''simple docstring'''



       super().__init__()
       __A													=      math.sqrt(2 / math.pi      )






 def 						lowercase_    (      self	:Tuple						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044_715 * torch.pow(_A						,					3      ))      ))


class 			UpperCamelCase__	(     nn.Module):






 def 						lowercase_    (      self	:int						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return nn.functional.silu(_A      )


class 			UpperCamelCase__	(     nn.Module):






 def __init__(      self	:Any      )					->      Optional[int]:







       '''simple docstring'''



       super().__init__()
       if version.parse(torch.__version__      ) < version.parse('1.9.0'      ):
             __A													=      self._mish_python
       else:
             __A													=      nn.functional.mish






 def 						lowercase_    (      self	:Optional[Any]						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return input * torch.tanh(nn.functional.softplus(_A      )      )






 def 						lowercase_    (      self	:Dict						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return self.act(_A      )


class 			UpperCamelCase__	(     nn.Module):






 def 						lowercase_    (      self	:str						,					_A	:Tensor      )					->      Tensor:







       '''simple docstring'''



       return input


class 			UpperCamelCase__	(     nn.Module):






 def 						lowercase_    (      self	:List[str]						,					_A	:Optional[Any]						,					_A	:Tuple=0.707_107						,					_A	:List[str]=0.282_095      )					->      Union[str, Any]:







       '''simple docstring'''



       __A													=      (input - mu).div(sigma * math.sqrt(2.0      )      )
       return 0.5 * (1.0 + torch.erf(_A      ))


class 			UpperCamelCase__	(     nn.Module):






 def 						lowercase_    (      self	:Tuple						,					_A	:Any      )					->      Optional[Any]:







       '''simple docstring'''



       __A													=      nn.functional.relu(_A      )
       __A													=      torch.square(_A      )
       return squared


class 			UpperCamelCase__	(     SCREAMING_SNAKE_CASE):






 def __getitem__(      self	:Any						,					_A	:Optional[int]      )					->      Dict:







       '''simple docstring'''



       __A													=      super().__getitem__(_A      )
       __A						,   __A													=      content if isinstance(_A						,					_A      ) else (content, {})
       return cls(**_A      )
a__		: Optional[int]  							=							{
    "gelu": GELUActivation,
    "gelu_10": (ClippedGELUActivation, {"min": -1_0, "max": 1_0}),
    "gelu_fast": FastGELUActivation,
    "gelu_new": NewGELUActivation,
    "gelu_python": (GELUActivation, {"use_gelu_python": True}),
    "gelu_pytorch_tanh": PytorchGELUTanh,
    "gelu_accurate": AccurateGELUActivation,
    "laplace": LaplaceActivation,
    "linear": LinearActivation,
    "mish": MishActivation,
    "quick_gelu": QuickGELUActivation,
    "relu": nn.ReLU,
    "relu2": ReLUSquaredActivation,
    "relu6": nn.ReLUa,
    "sigmoid": nn.Sigmoid,
    "silu": SiLUActivation,
    "swish": SiLUActivation,
    "tanh": nn.Tanh,
}
a__		: str  							=							ClassInstantier(ACTaCLS)





def 							snake_case				(		UpperCAmelCase						)->      Any:







      """simple docstring"""
      if activation_string in ACTaFN:
            return ACTaFN[activation_string]
      else:
            raise KeyError(f'function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()						)}'						)


# For backwards compatibility with: from activations import gelu_python
a__		: Dict  							=							get_activation("gelu_python")
a__		: int  							=							get_activation("gelu_new")
a__		: Optional[int]  							=							get_activation("gelu")
a__		: Optional[Any]  							=							get_activation("gelu_fast")
a__		: Optional[Any]  							=							get_activation("quick_gelu")
a__		: Dict  							=							get_activation("silu")
a__		: Dict  							=							get_activation("mish")
a__		: int  							=							get_activation("linear")











import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


__lowerCamelCase								=      logging.getLogger(__name__)


@dataclass
class 	snake_case_			:




       """simple docstring"""






       _lowerCamelCase           =					field(
           metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}	)
       _lowerCamelCase           =					field(
           default=lowercase__			,			metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}	)
       _lowerCamelCase           =					field(
           default=lowercase__			,			metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}	)
       _lowerCamelCase           =					field(default=lowercase__			,			metadata={"""help""": """Set this flag to use fast tokenization."""}	)
       # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
       # or just modify its tokenizer_config.json.
       _lowerCamelCase           =					field(
           default=lowercase__			,			metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}			,			)







def  _snake_case			(							)  ->  Union[str, Any]:






      '''simple docstring'''







      UpperCAmelCase_	:						Union[str, Any]					   =		HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )

      if len(sys.argv     ) == 2 and sys.argv[1].endswith(".json"     ):
            # If we pass only one argument to the script and it's the path to a json file,
            # let's parse it to get our arguments.
            UpperCAmelCase_   ,   UpperCAmelCase_   ,   UpperCAmelCase_	:						Any					   =		parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]     )     )
      else:
            UpperCAmelCase_   ,   UpperCAmelCase_   ,   UpperCAmelCase_	:						Union[str, Any]					   =		parser.parse_args_into_dataclasses()

      if (
          os.path.exists(training_args.output_dir     )
          and os.listdir(training_args.output_dir     )
          and training_args.do_train
          and not training_args.overwrite_output_dir
      ):
            raise ValueError(
                F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
                " --overwrite_output_dir to overcome."     )

      # Setup logging
      logging.basicConfig(
          format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"       ,	datefmt="%m/%d/%Y %H:%M:%S"       ,	level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN       ,	)
      logger.warning(
          "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"       ,	training_args.local_rank       ,	training_args.device       ,	training_args.n_gpu       ,	bool(training_args.local_rank != -1     )       ,	training_args.fpaa       ,	)
      # Set the verbosity to info of the Transformers logger (on main process only):
      if is_main_process(training_args.local_rank     ):
            transformers.utils.logging.set_verbosity_info()
            transformers.utils.logging.enable_default_handler()
            transformers.utils.logging.enable_explicit_format()
      logger.info("Training/evaluation parameters %s"       ,	__snake_case     )

      # Prepare Question-Answering task
      # Load pretrained model and tokenizer
      #
      # Distributed training:
      # The .from_pretrained methods guarantee that only one local process can concurrently
      # download model & vocab.

      UpperCAmelCase_	:						Any					   =		AutoConfig.from_pretrained(
          model_args.config_name if model_args.config_name else model_args.model_name_or_path       ,	cache_dir=model_args.cache_dir       ,	)
      UpperCAmelCase_	:						List[str]					   =		AutoTokenizer.from_pretrained(
          model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path       ,	cache_dir=model_args.cache_dir       ,	use_fast=__snake_case       ,	)
      UpperCAmelCase_	:						Optional[int]					   =		AutoModelForQuestionAnswering.from_pretrained(
          model_args.model_name_or_path       ,	from_tf=bool(".ckpt" in model_args.model_name_or_path     )       ,	config=__snake_case       ,	cache_dir=model_args.cache_dir       ,	)

      # Get datasets
      UpperCAmelCase_	:						str					   =		hasattr(model.config       ,	"lang2id"     )
      UpperCAmelCase_	:						Optional[Any]					   =		(
          SquadDataset(
              __snake_case       ,	tokenizer=__snake_case       ,	is_language_sensitive=__snake_case       ,	cache_dir=model_args.cache_dir     )
          if training_args.do_train
          else None
      )
      UpperCAmelCase_	:						Optional[Any]					   =		(
          SquadDataset(
              __snake_case       ,	tokenizer=__snake_case       ,	mode="dev"       ,	is_language_sensitive=__snake_case       ,	cache_dir=model_args.cache_dir       ,	)
          if training_args.do_eval
          else None
      )

      # Data collator
      UpperCAmelCase_	:						Optional[Any]					   =		DataCollatorWithPadding(__snake_case       ,	pad_to_multiple_of=8     ) if training_args.fpaa else None

      # Initialize our Trainer
      UpperCAmelCase_	:						Any					   =		Trainer(
          model=__snake_case       ,	args=__snake_case       ,	train_dataset=__snake_case       ,	eval_dataset=__snake_case       ,	data_collator=__snake_case       ,	)

      # Training
      if training_args.do_train:
            trainer.train(
                model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path     ) else None     )
            trainer.save_model()
            # For convenience, we also re-save the tokenizer to the same directory,
            # so that you can share your model easily on huggingface.co/models =)
            if trainer.is_world_master():
                  tokenizer.save_pretrained(training_args.output_dir     )







def  _snake_case			(							__snake_case     )  ->  List[str]:






      '''simple docstring'''







      main()


if __name__ == "__main__":
   main()
