





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase_       : List[Any]							=    logging.get_logger(__name__)

UpperCAmelCase_       : Union[str, Any]							=    {
    """mnaylor/mega-base-wikitext""": """https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json""",
}





class   lowercase__     (				__A	):
  __UpperCamelCase						  =	"""mega"""







  def __init__(						self     ,    _lowercase=30_522     ,    _lowercase=128     ,    _lowercase=4     ,    _lowercase=256     ,    _lowercase=16     ,    _lowercase=True     ,    _lowercase=64     ,    _lowercase=False     ,    _lowercase=-1     ,    _lowercase=None     ,    _lowercase=True     ,    _lowercase="scalenorm"     ,    _lowercase=True     ,    _lowercase="silu"     ,    _lowercase="softmax"     ,    _lowercase=0.1     ,    _lowercase=0.1     ,    _lowercase=0.1     ,    _lowercase=False     ,    _lowercase=True     ,    _lowercase=256     ,    _lowercase=True     ,    _lowercase=0.1     ,    _lowercase=2_048     ,    _lowercase=False     ,    _lowercase=2     ,    _lowercase=0.02     ,    _lowercase=0.2     ,    _lowercase=0.02     ,    _lowercase=1.0     ,    _lowercase=1     ,    _lowercase=0     ,    _lowercase=2     ,    _lowercase="rotary"     ,    _lowercase=None     ,    _lowercase=True     ,    _lowercase=True     ,    **_lowercase     ,    ):
       super().__init__(pad_token_id=_lowercase     ,    bos_token_id=_lowercase     ,    eos_token_id=_lowercase     ,    **_lowercase    )

       lowerCAmelCase_       :							Union[str, Any]			 =		vocab_size
       lowerCAmelCase_       :							Optional[Any]			 =		hidden_size
       lowerCAmelCase_       :							List[str]			 =		num_hidden_layers
       lowerCAmelCase_       :							Optional[int]			 =		activation
       lowerCAmelCase_       :							Tuple			 =		attention_activation
       lowerCAmelCase_       :							Any			 =		intermediate_size
       lowerCAmelCase_       :							int			 =		ema_projection_size
       lowerCAmelCase_       :							int			 =		bidirectional
       lowerCAmelCase_       :							Union[str, Any]			 =		shared_representation_size
       lowerCAmelCase_       :							List[str]			 =		use_chunking
       lowerCAmelCase_       :							Optional[Any]			 =		chunk_size
       lowerCAmelCase_       :							Optional[int]			 =		truncation
       lowerCAmelCase_       :							List[Any]			 =		normalize_before_mega
       lowerCAmelCase_       :							Optional[Any]			 =		normalization_type
       lowerCAmelCase_       :							Tuple			 =		norm_affine
       lowerCAmelCase_       :							str			 =		dropout_prob
       lowerCAmelCase_       :							Optional[int]			 =		hidden_dropout_prob
       lowerCAmelCase_       :							List[str]			 =		attention_probs_dropout_prob
       lowerCAmelCase_       :							List[Any]			 =		use_feature_dropout
       lowerCAmelCase_       :							Optional[Any]			 =		use_normalized_ffn
       lowerCAmelCase_       :							str			 =		nffn_hidden_size
       lowerCAmelCase_       :							Optional[Any]			 =		normalize_before_ffn
       lowerCAmelCase_       :							Any			 =		nffn_activation_dropout_prob
       lowerCAmelCase_       :							List[str]			 =		max_positions
       lowerCAmelCase_       :							List[str]			 =		add_token_type_embeddings
       lowerCAmelCase_       :							Union[str, Any]			 =		type_vocab_size
       lowerCAmelCase_       :							Dict			 =		initializer_range
       lowerCAmelCase_       :							Tuple			 =		ema_delta_alpha_range
       lowerCAmelCase_       :							Union[str, Any]			 =		ema_beta_range
       lowerCAmelCase_       :							Tuple			 =		ema_gamma_omega_range
       lowerCAmelCase_       :							Any			 =		relative_positional_bias
       lowerCAmelCase_       :							Optional[int]			 =		use_cache
       lowerCAmelCase_       :							int			 =		classifier_dropout
       lowerCAmelCase_       :							Union[str, Any]			 =		add_lm_hidden_dense_layer
       lowerCAmelCase_       :							Tuple			 =		1  # not used but required by Hugging Face





class   lowercase__     (				__A	):







  @property
  def 						UpperCAmelCase__		(						self    ):
       if self.task == "multiple-choice":
            lowerCAmelCase_       :							Optional[Any]			 =		{0: "batch", 1: "choice", 2: "sequence"}
       else:
            lowerCAmelCase_       :							str			 =		{0: "batch", 1: "sequence"}
       return OrderedDict(
           [
               ("""input_ids""", dynamic_axis),
               ("""attention_mask""", dynamic_axis),
           ]    )





import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
      import torch_xla.core.xla_model as xm






def 						_lowerCAmelCase    (					__lowerCamelCase :  Any		,						__lowerCamelCase :  str  ):



 """simple docstring"""





 if isinstance(__lowerCamelCase		,						(list, tuple)  ):
  return honor_type(__lowerCamelCase		,						(move_to_device(__lowerCamelCase		,						__lowerCamelCase  ) for t in state)  )
 elif isinstance(__lowerCamelCase		,						__lowerCamelCase  ):
  return type(__lowerCamelCase  )({k: move_to_device(__lowerCamelCase		,						__lowerCamelCase  ) for k, v in state.items()}  )
 elif isinstance(__lowerCamelCase		,						torch.Tensor  ):
  return state.to(__lowerCamelCase  )
 return state






class    _SCREAMING_SNAKE_CASE						(torch.optim.Optimizer			):





     def __init__(							self			:				int  ,	UpperCamelCase			:				Any  ,	UpperCamelCase			:				List[str]=True  ,	UpperCamelCase			:				List[Any]=None					)->Optional[int]:
      __SCREAMING_SNAKE_CASE							:     str							=						optimizer
      __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						scaler
      __SCREAMING_SNAKE_CASE							:     Any							=						AcceleratorState()
      __SCREAMING_SNAKE_CASE							:     int							=						GradientState()
      __SCREAMING_SNAKE_CASE							:     str							=						device_placement
      __SCREAMING_SNAKE_CASE							:     int							=						False
      __SCREAMING_SNAKE_CASE							:     Dict							=						None

      # Handle device placement
      if device_placement:
       __SCREAMING_SNAKE_CASE							:     List[str]							=						self.optimizer.state_dict()
       if self.accelerator_state.distributed_type == DistributedType.TPU:
        xm.send_cpu_data_to_device(UpperCamelCase  ,	self.accelerator_state.device					)
       else:
        __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						move_to_device(UpperCamelCase  ,	self.accelerator_state.device					)
       self.optimizer.load_state_dict(UpperCamelCase					)





     @property
     def 						__snake_case     (							self			:				Optional[Any]					)->List[str]:
      return self.optimizer.state





     @state.setter
     def 						__snake_case     (							self			:				Optional[int]  ,	UpperCamelCase			:				List[str]					)->Dict:
      __SCREAMING_SNAKE_CASE							:     List[str]							=						state





     @property
     def 						__snake_case     (							self			:				List[Any]					)->str:
      return self.optimizer.param_groups





     @param_groups.setter
     def 						__snake_case     (							self			:				List[str]  ,	UpperCamelCase			:				Any					)->Dict:
      __SCREAMING_SNAKE_CASE							:     Optional[int]							=						param_groups





     @property
     def 						__snake_case     (							self			:				str					)->str:
      return self.optimizer.defaults





     @defaults.setter
     def 						__snake_case     (							self			:				int  ,	UpperCamelCase			:				Tuple					)->List[str]:
      __SCREAMING_SNAKE_CASE							:     List[str]							=						defaults





     def 						__snake_case     (							self			:				Any  ,	UpperCamelCase			:				Tuple					)->Optional[int]:
      self.optimizer.add_param_group(UpperCamelCase					)





     def 						__snake_case     (							self			:				Union[str, Any]  ,	UpperCamelCase			:				Union[str, Any]					)->Dict:
      if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
       xm.send_cpu_data_to_device(UpperCamelCase  ,	self.accelerator_state.device					)
      self.optimizer.load_state_dict(UpperCamelCase					)





     def 						__snake_case     (							self			:				List[str]					)->List[Any]:
      return self.optimizer.state_dict()





     def 						__snake_case     (							self			:				Optional[int]  ,	UpperCamelCase			:				Any=None					)->Any:
      if self.gradient_state.sync_gradients:
       __SCREAMING_SNAKE_CASE							:     Optional[int]							=						"set_to_none" in inspect.signature(self.optimizer.zero_grad					).parameters
       if accept_arg:
        if set_to_none is None:
         __SCREAMING_SNAKE_CASE							:     Optional[int]							=						False
        self.optimizer.zero_grad(set_to_none=UpperCamelCase					)
       else:
        if set_to_none is not None:
         raise ValueError("`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer."					)
        self.optimizer.zero_grad()





     def 						__snake_case     (							self			:				Dict  ,	UpperCamelCase			:				Tuple=None					)->List[Any]:
      if self.gradient_state.sync_gradients:
       if self.accelerator_state.distributed_type == DistributedType.TPU:
        __SCREAMING_SNAKE_CASE							:     List[str]							=						{"closure": closure} if closure is not None else {}
        xm.optimizer_step(self.optimizer  ,	optimizer_args=UpperCamelCase					)
       elif self.scaler is not None:
        __SCREAMING_SNAKE_CASE							:     Optional[int]							=						False
        if self._last_scale is None:
         # `get_scale` is an async operation requiring full synchronization
         # on CPU and GPUs before finishing. As a result, we store away
         # the prior one to reduce the call overhead
         __SCREAMING_SNAKE_CASE							:     List[Any]							=						self.scaler.get_scale()
         __SCREAMING_SNAKE_CASE							:     List[Any]							=						True
        self.scaler.step(self.optimizer  ,	UpperCamelCase					)
        self.scaler.update()
        __SCREAMING_SNAKE_CASE							:     Dict							=						self.scaler.get_scale()
        if not new_scale:
         # If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
         __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						scale_after < self._last_scale
        __SCREAMING_SNAKE_CASE							:     List[Any]							=						scale_after

       else:
        self.optimizer.step(UpperCamelCase					)





     def 						__snake_case     (							self			:				Optional[int]  ,	UpperCamelCase			:				Optional[Any]					)->Any:
      for param_group in self.optimizer.param_groups:
       __SCREAMING_SNAKE_CASE							:     Tuple							=						[parameters_map.get(UpperCamelCase  ,	UpperCamelCase					) for p in param_group["params"]]





     @property
     def 						__snake_case     (							self			:				List[Any]					)->int:
      warnings.warn(
          "The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use "
          "`optimizer.step_was_skipped` instead."  ,	UpperCamelCase  ,	)
      return self._is_overflow





     @property
     def 						__snake_case     (							self			:				Optional[int]					)->List[Any]:
      return self._is_overflow





     def __getstate__(							self			:				List[str]					)->List[str]:
      return self.__dict__.copy()






     def __setstate__(							self			:				Union[str, Any]  ,	UpperCamelCase			:				str					)->int:
      self.__dict__.update(UpperCamelCase					)






