


import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
					import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


UpperCamelCase			:					Dict         =							get_logger(__name__)
def 				UpperCamelCase_	(					__snake_case      :    str						,      __snake_case      :    List[dict]						,      __snake_case      :    list						,      __snake_case      :    list						,      __snake_case      :    int						,      __snake_case      :    GradScaler = None						,      )->   Union[str, Any]:




						'''simple docstring'''
						# Model states
						for i, state in enumerate(__snake_case  ):
												__magic_name__					=      F'''{MODEL_NAME}.bin''' if i == 0 else F'''{MODEL_NAME}_{i}.bin'''
												__magic_name__					=      os.path.join(__snake_case						,      __snake_case  )
												save(__snake_case						,      __snake_case  )
												logger.info(F'''Model weights saved in {output_model_file}'''  )
						# Optimizer states
						for i, opt in enumerate(__snake_case  ):
												__magic_name__					=      opt.state_dict()
												__magic_name__					=      F'''{OPTIMIZER_NAME}.bin''' if i == 0 else F'''{OPTIMIZER_NAME}_{i}.bin'''
												__magic_name__					=      os.path.join(__snake_case						,      __snake_case  )
												save(__snake_case						,      __snake_case  )
												logger.info(F'''Optimizer state saved in {output_optimizer_file}'''  )
						# Scheduler states
						for i, scheduler in enumerate(__snake_case  ):
												__magic_name__					=      scheduler.state_dict()
												__magic_name__					=      F'''{SCHEDULER_NAME}.bin''' if i == 0 else F'''{SCHEDULER_NAME}_{i}.bin'''
												__magic_name__					=      os.path.join(__snake_case						,      __snake_case  )
												save(__snake_case						,      __snake_case  )
												logger.info(F'''Scheduler state saved in {output_scheduler_file}'''  )
						# GradScaler state
						if scaler is not None:
												__magic_name__					=      scaler.state_dict()
												__magic_name__					=      os.path.join(__snake_case						,      __snake_case  )
												torch.save(__snake_case						,      __snake_case  )
												logger.info(F'''Gradient scaler state saved in {output_scaler_file}'''  )
						# Random number generator states
						__magic_name__					=      {}
						__magic_name__					=      F'''{RNG_STATE_NAME}_{process_index}.pkl'''
						__magic_name__					=      random.getstate()
						__magic_name__					=      np.random.get_state()
						__magic_name__					=      torch.get_rng_state()
						if is_xpu_available():
												__magic_name__					=      torch.xpu.get_rng_state_all()
						else:
												__magic_name__					=      torch.cuda.get_rng_state_all()
						if is_tpu_available():
												__magic_name__					=      xm.get_rng_state()
						__magic_name__					=      os.path.join(__snake_case						,      __snake_case  )
						torch.save(__snake_case						,      __snake_case  )
						logger.info(F'''Random states saved in {output_states_file}'''  )
						return output_dir
def 				UpperCamelCase_	(					__snake_case      :    Optional[int]						,      __snake_case      :    Any						,      __snake_case      :    str						,      __snake_case      :    Optional[Any]						,      __snake_case      :    Any						,      __snake_case      :    Dict=None						,      __snake_case      :    Any=None						,      **__snake_case      :    Optional[int]						,      )->   Any:




						'''simple docstring'''
						if map_location not in [None, "cpu", "on_device"]:
												raise TypeError(
												    'Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`'  )
						if map_location is None:
												__magic_name__					=      'cpu'
						elif map_location == "on_device":
												__magic_name__					=      PartialState().device
						# Model states
						for i, model in enumerate(__snake_case  ):
												__magic_name__					=      F'''{MODEL_NAME}.bin''' if i == 0 else F'''{MODEL_NAME}_{i}.bin'''
												__magic_name__					=      os.path.join(__snake_case						,      __snake_case  )
												models[i].load_state_dict(torch.load(__snake_case						,      map_location=__snake_case  )						,      **__snake_case  )
						logger.info('All model weights loaded successfully'  )

						# Optimizer states
						for i, opt in enumerate(__snake_case  ):
												__magic_name__					=      F'''{OPTIMIZER_NAME}.bin''' if i == 0 else F'''{OPTIMIZER_NAME}_{i}.bin'''
												__magic_name__					=      os.path.join(__snake_case						,      __snake_case  )
												__magic_name__					=      torch.load(__snake_case						,      map_location=__snake_case  )
												optimizers[i].load_state_dict(__snake_case  )
						logger.info('All optimizer states loaded successfully'  )

						# Scheduler states
						for i, scheduler in enumerate(__snake_case  ):
												__magic_name__					=      F'''{SCHEDULER_NAME}.bin''' if i == 0 else F'''{SCHEDULER_NAME}_{i}.bin'''
												__magic_name__					=      os.path.join(__snake_case						,      __snake_case  )
												scheduler.load_state_dict(torch.load(__snake_case  )  )
						logger.info('All scheduler states loaded successfully'  )

						# GradScaler state
						if scaler is not None:
												__magic_name__					=      os.path.join(__snake_case						,      __snake_case  )
												scaler.load_state_dict(torch.load(__snake_case  )  )
												logger.info('GradScaler state loaded successfully'  )

						# Random states
						try:
												__magic_name__					=      torch.load(os.path.join(__snake_case						,      F'''{RNG_STATE_NAME}_{process_index}.pkl'''  )  )
												random.setstate(states['random_state']  )
												np.random.set_state(states['numpy_random_seed']  )
												torch.set_rng_state(states['torch_manual_seed']  )
												if is_xpu_available():
																		torch.xpu.set_rng_state_all(states['torch_xpu_manual_seed']  )
												else:
																		torch.cuda.set_rng_state_all(states['torch_cuda_manual_seed']  )
												if is_tpu_available():
																		xm.set_rng_state(states['xm_seed']  )
												logger.info('All random states loaded successfully'  )
						except Exception:
												logger.info('Could not load random states'  )
def 				UpperCamelCase_	(					__snake_case      :    Union[str, Any]						,      __snake_case      :    Dict						,      __snake_case      :    int = 0  )->   Optional[int]:




						'''simple docstring'''
						__magic_name__					=      Path(__snake_case  ) / F'''custom_checkpoint_{index}.pkl'''
						logger.info(F'''Saving the state of {get_pretty_name(__snake_case  )} to {save_location}'''  )
						torch.save(obj.state_dict()						,      __snake_case  )




def 				UpperCamelCase_	(					__snake_case      :    str						,      __snake_case      :    Any						,      __snake_case      :    int = 0  )->   List[Any]:




						'''simple docstring'''
						__magic_name__					=      F'''{path}/custom_checkpoint_{index}.pkl'''
						logger.info(F'''Loading the state of {get_pretty_name(__snake_case  )} from {load_location}'''  )
						obj.load_state_dict(torch.load(__snake_case						,      map_location='cpu'  )  )





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase			:					List[Any]         =							logging.get_logger(__name__)

UpperCamelCase			:					Tuple         =							{
    '''mnaylor/mega-base-wikitext''': '''https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json''',
}

class      __snake_case				(   _UpperCAmelCase):
					SCREAMING_SNAKE_CASE_		:       str								=			"mega"
					def __init__(						self       :						Optional[int]	,      _lowerCAmelCase       :						List[Any]=3_0_5_2_2	,      _lowerCAmelCase       :						List[str]=1_2_8	,      _lowerCAmelCase       :						Dict=4	,      _lowerCAmelCase       :						Optional[Any]=2_5_6	,      _lowerCAmelCase       :						Optional[Any]=1_6	,      _lowerCAmelCase       :						Dict=True	,      _lowerCAmelCase       :						Optional[Any]=6_4	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						Optional[int]=-1	,      _lowerCAmelCase       :						Any=None	,      _lowerCAmelCase       :						List[Any]=True	,      _lowerCAmelCase       :						List[Any]="scalenorm"	,      _lowerCAmelCase       :						str=True	,      _lowerCAmelCase       :						Union[str, Any]="silu"	,      _lowerCAmelCase       :						Optional[Any]="softmax"	,      _lowerCAmelCase       :						Optional[Any]=0.1	,      _lowerCAmelCase       :						Dict=0.1	,      _lowerCAmelCase       :						Any=0.1	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						Any=True	,      _lowerCAmelCase       :						int=2_5_6	,      _lowerCAmelCase       :						List[str]=True	,      _lowerCAmelCase       :						str=0.1	,      _lowerCAmelCase       :						List[str]=2_0_4_8	,      _lowerCAmelCase       :						Optional[Any]=False	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Dict=0.02	,      _lowerCAmelCase       :						str=0.2	,      _lowerCAmelCase       :						List[str]=0.02	,      _lowerCAmelCase       :						int=1.0	,      _lowerCAmelCase       :						Optional[int]=1	,      _lowerCAmelCase       :						List[str]=0	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Optional[Any]="rotary"	,      _lowerCAmelCase       :						int=None	,      _lowerCAmelCase       :						int=True	,      _lowerCAmelCase       :						Optional[Any]=True	,      **_lowerCAmelCase       :						str	,      )							->					Any:



											'''simple docstring'''





											super().__init__(pad_token_id=_lowerCAmelCase	,      bos_token_id=_lowerCAmelCase	,      eos_token_id=_lowerCAmelCase	,      **_lowerCAmelCase  )

											__magic_name__					=      vocab_size
											__magic_name__					=      hidden_size
											__magic_name__					=      num_hidden_layers
											__magic_name__					=      activation
											__magic_name__					=      attention_activation
											__magic_name__					=      intermediate_size
											__magic_name__					=      ema_projection_size
											__magic_name__					=      bidirectional
											__magic_name__					=      shared_representation_size
											__magic_name__					=      use_chunking
											__magic_name__					=      chunk_size
											__magic_name__					=      truncation
											__magic_name__					=      normalize_before_mega
											__magic_name__					=      normalization_type
											__magic_name__					=      norm_affine
											__magic_name__					=      dropout_prob
											__magic_name__					=      hidden_dropout_prob
											__magic_name__					=      attention_probs_dropout_prob
											__magic_name__					=      use_feature_dropout
											__magic_name__					=      use_normalized_ffn
											__magic_name__					=      nffn_hidden_size
											__magic_name__					=      normalize_before_ffn
											__magic_name__					=      nffn_activation_dropout_prob
											__magic_name__					=      max_positions
											__magic_name__					=      add_token_type_embeddings
											__magic_name__					=      type_vocab_size
											__magic_name__					=      initializer_range
											__magic_name__					=      ema_delta_alpha_range
											__magic_name__					=      ema_beta_range
											__magic_name__					=      ema_gamma_omega_range
											__magic_name__					=      relative_positional_bias
											__magic_name__					=      use_cache
											__magic_name__					=      classifier_dropout
											__magic_name__					=      add_lm_hidden_dense_layer
											__magic_name__					=      1  # not used but required by Hugging Face







class      __snake_case				(   _UpperCAmelCase):
					@property
					def 	a			(						self       :						int  )							->					Mapping[str, Mapping[int, str]]:



											'''simple docstring'''





											if self.task == "multiple-choice":
																	__magic_name__					=      {0: 'batch', 1: 'choice', 2: 'sequence'}
											else:
																	__magic_name__					=      {0: 'batch', 1: 'sequence'}
											return OrderedDict(
											    [
											        ('input_ids', dynamic_axis),
											        ('attention_mask', dynamic_axis),
											    ]  )

