



"""simple docstring"""


from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


snake_case_								=			logging.get_logger(__name__)







class  A_     (	SCREAMING_SNAKE_CASE_							):



						"""simple docstring"""
						__UpperCamelCase													=				"""mt5"""
						__UpperCamelCase													=				["""past_key_values"""]






						def __init__(   self					:Any  ,    lowercase_					:Optional[Any]=25_01_12  ,    lowercase_					:List[str]=5_12  ,    lowercase_					:Any=64  ,    lowercase_					:List[str]=10_24  ,    lowercase_					:int=8  ,    lowercase_					:int=None  ,    lowercase_					:int=6  ,    lowercase_					:Any=32  ,    lowercase_					:List[Any]=1_28  ,    lowercase_					:int=0.1  ,    lowercase_					:List[str]=1E-6  ,    lowercase_					:Optional[Any]=1.0  ,    lowercase_					:Union[str, Any]="gated-gelu"  ,    lowercase_					:Optional[int]=True  ,    lowercase_					:Tuple=True  ,    lowercase_					:Union[str, Any]="T5Tokenizer"  ,    lowercase_					:Tuple=False  ,    lowercase_					:str=0  ,    lowercase_					:Optional[Any]=1  ,    lowercase_					:Optional[int]=0  ,    **lowercase_					:Tuple  ,    )							->     List[str]:
												super().__init__(
												    is_encoder_decoder=lowercase_  ,    tokenizer_class=lowercase_  ,    tie_word_embeddings=lowercase_  ,    pad_token_id=lowercase_  ,    eos_token_id=lowercase_  ,    decoder_start_token_id=lowercase_  ,    **lowercase_  ,    )
												UpperCAmelCase			 =					vocab_size
												UpperCAmelCase			 =					d_model
												UpperCAmelCase			 =					d_kv
												UpperCAmelCase			 =					d_ff
												UpperCAmelCase			 =					num_layers
												UpperCAmelCase			 =					(
												    num_decoder_layers if num_decoder_layers is not None else self.num_layers
												)  # default = symmetry
												UpperCAmelCase			 =					num_heads
												UpperCAmelCase			 =					relative_attention_num_buckets
												UpperCAmelCase			 =					relative_attention_max_distance
												UpperCAmelCase			 =					dropout_rate
												UpperCAmelCase			 =					layer_norm_epsilon
												UpperCAmelCase			 =					initializer_factor
												UpperCAmelCase			 =					feed_forward_proj
												UpperCAmelCase			 =					use_cache

												UpperCAmelCase			 =					self.feed_forward_proj.split('-'   )
												UpperCAmelCase			 =					act_info[-1]
												UpperCAmelCase			 =					act_info[0] == 'gated'

												if len(lowercase_   ) > 1 and act_info[0] != "gated" or len(lowercase_   ) > 2:
																		raise ValueError(
																		    f"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
																		    'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '
																		    '\'gated-gelu\' or \'relu\''   )

												# for backwards compatibility
												if feed_forward_proj == "gated-gelu":
																		UpperCAmelCase			 =					'gelu_new'






						@property
						def       UpperCAmelCase__	(   self					:Union[str, Any]   )							->     Optional[int]:
												return self.d_model






						@property
						def       UpperCAmelCase__	(   self					:List[Any]   )							->     Optional[int]:
												return self.num_heads






						@property
						def       UpperCAmelCase__	(   self					:List[Any]   )							->     Dict:
												return self.num_layers







class  A_     (	SCREAMING_SNAKE_CASE_							):



						"""simple docstring"""






						@property
						# Copied from transformers.models.t5.configuration_t5.T5OnnxConfig.inputs
						def       UpperCAmelCase__	(   self					:Optional[int]   )							->     Mapping[str, Mapping[int, str]]:
												UpperCAmelCase			 =					{
												    'input_ids': {0: 'batch', 1: 'encoder_sequence'},
												    'attention_mask': {0: 'batch', 1: 'encoder_sequence'},
												}
												if self.use_past:
																		UpperCAmelCase			 =					'past_encoder_sequence + sequence'
																		UpperCAmelCase			 =					{0: 'batch'}
																		UpperCAmelCase			 =					{0: 'batch', 1: 'past_decoder_sequence + sequence'}
												else:
																		UpperCAmelCase			 =					{0: 'batch', 1: 'decoder_sequence'}
																		UpperCAmelCase			 =					{0: 'batch', 1: 'decoder_sequence'}

												if self.use_past:
																		self.fill_with_past_key_values_(lowercase_  ,    direction='inputs'   )

												return common_inputs






						@property
						# Copied from transformers.models.t5.configuration_t5.T5OnnxConfig.default_onnx_opset
						def       UpperCAmelCase__	(   self					:Any   )							->     int:
												return 13






						@property
						def       UpperCAmelCase__	(   self					:Optional[int]   )							->     float:
												return 5E-4













"""simple docstring"""



import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
						import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


A_     =				get_logger(__name__)




def   _lowerCAmelCase		(				UpperCAmelCase__							:       str,    UpperCAmelCase__							:       List[dict],    UpperCAmelCase__							:       list,    UpperCAmelCase__							:       list,    UpperCAmelCase__							:       int,    UpperCAmelCase__							:       GradScaler = None,    )		->Any:
						# Model states
						for i, state in enumerate(UpperCAmelCase__   ):
												A__    :	Optional[int]			=	f'{MODEL_NAME}.bin' if i == 0 else f'{MODEL_NAME}_{i}.bin'
												A__    :	int			=	os.path.join(UpperCAmelCase__,    UpperCAmelCase__   )
												save(UpperCAmelCase__,    UpperCAmelCase__   )
												logger.info(f'Model weights saved in {output_model_file}'   )
						# Optimizer states
						for i, opt in enumerate(UpperCAmelCase__   ):
												A__    :	str			=	opt.state_dict()
												A__    :	Any			=	f'{OPTIMIZER_NAME}.bin' if i == 0 else f'{OPTIMIZER_NAME}_{i}.bin'
												A__    :	Optional[int]			=	os.path.join(UpperCAmelCase__,    UpperCAmelCase__   )
												save(UpperCAmelCase__,    UpperCAmelCase__   )
												logger.info(f'Optimizer state saved in {output_optimizer_file}'   )
						# Scheduler states
						for i, scheduler in enumerate(UpperCAmelCase__   ):
												A__    :	Union[str, Any]			=	scheduler.state_dict()
												A__    :	Union[str, Any]			=	f'{SCHEDULER_NAME}.bin' if i == 0 else f'{SCHEDULER_NAME}_{i}.bin'
												A__    :	Tuple			=	os.path.join(UpperCAmelCase__,    UpperCAmelCase__   )
												save(UpperCAmelCase__,    UpperCAmelCase__   )
												logger.info(f'Scheduler state saved in {output_scheduler_file}'   )
						# GradScaler state
						if scaler is not None:
												A__    :	List[str]			=	scaler.state_dict()
												A__    :	Optional[Any]			=	os.path.join(UpperCAmelCase__,    UpperCAmelCase__   )
												torch.save(UpperCAmelCase__,    UpperCAmelCase__   )
												logger.info(f'Gradient scaler state saved in {output_scaler_file}'   )
						# Random number generator states
						A__    :	str			=	{}
						A__    :	str			=	f'{RNG_STATE_NAME}_{process_index}.pkl'
						A__    :	Optional[Any]			=	random.getstate()
						A__    :	str			=	np.random.get_state()
						A__    :	List[str]			=	torch.get_rng_state()
						if is_xpu_available():
												A__    :	Optional[Any]			=	torch.xpu.get_rng_state_all()
						else:
												A__    :	str			=	torch.cuda.get_rng_state_all()
						if is_tpu_available():
												A__    :	Tuple			=	xm.get_rng_state()
						A__    :	str			=	os.path.join(UpperCAmelCase__,    UpperCAmelCase__   )
						torch.save(UpperCAmelCase__,    UpperCAmelCase__   )
						logger.info(f'Random states saved in {output_states_file}'   )
						return output_dir




def   _lowerCAmelCase		(				UpperCAmelCase__							:       Any,    UpperCAmelCase__							:       Union[str, Any],    UpperCAmelCase__							:       Union[str, Any],    UpperCAmelCase__							:       Dict,    UpperCAmelCase__							:       Tuple,    UpperCAmelCase__							:       Optional[Any]=None,    UpperCAmelCase__							:       str=None,    **UpperCAmelCase__							:       Optional[int],    )		->List[str]:
						if map_location not in [None, "cpu", "on_device"]:
												raise TypeError(
												    """Unsupported optimizer map location passed, please choose one of `None`, `'cpu'`, or `'on_device'`"""   )
						if map_location is None:
												A__    :	Union[str, Any]			=	"""cpu"""
						elif map_location == "on_device":
												A__    :	Any			=	PartialState().device
						# Model states
						for i, model in enumerate(UpperCAmelCase__   ):
												A__    :	Any			=	f'{MODEL_NAME}.bin' if i == 0 else f'{MODEL_NAME}_{i}.bin'
												A__    :	List[str]			=	os.path.join(UpperCAmelCase__,    UpperCAmelCase__   )
												models[i].load_state_dict(torch.load(UpperCAmelCase__,    map_location=UpperCAmelCase__   ),    **UpperCAmelCase__   )
						logger.info("""All model weights loaded successfully"""   )

						# Optimizer states
						for i, opt in enumerate(UpperCAmelCase__   ):
												A__    :	Union[str, Any]			=	f'{OPTIMIZER_NAME}.bin' if i == 0 else f'{OPTIMIZER_NAME}_{i}.bin'
												A__    :	Dict			=	os.path.join(UpperCAmelCase__,    UpperCAmelCase__   )
												A__    :	List[Any]			=	torch.load(UpperCAmelCase__,    map_location=UpperCAmelCase__   )
												optimizers[i].load_state_dict(UpperCAmelCase__   )
						logger.info("""All optimizer states loaded successfully"""   )

						# Scheduler states
						for i, scheduler in enumerate(UpperCAmelCase__   ):
												A__    :	List[Any]			=	f'{SCHEDULER_NAME}.bin' if i == 0 else f'{SCHEDULER_NAME}_{i}.bin'
												A__    :	Optional[Any]			=	os.path.join(UpperCAmelCase__,    UpperCAmelCase__   )
												scheduler.load_state_dict(torch.load(UpperCAmelCase__   )   )
						logger.info("""All scheduler states loaded successfully"""   )

						# GradScaler state
						if scaler is not None:
												A__    :	List[str]			=	os.path.join(UpperCAmelCase__,    UpperCAmelCase__   )
												scaler.load_state_dict(torch.load(UpperCAmelCase__   )   )
												logger.info("""GradScaler state loaded successfully"""   )

						# Random states
						try:
												A__    :	Any			=	torch.load(os.path.join(UpperCAmelCase__,    f'{RNG_STATE_NAME}_{process_index}.pkl'   )   )
												random.setstate(states["""random_state"""]   )
												np.random.set_state(states["""numpy_random_seed"""]   )
												torch.set_rng_state(states["""torch_manual_seed"""]   )
												if is_xpu_available():
																		torch.xpu.set_rng_state_all(states["""torch_xpu_manual_seed"""]   )
												else:
																		torch.cuda.set_rng_state_all(states["""torch_cuda_manual_seed"""]   )
												if is_tpu_available():
																		xm.set_rng_state(states["""xm_seed"""]   )
												logger.info("""All random states loaded successfully"""   )
						except Exception:
												logger.info("""Could not load random states"""   )




def   _lowerCAmelCase		(				UpperCAmelCase__							:       Tuple,    UpperCAmelCase__							:       Dict,    UpperCAmelCase__							:       int = 0   )		->Dict:
						A__    :	Dict			=	Path(UpperCAmelCase__   ) / f'custom_checkpoint_{index}.pkl'
						logger.info(f'Saving the state of {get_pretty_name(UpperCAmelCase__   )} to {save_location}'   )
						torch.save(obj.state_dict(),    UpperCAmelCase__   )




def   _lowerCAmelCase		(				UpperCAmelCase__							:       Tuple,    UpperCAmelCase__							:       str,    UpperCAmelCase__							:       int = 0   )		->str:
						A__    :	Optional[int]			=	f'{path}/custom_checkpoint_{index}.pkl'
						logger.info(f'Loading the state of {get_pretty_name(UpperCAmelCase__   )} from {load_location}'   )
						obj.load_state_dict(torch.load(UpperCAmelCase__,    map_location="""cpu"""   )   )






