
from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import Dataset

from .deepspeed import is_deepspeed_zeroa_enabled
from .generation.configuration_utils import GenerationConfig
from .trainer import Trainer
from .utils import logging


if TYPE_CHECKING:
							from .data.data_collator import DataCollator
							from .modeling_utils import PreTrainedModel
							from .tokenization_utils_base import PreTrainedTokenizerBase
							from .trainer_callback import TrainerCallback
							from .trainer_utils import EvalPrediction, PredictionOutput
							from .training_args import TrainingArguments


lowercase__						:List[Any]					    =       logging.get_logger(__name__)







class 							lowercase						(   SCREAMING_SNAKE_CASE__						):




							def __init__(							self  ,A__ = None  ,A__ = None  ,A__ = None  ,A__ = None  ,A__ = None  ,A__ = None  ,A__ = None  ,A__ = None  ,A__ = None  ,A__ = (None, None)  ,A__ = None  ,):
												super().__init__(
												    model=A__  ,args=A__  ,data_collator=A__  ,train_dataset=A__  ,eval_dataset=A__  ,tokenizer=A__  ,model_init=A__  ,compute_metrics=A__  ,callbacks=A__  ,optimizers=A__  ,preprocess_logits_for_metrics=A__  ,)

												# Override self.model.generation_config if a GenerationConfig is specified in args.
												# Priority: args.generation_config > model.generation_config > default GenerationConfig.
												if self.args.generation_config is not None:
																	lowercase		      =		self.load_generation_config(self.args.generation_config)
																	lowercase		      =		gen_config




							@staticmethod
							def 					A__ (							A__):

												# GenerationConfig provided, nothing to do
												if isinstance(A__  ,A__):
																	return deepcopy(A__)

												# str or Path
												lowercase		      =		Path(A__) if isinstance(A__  ,A__) else gen_config_arg
												lowercase		      =		None

												# Figuring if it is path pointing to a file, pointing to a directory or else a model id or URL
												# This step is required in order to determine config_file_name
												if pretrained_model_name.is_file():
																	lowercase		      =		pretrained_model_name.name
																	lowercase		      =		pretrained_model_name.parent
												# dir path
												elif pretrained_model_name.is_dir():
																	pass
												# model id or URL
												else:
																	lowercase		      =		gen_config_arg

												lowercase		      =		GenerationConfig.from_pretrained(A__  ,A__)
												return gen_config




							def 					A__ (							self  ,A__ = None  ,A__ = None  ,A__ = "eval"  ,**A__  ,):
												lowercase		      =		gen_kwargs.copy()
												if gen_kwargs.get('''max_length''') is None and gen_kwargs.get('''max_new_tokens''') is None:
																	lowercase		      =		self.args.generation_max_length
												lowercase		      =		(
												    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams''') is not None else self.args.generation_num_beams
												)
												lowercase		      =		gen_kwargs

												return super().evaluate(A__  ,ignore_keys=A__  ,metric_key_prefix=A__)




							def 					A__ (							self  ,A__  ,A__ = None  ,A__ = "test"  ,**A__  ,):
												lowercase		      =		gen_kwargs.copy()
												if gen_kwargs.get('''max_length''') is None and gen_kwargs.get('''max_new_tokens''') is None:
																	lowercase		      =		self.args.generation_max_length
												lowercase		      =		(
												    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams''') is not None else self.args.generation_num_beams
												)
												lowercase		      =		gen_kwargs

												return super().predict(A__  ,ignore_keys=A__  ,metric_key_prefix=A__)




							def 					A__ (							self  ,A__  ,A__  ,A__  ,A__ = None  ,**A__  ,):

												if not self.args.predict_with_generate or prediction_loss_only:
																	return super().prediction_step(
																	    A__  ,A__  ,prediction_loss_only=A__  ,ignore_keys=A__)

												lowercase		      =		'''labels''' in inputs
												lowercase		      =		self._prepare_inputs(A__)

												# XXX: adapt synced_gpus for fairscale as well
												# Priority (handled in generate):
												# gen_kwargs > model.generation_config > default GenerationConfig()

												if len(A__) == 0 and hasattr(self  ,'''_gen_kwargs'''):
																	lowercase		      =		self._gen_kwargs.copy()

												if gen_kwargs.get('''max_length''') is None and gen_kwargs.get('''max_new_tokens''') is None:
																	lowercase		      =		self.model.config.max_length
												lowercase		      =		(
												    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams''') is not None else self.model.config.num_beams
												)
												lowercase		      =		True if is_deepspeed_zeroa_enabled() else False
												lowercase		      =		(
												    gen_kwargs['''synced_gpus'''] if gen_kwargs.get('''synced_gpus''') is not None else default_synced_gpus
												)

												# If the `decoder_input_ids` was created from `labels`, evict the former, so that the model can freely generate
												# (otherwise, it would continue generating from the padded `decoder_input_ids`)
												if (
												    "labels" in inputs
												    and "decoder_input_ids" in inputs
												    and inputs["labels"].shape == inputs["decoder_input_ids"].shape
												):
																	lowercase		      =		{k: v for k, v in inputs.items() if k != '''decoder_input_ids'''}
												lowercase		      =		self.model.generate(**A__  ,**A__)

												# Temporary hack to ensure the generation config is not initialized for each iteration of the evaluation loop
												# TODO: remove this hack when the legacy code that initializes generation_config from a model config is
												# removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183
												if self.model.generation_config._from_model_config:
																	lowercase		      =		False

												# Retrieves GenerationConfig from model.generation_config
												lowercase		      =		self.model.generation_config
												# in case the batch is shorter than max length, the output should be padded
												if generated_tokens.shape[-1] < gen_config.max_length:
																	lowercase		      =		self._pad_tensors_to_max_len(A__  ,gen_config.max_length)
												elif gen_config.max_new_tokens is not None and generated_tokens.shape[-1] < gen_config.max_new_tokens + 1:
																	lowercase		      =		self._pad_tensors_to_max_len(A__  ,gen_config.max_new_tokens + 1)

												with torch.no_grad():
																	if has_labels:
																						with self.compute_loss_context_manager():
																											lowercase		      =		model(**A__)
																						if self.label_smoother is not None:
																											lowercase		      =		self.label_smoother(A__  ,inputs['''labels''']).mean().detach()
																						else:
																											lowercase		      =		(outputs['''loss'''] if isinstance(A__  ,A__) else outputs[0]).mean().detach()
																	else:
																						lowercase		      =		None

												if self.args.prediction_loss_only:
																	return loss, None, None

												if has_labels:
																	lowercase		      =		inputs['''labels''']
																	if labels.shape[-1] < gen_config.max_length:
																						lowercase		      =		self._pad_tensors_to_max_len(A__  ,gen_config.max_length)
																	elif gen_config.max_new_tokens is not None and labels.shape[-1] < gen_config.max_new_tokens + 1:
																						lowercase		      =		self._pad_tensors_to_max_len(A__  ,gen_config.max_new_tokens + 1)
												else:
																	lowercase		      =		None

												return loss, generated_tokens, labels





							def 					A__ (							self  ,A__  ,A__):
												if self.tokenizer is not None and hasattr(self.tokenizer  ,'''pad_token_id'''):
																	# If PAD token is not defined at least EOS token has to be defined
																	lowercase		      =		(
																	    self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
																	)
												else:
																	if self.model.config.pad_token_id is not None:
																						lowercase		      =		self.model.config.pad_token_id
																	else:
																						raise ValueError('''Pad_token_id must be set in the configuration of the model, in order to pad tensors''')

												lowercase		      =		pad_token_id * torch.ones(
												    (tensor.shape[0], max_length)  ,dtype=tensor.dtype  ,device=tensor.device)
												lowercase		      =		tensor
												return padded_tensor






import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
							import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


lowercase__						:Optional[Any]					    =       get_logger(__name__)



def 			UpperCamelCase       (				lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__ = None					,     ):
					'''simple docstring'''
					# Model states
					for i, state in enumerate(lowerCAmelCase__						):
										lowercase		      =		f'{MODEL_NAME}.bin' if i == 0 else f'{MODEL_NAME}_{i}.bin'
										lowercase		      =		os.path.join(lowerCAmelCase__					,     lowerCAmelCase__						)
										save(lowerCAmelCase__					,     lowerCAmelCase__						)
										logger.info(f'Model weights saved in {output_model_file}'						)
					# Optimizer states
					for i, opt in enumerate(lowerCAmelCase__						):
										lowercase		      =		opt.state_dict()
										lowercase		      =		f'{OPTIMIZER_NAME}.bin' if i == 0 else f'{OPTIMIZER_NAME}_{i}.bin'
										lowercase		      =		os.path.join(lowerCAmelCase__					,     lowerCAmelCase__						)
										save(lowerCAmelCase__					,     lowerCAmelCase__						)
										logger.info(f'Optimizer state saved in {output_optimizer_file}'						)
					# Scheduler states
					for i, scheduler in enumerate(lowerCAmelCase__						):
										lowercase		      =		scheduler.state_dict()
										lowercase		      =		f'{SCHEDULER_NAME}.bin' if i == 0 else f'{SCHEDULER_NAME}_{i}.bin'
										lowercase		      =		os.path.join(lowerCAmelCase__					,     lowerCAmelCase__						)
										save(lowerCAmelCase__					,     lowerCAmelCase__						)
										logger.info(f'Scheduler state saved in {output_scheduler_file}'						)
					# GradScaler state
					if scaler is not None:
										lowercase		      =		scaler.state_dict()
										lowercase		      =		os.path.join(lowerCAmelCase__					,     lowerCAmelCase__						)
										torch.save(lowerCAmelCase__					,     lowerCAmelCase__						)
										logger.info(f'Gradient scaler state saved in {output_scaler_file}'						)
					# Random number generator states
					lowercase		      =		{}
					lowercase		      =		f'{RNG_STATE_NAME}_{process_index}.pkl'
					lowercase		      =		random.getstate()
					lowercase		      =		np.random.get_state()
					lowercase		      =		torch.get_rng_state()
					if is_xpu_available():
										lowercase		      =		torch.xpu.get_rng_state_all()
					else:
										lowercase		      =		torch.cuda.get_rng_state_all()
					if is_tpu_available():
										lowercase		      =		xm.get_rng_state()
					lowercase		      =		os.path.join(lowerCAmelCase__					,     lowerCAmelCase__						)
					torch.save(lowerCAmelCase__					,     lowerCAmelCase__						)
					logger.info(f'Random states saved in {output_states_file}'						)
					return output_dir



def 			UpperCamelCase       (				lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__=None					,     lowerCAmelCase__=None					,     **lowerCAmelCase__					,     ):
					'''simple docstring'''
					if map_location not in [None, "cpu", "on_device"]:
										raise TypeError(
										    '''Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`'''						)
					if map_location is None:
										lowercase		      =		'''cpu'''
					elif map_location == "on_device":
										lowercase		      =		PartialState().device
					# Model states
					for i, model in enumerate(lowerCAmelCase__						):
										lowercase		      =		f'{MODEL_NAME}.bin' if i == 0 else f'{MODEL_NAME}_{i}.bin'
										lowercase		      =		os.path.join(lowerCAmelCase__					,     lowerCAmelCase__						)
										models[i].load_state_dict(torch.load(lowerCAmelCase__					,     map_location=lowerCAmelCase__						)					,     **lowerCAmelCase__						)
					logger.info('''All model weights loaded successfully'''						)

					# Optimizer states
					for i, opt in enumerate(lowerCAmelCase__						):
										lowercase		      =		f'{OPTIMIZER_NAME}.bin' if i == 0 else f'{OPTIMIZER_NAME}_{i}.bin'
										lowercase		      =		os.path.join(lowerCAmelCase__					,     lowerCAmelCase__						)
										lowercase		      =		torch.load(lowerCAmelCase__					,     map_location=lowerCAmelCase__						)
										optimizers[i].load_state_dict(lowerCAmelCase__						)
					logger.info('''All optimizer states loaded successfully'''						)

					# Scheduler states
					for i, scheduler in enumerate(lowerCAmelCase__						):
										lowercase		      =		f'{SCHEDULER_NAME}.bin' if i == 0 else f'{SCHEDULER_NAME}_{i}.bin'
										lowercase		      =		os.path.join(lowerCAmelCase__					,     lowerCAmelCase__						)
										scheduler.load_state_dict(torch.load(lowerCAmelCase__						)						)
					logger.info('''All scheduler states loaded successfully'''						)

					# GradScaler state
					if scaler is not None:
										lowercase		      =		os.path.join(lowerCAmelCase__					,     lowerCAmelCase__						)
										scaler.load_state_dict(torch.load(lowerCAmelCase__						)						)
										logger.info('''GradScaler state loaded successfully'''						)

					# Random states
					try:
										lowercase		      =		torch.load(os.path.join(lowerCAmelCase__					,     f'{RNG_STATE_NAME}_{process_index}.pkl'						)						)
										random.setstate(states['''random_state''']						)
										np.random.set_state(states['''numpy_random_seed''']						)
										torch.set_rng_state(states['''torch_manual_seed''']						)
										if is_xpu_available():
															torch.xpu.set_rng_state_all(states['''torch_xpu_manual_seed''']						)
										else:
															torch.cuda.set_rng_state_all(states['''torch_cuda_manual_seed''']						)
										if is_tpu_available():
															xm.set_rng_state(states['''xm_seed''']						)
										logger.info('''All random states loaded successfully'''						)
					except Exception:
										logger.info('''Could not load random states'''						)



def 			UpperCamelCase       (				lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__ = 0						):
					'''simple docstring'''
					lowercase		      =		Path(lowerCAmelCase__						) / f'custom_checkpoint_{index}.pkl'
					logger.info(f'Saving the state of {get_pretty_name(lowerCAmelCase__						)} to {save_location}'						)
					torch.save(obj.state_dict()					,     lowerCAmelCase__						)



def 			UpperCamelCase       (				lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__ = 0						):
					'''simple docstring'''
					lowercase		      =		f'{path}/custom_checkpoint_{index}.pkl'
					logger.info(f'Loading the state of {get_pretty_name(lowerCAmelCase__						)} from {load_location}'						)
					obj.load_state_dict(torch.load(lowerCAmelCase__					,     map_location='''cpu'''						)						)




