



from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import Dataset

from .deepspeed import is_deepspeed_zeroa_enabled
from .generation.configuration_utils import GenerationConfig
from .trainer import Trainer
from .utils import logging


if TYPE_CHECKING:
	from .data.data_collator import DataCollator
	from .modeling_utils import PreTrainedModel
	from .tokenization_utils_base import PreTrainedTokenizerBase
	from .trainer_callback import TrainerCallback
	from .trainer_utils import EvalPrediction, PredictionOutput
	from .training_args import TrainingArguments


A__       =			logging.get_logger(__name__)





class    __lowerCAmelCase					(	lowerCamelCase__			):




				def __init__(				self		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case = (None, None)		,							_snake_case = None		,							):



									"""simple docstring"""





									super().__init__(
									    model=_snake_case		,							args=_snake_case		,							data_collator=_snake_case		,							train_dataset=_snake_case		,							eval_dataset=_snake_case		,							tokenizer=_snake_case		,							model_init=_snake_case		,							compute_metrics=_snake_case		,							callbacks=_snake_case		,							optimizers=_snake_case		,							preprocess_logits_for_metrics=_snake_case		,							)

									# Override self.model.generation_config if a GenerationConfig is specified in args.
									# Priority: args.generation_config > model.generation_config > default GenerationConfig.
									if self.args.generation_config is not None:
														_lowerCAmelCase         =       self.load_generation_config(self.args.generation_config						)
														_lowerCAmelCase         =       gen_config




				@staticmethod
				def 				snake_case  (				_snake_case						):



									"""simple docstring"""





									if isinstance(_snake_case		,							_snake_case						):
														return deepcopy(_snake_case						)

									# str or Path
									_lowerCAmelCase         =       Path(_snake_case						) if isinstance(_snake_case		,							_snake_case						) else gen_config_arg
									_lowerCAmelCase         =       None

									# Figuring if it is path pointing to a file, pointing to a directory or else a model id or URL
									# This step is required in order to determine config_file_name
									if pretrained_model_name.is_file():
														_lowerCAmelCase         =       pretrained_model_name.name
														_lowerCAmelCase         =       pretrained_model_name.parent
									# dir path
									elif pretrained_model_name.is_dir():
														pass
									# model id or URL
									else:
														_lowerCAmelCase         =       gen_config_arg

									_lowerCAmelCase         =       GenerationConfig.from_pretrained(_snake_case		,							_snake_case						)
									return gen_config




				def 				snake_case  (				self		,							_snake_case = None		,							_snake_case = None		,							_snake_case = "eval"		,							**_snake_case		,							):



									"""simple docstring"""





									_lowerCAmelCase         =       gen_kwargs.copy()
									if gen_kwargs.get("""max_length"""						) is None and gen_kwargs.get("""max_new_tokens"""						) is None:
														_lowerCAmelCase         =       self.args.generation_max_length
									_lowerCAmelCase         =       (
									    gen_kwargs["""num_beams"""] if gen_kwargs.get("""num_beams"""						) is not None else self.args.generation_num_beams
									)
									_lowerCAmelCase         =       gen_kwargs

									return super().evaluate(_snake_case		,							ignore_keys=_snake_case		,							metric_key_prefix=_snake_case						)




				def 				snake_case  (				self		,							_snake_case		,							_snake_case = None		,							_snake_case = "test"		,							**_snake_case		,							):



									"""simple docstring"""





									_lowerCAmelCase         =       gen_kwargs.copy()
									if gen_kwargs.get("""max_length"""						) is None and gen_kwargs.get("""max_new_tokens"""						) is None:
														_lowerCAmelCase         =       self.args.generation_max_length
									_lowerCAmelCase         =       (
									    gen_kwargs["""num_beams"""] if gen_kwargs.get("""num_beams"""						) is not None else self.args.generation_num_beams
									)
									_lowerCAmelCase         =       gen_kwargs

									return super().predict(_snake_case		,							ignore_keys=_snake_case		,							metric_key_prefix=_snake_case						)




				def 				snake_case  (				self		,							_snake_case		,							_snake_case		,							_snake_case		,							_snake_case = None		,							**_snake_case		,							):



									"""simple docstring"""





									if not self.args.predict_with_generate or prediction_loss_only:
														return super().prediction_step(
														    _snake_case		,							_snake_case		,							prediction_loss_only=_snake_case		,							ignore_keys=_snake_case						)

									_lowerCAmelCase         =       """labels""" in inputs
									_lowerCAmelCase         =       self._prepare_inputs(_snake_case						)

									# XXX: adapt synced_gpus for fairscale as well
									# Priority (handled in generate):
									# gen_kwargs > model.generation_config > default GenerationConfig()

									if len(_snake_case						) == 0 and hasattr(self		,							"""_gen_kwargs"""						):
														_lowerCAmelCase         =       self._gen_kwargs.copy()

									if gen_kwargs.get("""max_length"""						) is None and gen_kwargs.get("""max_new_tokens"""						) is None:
														_lowerCAmelCase         =       self.model.config.max_length
									_lowerCAmelCase         =       (
									    gen_kwargs["""num_beams"""] if gen_kwargs.get("""num_beams"""						) is not None else self.model.config.num_beams
									)
									_lowerCAmelCase         =       True if is_deepspeed_zeroa_enabled() else False
									_lowerCAmelCase         =       (
									    gen_kwargs["""synced_gpus"""] if gen_kwargs.get("""synced_gpus"""						) is not None else default_synced_gpus
									)

									# If the `decoder_input_ids` was created from `labels`, evict the former, so that the model can freely generate
									# (otherwise, it would continue generating from the padded `decoder_input_ids`)
									if (
									    "labels" in inputs
									    and "decoder_input_ids" in inputs
									    and inputs["labels"].shape == inputs["decoder_input_ids"].shape
									):
														_lowerCAmelCase         =       {k: v for k, v in inputs.items() if k != """decoder_input_ids"""}
									_lowerCAmelCase         =       self.model.generate(**_snake_case		,							**_snake_case						)

									# Temporary hack to ensure the generation config is not initialized for each iteration of the evaluation loop
									# TODO: remove this hack when the legacy code that initializes generation_config from a model config is
									# removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183
									if self.model.generation_config._from_model_config:
														_lowerCAmelCase         =       False

									# Retrieves GenerationConfig from model.generation_config
									_lowerCAmelCase         =       self.model.generation_config
									# in case the batch is shorter than max length, the output should be padded
									if generated_tokens.shape[-1] < gen_config.max_length:
														_lowerCAmelCase         =       self._pad_tensors_to_max_len(_snake_case		,							gen_config.max_length						)
									elif gen_config.max_new_tokens is not None and generated_tokens.shape[-1] < gen_config.max_new_tokens + 1:
														_lowerCAmelCase         =       self._pad_tensors_to_max_len(_snake_case		,							gen_config.max_new_tokens + 1						)

									with torch.no_grad():
														if has_labels:
																			with self.compute_loss_context_manager():
																								_lowerCAmelCase         =       model(**_snake_case						)
																			if self.label_smoother is not None:
																								_lowerCAmelCase         =       self.label_smoother(_snake_case		,							inputs["""labels"""]						).mean().detach()
																			else:
																								_lowerCAmelCase         =       (outputs["""loss"""] if isinstance(_snake_case		,							_snake_case						) else outputs[0]).mean().detach()
														else:
																			_lowerCAmelCase         =       None

									if self.args.prediction_loss_only:
														return loss, None, None

									if has_labels:
														_lowerCAmelCase         =       inputs["""labels"""]
														if labels.shape[-1] < gen_config.max_length:
																			_lowerCAmelCase         =       self._pad_tensors_to_max_len(_snake_case		,							gen_config.max_length						)
														elif gen_config.max_new_tokens is not None and labels.shape[-1] < gen_config.max_new_tokens + 1:
																			_lowerCAmelCase         =       self._pad_tensors_to_max_len(_snake_case		,							gen_config.max_new_tokens + 1						)
									else:
														_lowerCAmelCase         =       None

									return loss, generated_tokens, labels




				def 				snake_case  (				self		,							_snake_case		,							_snake_case						):



									"""simple docstring"""





									if self.tokenizer is not None and hasattr(self.tokenizer		,							"""pad_token_id"""						):
														# If PAD token is not defined at least EOS token has to be defined
														_lowerCAmelCase         =       (
														    self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
														)
									else:
														if self.model.config.pad_token_id is not None:
																			_lowerCAmelCase         =       self.model.config.pad_token_id
														else:
																			raise ValueError("""Pad_token_id must be set in the configuration of the model, in order to pad tensors"""						)

									_lowerCAmelCase         =       pad_token_id * torch.ones(
									    (tensor.shape[0], max_length)		,							dtype=tensor.dtype		,							device=tensor.device						)
									_lowerCAmelCase         =       tensor
									return padded_tensor








import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available


def 						_UpperCAmelCase							( snake_case      ,		snake_case      ,		snake_case      ,		snake_case=None     ):





					"""simple docstring"""

					_lowerCAmelCase         =       None
					# Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
					if str(weight.dtype     ) == "torch.bfloat16":
										# Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
										_lowerCAmelCase         =       weight.view(torch.intaa     )
										_lowerCAmelCase         =       """bfloat16"""
					_lowerCAmelCase         =       weight.cpu().numpy()
					_lowerCAmelCase         =       os.path.join(snake_case      ,		F'{weight_name}.dat'     )
					if index is not None:
										if dtype is None:
															_lowerCAmelCase         =       str(array.dtype     )
										_lowerCAmelCase         =       {"""dtype""": dtype, """shape""": list(array.shape     )}
					if array.ndim == 0:
										_lowerCAmelCase         =       array[None]
					_lowerCAmelCase         =       np.memmap(snake_case      ,		dtype=array.dtype      ,		mode="""w+"""      ,		shape=array.shape     )
					_lowerCAmelCase         =       array[:]
					file_array.flush()
					return index


def 						_UpperCAmelCase							( snake_case      ,		snake_case     ):





					"""simple docstring"""

					_lowerCAmelCase         =       tuple(weight_info["""shape"""]     )
					if shape == ():
										# NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
										_lowerCAmelCase         =       (1,)

					_lowerCAmelCase         =       weight_info["""dtype"""]
					if dtype == "bfloat16":
										# NumPy does not support bfloat16 so this was saved as a int16
										_lowerCAmelCase         =       """int16"""

					_lowerCAmelCase         =       np.memmap(snake_case      ,		dtype=snake_case      ,		shape=snake_case      ,		mode="""r"""     )

					if len(weight_info["""shape"""]     ) == 0:
										_lowerCAmelCase         =       weight[0]
					_lowerCAmelCase         =       torch.tensor(snake_case     )
					if weight_info["dtype"] == "bfloat16":
										_lowerCAmelCase         =       weight.view(torch.bfloataa     )

					return weight


def 						_UpperCAmelCase							( snake_case      ,		snake_case     ):





					"""simple docstring"""

					if index is None or len(snake_case     ) == 0:
										# Nothing to save
										return

					_lowerCAmelCase         =       os.path.join(snake_case      ,		"""index.json"""     )
					if os.path.isfile(snake_case     ):
										with open(snake_case      ,		"""r"""      ,		encoding="""utf-8"""     ) as f:
															_lowerCAmelCase         =       json.load(snake_case     )
					else:
										_lowerCAmelCase         =       {}
					current_index.update(snake_case     )

					with open(snake_case      ,		"""w"""      ,		encoding="""utf-8"""     ) as f:
										json.dump(snake_case      ,		snake_case      ,		indent=2     )


def 						_UpperCAmelCase							( snake_case      ,		snake_case     ):





					"""simple docstring"""

					os.makedirs(snake_case      ,		exist_ok=snake_case     )
					_lowerCAmelCase         =       {}
					for name, parameter in state_dict.items():
										_lowerCAmelCase         =       offload_weight(snake_case      ,		snake_case      ,		snake_case      ,		index=snake_case     )

					# Update index
					save_offload_index(snake_case      ,		snake_case     )





class    __lowerCAmelCase					(	lowerCamelCase__			):




				def __init__(				self		,							_snake_case		,							_snake_case						):



									"""simple docstring"""





									_lowerCAmelCase         =       dataset
									_lowerCAmelCase         =       prefix




				def __getitem__(				self		,							_snake_case						):



									"""simple docstring"""





									return self.dataset[F'{self.prefix}{key}']




				def __iter__(				self						):



									"""simple docstring"""





									return iter([key for key in self.dataset if key.startswith(self.prefix						)]						)




				def __len__(				self						):



									"""simple docstring"""





									return len(self.dataset						)





class    __lowerCAmelCase					(	lowerCamelCase__			):




				def __init__(				self		,							_snake_case = None		,							_snake_case = None		,							_snake_case = None		,							_snake_case=None		,							):



									"""simple docstring"""





									if state_dict is None and save_folder is None:
														raise ValueError("""Need either a `state_dict` or a `save_folder` containing offloaded weights."""						)

									_lowerCAmelCase         =       {} if state_dict is None else state_dict
									_lowerCAmelCase         =       save_folder
									if index is None and save_folder is not None:
														with open(os.path.join(_snake_case		,							"""index.json"""						)						) as f:
																			_lowerCAmelCase         =       json.load(_snake_case						)
									_lowerCAmelCase         =       {} if index is None else index
									_lowerCAmelCase         =       list(self.state_dict.keys()						)
									self.all_keys.extend([key for key in self.index if key not in self.all_keys]						)
									_lowerCAmelCase         =       device




				def __getitem__(				self		,							_snake_case						):



									"""simple docstring"""





									if key in self.state_dict:
														return self.state_dict[key]
									_lowerCAmelCase         =       self.index[key]
									if weight_info.get("""safetensors_file"""						) is not None:
														if not is_safetensors_available():
																			raise ImportError("""These offloaded weights require the use of safetensors: `pip install safetensors`."""						)

														from safetensors import safe_open

														_lowerCAmelCase         =       """cpu""" if self.device is None else self.device
														with safe_open(weight_info["""safetensors_file"""]		,							framework="""pt"""		,							device=_snake_case						) as f:
																			_lowerCAmelCase         =       f.get_tensor(weight_info.get("""weight_name"""		,							_snake_case						)						)

														if "dtype" in weight_info:
																			return tensor.to(getattr(_snake_case		,							weight_info["""dtype"""]						)						)
														else:
																			return tensor

									_lowerCAmelCase         =       os.path.join(self.save_folder		,							F'{key}.dat'						)
									return load_offloaded_weight(_snake_case		,							_snake_case						)




				def __iter__(				self						):



									"""simple docstring"""





									return iter(self.all_keys						)




				def __len__(				self						):



									"""simple docstring"""





									return len(self.all_keys						)


def 						_UpperCAmelCase							( snake_case      ,		snake_case     ):





					"""simple docstring"""

					_lowerCAmelCase         =       {}
					for module_name in submodule_names:
										# We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
										# submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
										result.update(
										    {
										        key: param
										        for key, param in state_dict.items()
										        if key == module_name or key.startswith(module_name + """."""     )
										    }     )
					return result



