



import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
		import torch

snake_case     :    Optional[Any]						=    logging.get_logger(__name__)
def 						snake_case__							(       )		->		Optional[int]:







						"""simple docstring"""

						return importlib.util.find_spec("deepspeed"							) is not None


if is_accelerate_available() and is_deepspeed_available():
		from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
		# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
		# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
		from builtins import object as DeepSpeedConfig




class 				lowerCAmelCase__	(						_SCREAMING_SNAKE_CASE    ):







					def __init__(							self   :  Any      ,		_A   :  str):
											set_hf_deepspeed_config(self)
											dep_version_check("accelerate")
											dep_version_check("deepspeed")
											super().__init__(_A)




class 				lowerCAmelCase__	(						_SCREAMING_SNAKE_CASE    ):







					def __init__(							self   :  Dict      ,		_A   :  Optional[Any]):
											super().__init__(_A)
											A__						:				Tuple         =			None
											A__						:				Optional[Any]         =			[]







					def 			_lowercase      (							self   :  Any):





											if self._dtype is None:
																	raise ValueError("trainer_config_process() wasn't called yet to tell dtype")
											return self._dtype







					def 			_lowercase      (							self   :  str      ,		_A   :  Optional[Any]):
											A__						:				Tuple         =			self.get_value(_A)
											if val is None:
																	return False
											else:
																	return val == "auto"







					def 			_lowercase      (							self   :  Any      ,		_A   :  int      ,		_A   :  Any      ,		_A   :  Dict=None      ,		_A   :  List[Any]=True):
											A__						:				int         =			self.find_config_node(_A)
											if config is None:
																	return

											if config.get(_A) == "auto":
																	A__						:				Tuple         =			hf_val
																	return

											if not must_match:
																	return

											A__						:				Dict         =			config.get(_A)
											if ds_val is not None and ds_val != hf_val:
																	self.mismatches.append(F'- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}')

					__A  :     Union[str, Any]     =    partialmethod(_SCREAMING_SNAKE_CASE       ,				must_match=_SCREAMING_SNAKE_CASE    )







					def 			_lowercase      (							self   :  List[Any]      ,		_A   :  List[Any]):
											A__						:				Any         =			args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
											self.fill_match(
											    "train_micro_batch_size_per_gpu"      ,		args.per_device_train_batch_size      ,		"per_device_train_batch_size")
											self.fill_match("gradient_accumulation_steps"      ,		args.gradient_accumulation_steps      ,		"gradient_accumulation_steps")
											self.fill_match("train_batch_size"      ,		_A      ,		"train_batch_size (calculated)")
											self.fill_match("gradient_clipping"      ,		args.max_grad_norm      ,		"max_grad_norm")

											self.fill_match("optimizer.params.lr"      ,		args.learning_rate      ,		"learning_rate")
											self.fill_match("optimizer.params.betas"      ,		[args.adam_betaa, args.adam_betaa]      ,		"adam_beta1+adam_beta2")
											self.fill_match("optimizer.params.eps"      ,		args.adam_epsilon      ,		"adam_epsilon")
											self.fill_match("optimizer.params.weight_decay"      ,		args.weight_decay      ,		"weight_decay")

											self.fill_only("scheduler.params.warmup_min_lr"      ,		0)  # not a trainer arg
											self.fill_match("scheduler.params.warmup_max_lr"      ,		args.learning_rate      ,		"learning_rate")
											# total_num_steps - will get set in trainer_config_finalize

											# fp16
											if args.fpaa or args.fpaa_full_eval:
																	A__						:				Optional[int]         =			"""apex""" if args.fpaa_backend == """apex""" else """amp"""
											else:
																	A__						:				List[str]         =			None

											if args.save_on_each_node:
																	# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
																	A__						:				int         =			self.config.get("checkpoint"      ,		{})
																	A__						:				Any         =			args.save_on_each_node

											# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
											# any here unless the user did the work
											self.fill_match(
											    "fp16.enabled"      ,		((args.fpaa or args.fpaa_full_eval) and fpaa_backend == "amp")      ,		"fp16|fp16_full_eval+fp16_backend(amp)"      ,		)

											# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
											# ZeRO features
											self.fill_match("amp.enabled"      ,		fpaa_backend == "apex"      ,		"fp16+fp16_backend(apex)")
											self.fill_match("amp.opt_level"      ,		args.fpaa_opt_level      ,		"fp16_opt_level")

											self.fill_match("bf16.enabled"      ,		(args.bfaa or args.bfaa_full_eval)      ,		"bf16|bf16_full_eval")

											# deepspeed's default mode is fp16 unless there is a config that says differently
											if self.is_true("bf16.enabled"):
																	A__						:				Union[str, Any]         =			torch.bfloataa
											elif self.is_false("fp16.enabled"):
																	A__						:				Optional[int]         =			torch.floataa
											else:
																	A__						:				Optional[int]         =			torch.floataa







					def 			_lowercase      (							self   :  Dict      ,		_A   :  List[str]      ,		_A   :  Union[str, Any]      ,		_A   :  Dict):
											A__						:				Any         =			[
											    """zero_optimization.reduce_bucket_size""",
											    """zero_optimization.stage3_prefetch_bucket_size""",
											    """zero_optimization.stage3_param_persistence_threshold""",
											]
											A__						:				Any         =			[x for x in hidden_size_based_keys if self.is_auto(_A)]

											if len(_A) > 0:
																	if hasattr(model.config      ,		"hidden_size"):
																							A__						:				Optional[Any]         =			model.config.hidden_size
																	elif hasattr(model.config      ,		"hidden_sizes"):
																							# if there are many hidden sizes pick the largest one
																							A__						:				Optional[int]         =			max(model.config.hidden_sizes)
																	else:
																							raise ValueError(
																							    "The model's config file has neither `hidden_size` nor `hidden_sizes` entry, "
																							    "therefore it's not possible to automatically fill out the following `auto` entries "
																							    F'in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '
																							    "`auto` values for these keys with an integer value of your choice.")

																	self.fill_only("zero_optimization.reduce_bucket_size"      ,		hidden_size * hidden_size)
																	if self.is_zeroa():
																							# automatically assign the optimal config values based on model config
																							self.fill_only("zero_optimization.stage3_prefetch_bucket_size"      ,		0.9 * hidden_size * hidden_size)
																							self.fill_only("zero_optimization.stage3_param_persistence_threshold"      ,		10 * hidden_size)

																							 # scheduler
											self.fill_match("scheduler.params.total_num_steps"      ,		_A      ,		"num_training_steps (calculated)")
											self.fill_match("scheduler.params.warmup_num_steps"      ,		args.get_warmup_steps(_A)      ,		"warmup_steps")

											if len(self.mismatches) > 0:
																	A__						:				int         =			"""\n""".join(self.mismatches)
																	raise ValueError(
																	    "Please correct the following DeepSpeed config values that mismatch TrainingArguments"
																	    F' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.')




snake_case     :    Tuple						=    None
def 						snake_case__							(       __lowercase							)		->		Optional[int]:







						"""simple docstring"""

						global _hf_deepspeed_config_weak_ref
						# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
						A__						:				List[str]         =			weakref.ref(__SCREAMING_SNAKE_CASE							)
def 						snake_case__							(       )		->		Dict:







						"""simple docstring"""

						global _hf_deepspeed_config_weak_ref
						A__						:				Any         =			None
def 						snake_case__							(       )		->		str:







						"""simple docstring"""

						if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
												return _hf_deepspeed_config_weak_ref().is_zeroa()
						else:
												return False
def 						snake_case__							(       )		->		Union[str, Any]:







						"""simple docstring"""

						if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
												return _hf_deepspeed_config_weak_ref().config
						else:
												return None
def 						snake_case__							(       __lowercase				,  __lowercase				,  __lowercase				,  __lowercase				,  __lowercase							)		->		Dict:







						"""simple docstring"""

						from accelerate.utils import DummyOptim, DummyScheduler

						A__						:				Optional[int]         =			hf_deepspeed_config.config

						# Optimizer + Scheduler
						# Currently supported combos:
						# 1. DS scheduler + DS optimizer: Yes
						# 2. HF scheduler + HF optimizer: Yes
						# 3. DS scheduler + HF optimizer: Yes
						# 4. HF scheduler + DS optimizer: No
						#
						# Unless Offload is enabled in which case it's:
						# 1. DS scheduler + DS optimizer: Yes
						# 2. HF scheduler + HF optimizer: Mostly*
						# 3. DS scheduler + HF optimizer: Mostly*
						# 4. HF scheduler + DS optimizer: No
						#
						# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

						A__						:				Tuple         =			None
						if "optimizer" in config:
												if args.adafactor:
																		raise ValueError(
																		    "--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. "
																		    "Only one optimizer can be configured."							)
												A__						:				Dict         =			DummyOptim(params=__SCREAMING_SNAKE_CASE							)
						else:
												if hf_deepspeed_config.is_offload():
																		logger.info(
																		    "Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"
																		    " custom optimizer has both CPU and GPU implementation (except LAMB)"							)

												# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
												# But trainer uses AdamW by default.
												A__						:				Optional[Any]         =			trainer.create_optimizer()
												# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
												A__						:				Optional[Any]         =			True

						A__						:				Any         =			None
						if "scheduler" in config:
												A__						:				int         =			DummyScheduler(__SCREAMING_SNAKE_CASE							)
						else:
												if isinstance(__SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE							):
																		raise ValueError(
																		    "Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. "
																		    "Please configure a scheduler in the DeepSpeed config."							)
												A__						:				int         =			trainer.create_scheduler(num_training_steps=__SCREAMING_SNAKE_CASE				,  optimizer=__SCREAMING_SNAKE_CASE							)

						return optimizer, lr_scheduler
def 						snake_case__							(       __lowercase				,  __lowercase				,  __lowercase=False							)		->		Optional[Any]:







						"""simple docstring"""

						from deepspeed.utils import logger as ds_logger

						A__						:				List[str]         =			trainer.model
						A__						:				Tuple         =			trainer.args

						A__						:				str         =			trainer.accelerator.state.deepspeed_plugin.hf_ds_config

						# resume config update - some bits like `model` and `num_training_steps` only become available during train
						hf_deepspeed_config.trainer_config_finalize(__SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE							)

						# set the Deepspeed log level consistent with the Trainer
						ds_logger.setLevel(args.get_process_log_level()							)

						if inference:
												# only Z3 makes sense for the inference
												if not hf_deepspeed_config.is_zeroa():
																		raise ValueError("ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"							)

												# in case the training config is re-used for inference
												hf_deepspeed_config.del_config_sub_tree("optimizer"							)
												hf_deepspeed_config.del_config_sub_tree("lr_scheduler"							)
												A__						:				Tuple         =			None, None
												A__						:				Dict         =			None
						else:
												A__						:				Dict         =			None  # important for when deepspeed_init is used as re-init
												A__						:				Any         =			list(filter(lambda __lowercase							: p.requires_grad				,  model.parameters()							)							)
												A__						:				int         =			deepspeed_optim_sched(
												    __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE							)

						# keep for quick debug:
						# from pprint import pprint; pprint(config)

						return optimizer, lr_scheduler




def 						snake_case__							(       __lowercase				,  __lowercase							)		->		Dict:







						"""simple docstring"""

						import glob

						A__						:				int         =			sorted(glob.glob(F'{checkpoint_path}/global_step*'							)							)

						if len(__SCREAMING_SNAKE_CASE							) > 0:
												logger.info(F'Attempting to resume from {checkpoint_path}'							)
												# this magically updates self.optimizer and self.lr_scheduler
												A__						:				str         =			deepspeed_engine.load_checkpoint(
												    __SCREAMING_SNAKE_CASE				,  load_optimizer_states=__SCREAMING_SNAKE_CASE				,  load_lr_scheduler_states=__SCREAMING_SNAKE_CASE							)
												if load_path is None:
																		raise ValueError(F'[deepspeed] failed to resume from checkpoint {checkpoint_path}'							)
						else:
												raise ValueError(F'Can\'t find a valid checkpoint at {checkpoint_path}'							)

'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


A__ :		str					={'configuration_t5': ['T5_PRETRAINED_CONFIG_ARCHIVE_MAP', 'T5Config', 'T5OnnxConfig']}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     A__ :		Optional[int]					=['T5Tokenizer']

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     A__ :		Union[str, Any]					=['T5TokenizerFast']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     A__ :		Union[str, Any]					=[
         'T5_PRETRAINED_MODEL_ARCHIVE_LIST',
         'T5EncoderModel',
         'T5ForConditionalGeneration',
         'T5Model',
         'T5PreTrainedModel',
         'load_tf_weights_in_t5',
         'T5ForQuestionAnswering',
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     A__ :		List[str]					=[
         'TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST',
         'TFT5EncoderModel',
         'TFT5ForConditionalGeneration',
         'TFT5Model',
         'TFT5PreTrainedModel',
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     A__ :		Union[str, Any]					=[
         'FlaxT5EncoderModel',
         'FlaxT5ForConditionalGeneration',
         'FlaxT5Model',
         'FlaxT5PreTrainedModel',
     ]


if TYPE_CHECKING:
     from .configuration_ta import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, TaConfig, TaOnnxConfig

     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_ta import TaTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_ta_fast import TaTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_ta import (
              T5_PRETRAINED_MODEL_ARCHIVE_LIST,
              TaEncoderModel,
              TaForConditionalGeneration,
              TaForQuestionAnswering,
              TaModel,
              TaPreTrainedModel,
              load_tf_weights_in_ta,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_ta import (
              TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFTaEncoderModel,
              TFTaForConditionalGeneration,
              TFTaModel,
              TFTaPreTrainedModel,
          )

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_ta import (
              FlaxTaEncoderModel,
              FlaxTaForConditionalGeneration,
              FlaxTaModel,
              FlaxTaPreTrainedModel,
          )


else:
     import sys

     A__ :		Dict					=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



