

"""simple docstring"""





import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
				import torch

_SCREAMING_SNAKE_CASE							:	Tuple										=			logging.get_logger(__name__)



def 							_lowerCAmelCase   (    ):


					'''simple docstring'''






					return importlib.util.find_spec('''deepspeed'''					) is not None


if is_accelerate_available() and is_deepspeed_available():
				from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
				# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
				# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
				from builtins import object as DeepSpeedConfig

class      __a (   snake_case__     ):

			"""simple docstring"""

			def __init__(				self  :       Dict    ,	lowercase_  :       Any    ):
								# set global weakref object
								set_hf_deepspeed_config(self    )
								dep_version_check('''accelerate'''    )
								dep_version_check('''deepspeed'''    )
								super().__init__(lowercase_    )



class      __a (   snake_case__     ):

			"""simple docstring"""

			def __init__(				self  :       List[Any]    ,	lowercase_  :       Optional[int]    ):
								super().__init__(lowercase_    )
								UpperCamelCase__		:     Union[str, Any]								=None
								UpperCamelCase__		:     Optional[int]								=[]






			def  _lowerCAmelCase      (				self  :       List[Any]    ):
								if self._dtype is None:
													raise ValueError('''trainer_config_process() wasn\'t called yet to tell dtype'''    )
								return self._dtype






			def  _lowerCAmelCase      (				self  :       List[str]    ,	lowercase_  :       int    ):
								UpperCamelCase__		:     str								=self.get_value(lowercase_    )
								if val is None:
													return False
								else:
													return val == "auto"






			def  _lowerCAmelCase      (				self  :       Optional[Any]    ,	lowercase_  :       List[Any]    ,	lowercase_  :       List[Any]    ,	lowercase_  :       Union[str, Any]=None    ,	lowercase_  :       Union[str, Any]=True    ):
								UpperCamelCase__	,      UpperCamelCase__		:     Tuple								=self.find_config_node(lowercase_    )
								if config is None:
													return

								if config.get(lowercase_    ) == "auto":
													UpperCamelCase__		:     Union[str, Any]								=hf_val
													return

								if not must_match:
													return

								UpperCamelCase__		:     Tuple								=config.get(lowercase_    )
								if ds_val is not None and ds_val != hf_val:
													self.mismatches.append(f'''- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}'''    )

			SCREAMING_SNAKE_CASE_      =    partialmethod(snake_case__,     must_match=snake_case__     )






			def  _lowerCAmelCase      (				self  :       Any    ,	lowercase_  :       int    ):
								UpperCamelCase__		:     List[str]								=args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
								self.fill_match(
								    '''train_micro_batch_size_per_gpu'''    ,	args.per_device_train_batch_size    ,	'''per_device_train_batch_size'''    )
								self.fill_match('''gradient_accumulation_steps'''    ,	args.gradient_accumulation_steps    ,	'''gradient_accumulation_steps'''    )
								self.fill_match('''train_batch_size'''    ,	lowercase_    ,	'''train_batch_size (calculated)'''    )
								self.fill_match('''gradient_clipping'''    ,	args.max_grad_norm    ,	'''max_grad_norm'''    )

								self.fill_match('''optimizer.params.lr'''    ,	args.learning_rate    ,	'''learning_rate'''    )
								self.fill_match('''optimizer.params.betas'''    ,	[args.adam_betaa, args.adam_betaa]    ,	'''adam_beta1+adam_beta2'''    )
								self.fill_match('''optimizer.params.eps'''    ,	args.adam_epsilon    ,	'''adam_epsilon'''    )
								self.fill_match('''optimizer.params.weight_decay'''    ,	args.weight_decay    ,	'''weight_decay'''    )

								self.fill_only('''scheduler.params.warmup_min_lr'''    ,	0    )  # not a trainer arg
								self.fill_match('''scheduler.params.warmup_max_lr'''    ,	args.learning_rate    ,	'''learning_rate'''    )
								# total_num_steps - will get set in trainer_config_finalize

								# fp16
								if args.fpaa or args.fpaa_full_eval:
													UpperCamelCase__		:     List[str]								='''apex''' if args.fpaa_backend == '''apex''' else '''amp'''
								else:
													UpperCamelCase__		:     Union[str, Any]								=None

								if args.save_on_each_node:
													# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
													UpperCamelCase__		:     Tuple								=self.config.get('''checkpoint'''    ,	{}    )
													UpperCamelCase__		:     Any								=args.save_on_each_node

								# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
								# any here unless the user did the work
								self.fill_match(
								    '''fp16.enabled'''    ,	((args.fpaa or args.fpaa_full_eval) and fpaa_backend == '''amp''')    ,	'''fp16|fp16_full_eval+fp16_backend(amp)'''    ,	)

								# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
								# ZeRO features
								self.fill_match('''amp.enabled'''    ,	fpaa_backend == '''apex'''    ,	'''fp16+fp16_backend(apex)'''    )
								self.fill_match('''amp.opt_level'''    ,	args.fpaa_opt_level    ,	'''fp16_opt_level'''    )

								self.fill_match('''bf16.enabled'''    ,	(args.bfaa or args.bfaa_full_eval)    ,	'''bf16|bf16_full_eval'''    )

								# deepspeed's default mode is fp16 unless there is a config that says differently
								if self.is_true('''bf16.enabled'''    ):
													UpperCamelCase__		:     str								=torch.bfloataa
								elif self.is_false('''fp16.enabled'''    ):
													UpperCamelCase__		:     str								=torch.floataa
								else:
													UpperCamelCase__		:     str								=torch.floataa






			def  _lowerCAmelCase      (				self  :       List[str]    ,	lowercase_  :       int    ,	lowercase_  :       Optional[int]    ,	lowercase_  :       Dict    ):
								UpperCamelCase__		:     str								=[
								    '''zero_optimization.reduce_bucket_size''',
								    '''zero_optimization.stage3_prefetch_bucket_size''',
								    '''zero_optimization.stage3_param_persistence_threshold''',
								]
								UpperCamelCase__		:     Union[str, Any]								=[x for x in hidden_size_based_keys if self.is_auto(lowercase_    )]

								if len(lowercase_    ) > 0:
													if hasattr(model.config    ,	'''hidden_size'''    ):
																		UpperCamelCase__		:     Union[str, Any]								=model.config.hidden_size
													elif hasattr(model.config    ,	'''hidden_sizes'''    ):
																		# if there are many hidden sizes pick the largest one
																		UpperCamelCase__		:     List[str]								=max(model.config.hidden_sizes    )
													else:
																		raise ValueError(
																		    '''The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '''
																		    '''therefore it\'s not possible to automatically fill out the following `auto` entries '''
																		    f'''in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '''
																		    '''`auto` values for these keys with an integer value of your choice.'''    )

													self.fill_only('''zero_optimization.reduce_bucket_size'''    ,	hidden_size * hidden_size    )
													if self.is_zeroa():
																		# automatically assign the optimal config values based on model config
																		self.fill_only('''zero_optimization.stage3_prefetch_bucket_size'''    ,	0.9 * hidden_size * hidden_size    )
																		self.fill_only('''zero_optimization.stage3_param_persistence_threshold'''    ,	10 * hidden_size    )

        # scheduler
								self.fill_match('''scheduler.params.total_num_steps'''    ,	lowercase_    ,	'''num_training_steps (calculated)'''    )
								self.fill_match('''scheduler.params.warmup_num_steps'''    ,	args.get_warmup_steps(lowercase_    )    ,	'''warmup_steps'''    )

								if len(self.mismatches    ) > 0:
													UpperCamelCase__		:     Dict								='''\n'''.join(self.mismatches    )
													raise ValueError(
													    '''Please correct the following DeepSpeed config values that mismatch TrainingArguments'''
													    f''' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.'''    )



_SCREAMING_SNAKE_CASE							:	Optional[int]										=			None



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Optional[int]					):


					'''simple docstring'''






					global _hf_deepspeed_config_weak_ref
					# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
					UpperCamelCase__		:     Optional[Any]								=weakref.ref(UpperCAmelCase					)



def 							_lowerCAmelCase   (    ):


					'''simple docstring'''






					global _hf_deepspeed_config_weak_ref
					UpperCamelCase__		:     List[str]								=None



def 							_lowerCAmelCase   (    ):


					'''simple docstring'''






					if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
										return _hf_deepspeed_config_weak_ref().is_zeroa()
					else:
										return False



def 							_lowerCAmelCase   (    ):


					'''simple docstring'''






					if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
										return _hf_deepspeed_config_weak_ref().config
					else:
										return None



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Dict   ,				UpperCAmelCase  :				Tuple   ,				UpperCAmelCase  :				int   ,				UpperCAmelCase  :				Union[str, Any]   ,				UpperCAmelCase  :				List[str]					):


					'''simple docstring'''






					from accelerate.utils import DummyOptim, DummyScheduler

					UpperCamelCase__		:     Tuple								=hf_deepspeed_config.config

					# Optimizer + Scheduler
					# Currently supported combos:
					# 1. DS scheduler + DS optimizer: Yes
					# 2. HF scheduler + HF optimizer: Yes
					# 3. DS scheduler + HF optimizer: Yes
					# 4. HF scheduler + DS optimizer: No
					#
					# Unless Offload is enabled in which case it's:
					# 1. DS scheduler + DS optimizer: Yes
					# 2. HF scheduler + HF optimizer: Mostly*
					# 3. DS scheduler + HF optimizer: Mostly*
					# 4. HF scheduler + DS optimizer: No
					#
					# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

					UpperCamelCase__		:     Union[str, Any]								=None
					if "optimizer" in config:
										if args.adafactor:
															raise ValueError(
															    '''--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '''
															    '''Only one optimizer can be configured.'''					)
										UpperCamelCase__		:     Tuple								=DummyOptim(params=UpperCAmelCase					)
					else:
										if hf_deepspeed_config.is_offload():
															logger.info(
															    '''Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'''
															    ''' custom optimizer has both CPU and GPU implementation (except LAMB)'''					)

										# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
										# But trainer uses AdamW by default.
										UpperCamelCase__		:     str								=trainer.create_optimizer()
										# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
										UpperCamelCase__		:     Tuple								=True

					UpperCamelCase__		:     int								=None
					if "scheduler" in config:
										UpperCamelCase__		:     Optional[Any]								=DummyScheduler(UpperCAmelCase					)
					else:
										if isinstance(UpperCAmelCase   ,				UpperCAmelCase					):
															raise ValueError(
															    '''Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '''
															    '''Please configure a scheduler in the DeepSpeed config.'''					)
										UpperCamelCase__		:     Optional[Any]								=trainer.create_scheduler(num_training_steps=UpperCAmelCase   ,				optimizer=UpperCAmelCase					)

					return optimizer, lr_scheduler



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Any   ,				UpperCAmelCase  :				Optional[Any]   ,				UpperCAmelCase  :				Optional[Any]=False					):


					'''simple docstring'''






					from deepspeed.utils import logger as ds_logger

					UpperCamelCase__		:     Optional[int]								=trainer.model
					UpperCamelCase__		:     Union[str, Any]								=trainer.args

					UpperCamelCase__		:     str								=trainer.accelerator.state.deepspeed_plugin.hf_ds_config

					# resume config update - some bits like `model` and `num_training_steps` only become available during train
					hf_deepspeed_config.trainer_config_finalize(UpperCAmelCase   ,				UpperCAmelCase   ,				UpperCAmelCase					)

					# set the Deepspeed log level consistent with the Trainer
					ds_logger.setLevel(args.get_process_log_level()					)

					if inference:
										# only Z3 makes sense for the inference
										if not hf_deepspeed_config.is_zeroa():
															raise ValueError('''ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'''					)

										# in case the training config is re-used for inference
										hf_deepspeed_config.del_config_sub_tree('''optimizer'''					)
										hf_deepspeed_config.del_config_sub_tree('''lr_scheduler'''					)
										UpperCamelCase__	,      UpperCamelCase__		:     Union[str, Any]								=None, None
										UpperCamelCase__		:     Dict								=None
					else:
										UpperCamelCase__		:     List[str]								=None  # important for when deepspeed_init is used as re-init
										UpperCamelCase__		:     Optional[Any]								=list(filter(lambda UpperCAmelCase					: p.requires_grad   ,				model.parameters()					)					)
										UpperCamelCase__	,      UpperCamelCase__		:     List[Any]								=deepspeed_optim_sched(
										    UpperCAmelCase   ,				UpperCAmelCase   ,				UpperCAmelCase   ,				UpperCAmelCase   ,				UpperCAmelCase					)

					# keep for quick debug:
					# from pprint import pprint; pprint(config)

					return optimizer, lr_scheduler







def 							_lowerCAmelCase   (    UpperCAmelCase  :				Tuple   ,				UpperCAmelCase  :				List[Any]					):


					'''simple docstring'''






					import glob

					UpperCamelCase__		:     Optional[int]								=sorted(glob.glob(F'''{checkpoint_path}/global_step*'''					)					)

					if len(UpperCAmelCase					) > 0:
										logger.info(F'''Attempting to resume from {checkpoint_path}'''					)
										# this magically updates self.optimizer and self.lr_scheduler
										UpperCamelCase__	,      UpperCamelCase__		:     Union[str, Any]								=deepspeed_engine.load_checkpoint(
										    UpperCAmelCase   ,				load_optimizer_states=UpperCAmelCase   ,				load_lr_scheduler_states=UpperCAmelCase					)
										if load_path is None:
															raise ValueError(F'''[deepspeed] failed to resume from checkpoint {checkpoint_path}'''					)
					else:
										raise ValueError(F'''Can\'t find a valid checkpoint at {checkpoint_path}'''					)





"""simple docstring"""





def 							_lowerCAmelCase   (    UpperCAmelCase  :				int					):


					'''simple docstring'''






					if not isinstance(UpperCAmelCase   ,				UpperCAmelCase					):
										raise ValueError('''double_factorial() only accepts integral values'''					)
					if n < 0:
										raise ValueError('''double_factorial() not defined for negative values'''					)
					return 1 if n <= 1 else n * double_factorial(n - 2					)


if __name__ == "__main__":
				import doctest

				doctest.testmod()


