


"""simple docstring"""






import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
	import torch

_lowerCamelCase  =	logging.get_logger(__name__)


def __lowercase  (   ):
				return importlib.util.find_spec("deepspeed"	) is not None


if is_accelerate_available() and is_deepspeed_available():
	from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
	# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
	# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
	from builtins import object as DeepSpeedConfig


class       lowerCamelCase_		(      snake_case__ ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__  ):
								# set global weakref object
								set_hf_deepspeed_config(self  )
								dep_version_check("accelerate"  )
								dep_version_check("deepspeed"  )
								super().__init__(lowercase_  )


class       lowerCamelCase_		(      snake_case__ ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__  ):
								super().__init__(lowercase_  )
								SCREAMING_SNAKE_CASE__  				=					None
								SCREAMING_SNAKE_CASE__  				=					[]



				def 	lowerCAmelCase__		(    self  ):
								if self._dtype is None:
												raise ValueError("trainer_config_process() wasn\'t called yet to tell dtype"  )
								return self._dtype



				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__  ):
								SCREAMING_SNAKE_CASE__  				=					self.get_value(lowercase_  )
								if val is None:
												return False
								else:
												return val == "auto"



				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__							,		UpperCAmelCase__=None							,		UpperCAmelCase__=True  ):
								SCREAMING_SNAKE_CASE__  				=					self.find_config_node(lowercase_  )
								if config is None:
												return

								if config.get(lowercase_  ) == "auto":
												SCREAMING_SNAKE_CASE__  				=					hf_val
												return

								if not must_match:
												return

								SCREAMING_SNAKE_CASE__  				=					config.get(lowercase_  )
								if ds_val is not None and ds_val != hf_val:
												self.mismatches.append(f'''- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}'''  )

				_lowerCAmelCase   :   Tuple   				=       partialmethod(snake_case__							,	must_match=snake_case__ )



				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__  ):
								SCREAMING_SNAKE_CASE__  				=					args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
								self.fill_match(
								    "train_micro_batch_size_per_gpu"							,		args.per_device_train_batch_size							,		"per_device_train_batch_size"  )
								self.fill_match("gradient_accumulation_steps"							,		args.gradient_accumulation_steps							,		"gradient_accumulation_steps"  )
								self.fill_match("train_batch_size"							,		lowercase_							,		"train_batch_size (calculated)"  )
								self.fill_match("gradient_clipping"							,		args.max_grad_norm							,		"max_grad_norm"  )

								self.fill_match("optimizer.params.lr"							,		args.learning_rate							,		"learning_rate"  )
								self.fill_match("optimizer.params.betas"							,		[args.adam_betaa, args.adam_betaa]							,		"adam_beta1+adam_beta2"  )
								self.fill_match("optimizer.params.eps"							,		args.adam_epsilon							,		"adam_epsilon"  )
								self.fill_match("optimizer.params.weight_decay"							,		args.weight_decay							,		"weight_decay"  )

								self.fill_only("scheduler.params.warmup_min_lr"							,		0  )  # not a trainer arg
								self.fill_match("scheduler.params.warmup_max_lr"							,		args.learning_rate							,		"learning_rate"  )
								# total_num_steps - will get set in trainer_config_finalize

								# fp16
								if args.fpaa or args.fpaa_full_eval:
												SCREAMING_SNAKE_CASE__  				=					'''apex''' if args.fpaa_backend == '''apex''' else '''amp'''
								else:
												SCREAMING_SNAKE_CASE__  				=					None

								if args.save_on_each_node:
												# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
												SCREAMING_SNAKE_CASE__  				=					self.config.get("checkpoint"							,		{}  )
												SCREAMING_SNAKE_CASE__  				=					args.save_on_each_node

								# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
								# any here unless the user did the work
								self.fill_match(
								    "fp16.enabled"							,		((args.fpaa or args.fpaa_full_eval) and fpaa_backend == "amp")							,		"fp16|fp16_full_eval+fp16_backend(amp)"							,		)

								# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
								# ZeRO features
								self.fill_match("amp.enabled"							,		fpaa_backend == "apex"							,		"fp16+fp16_backend(apex)"  )
								self.fill_match("amp.opt_level"							,		args.fpaa_opt_level							,		"fp16_opt_level"  )

								self.fill_match("bf16.enabled"							,		(args.bfaa or args.bfaa_full_eval)							,		"bf16|bf16_full_eval"  )

								# deepspeed's default mode is fp16 unless there is a config that says differently
								if self.is_true("bf16.enabled"  ):
												SCREAMING_SNAKE_CASE__  				=					torch.bfloataa
								elif self.is_false("fp16.enabled"  ):
												SCREAMING_SNAKE_CASE__  				=					torch.floataa
								else:
												SCREAMING_SNAKE_CASE__  				=					torch.floataa






				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__							,		UpperCAmelCase__  ):
								SCREAMING_SNAKE_CASE__  				=					[
								    '''zero_optimization.reduce_bucket_size''',
								    '''zero_optimization.stage3_prefetch_bucket_size''',
								    '''zero_optimization.stage3_param_persistence_threshold''',
								]
								SCREAMING_SNAKE_CASE__  				=					[x for x in hidden_size_based_keys if self.is_auto(lowercase_  )]

								if len(lowercase_  ) > 0:
												if hasattr(model.config							,		"hidden_size"  ):
																SCREAMING_SNAKE_CASE__  				=					model.config.hidden_size
												elif hasattr(model.config							,		"hidden_sizes"  ):
																# if there are many hidden sizes pick the largest one
																SCREAMING_SNAKE_CASE__  				=					max(model.config.hidden_sizes  )
												else:
																raise ValueError(
																    "The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, "
																    "therefore it\'s not possible to automatically fill out the following `auto` entries "
																    f'''in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '''
																    "`auto` values for these keys with an integer value of your choice."  )

												self.fill_only("zero_optimization.reduce_bucket_size"							,		hidden_size * hidden_size  )
												if self.is_zeroa():
																# automatically assign the optimal config values based on model config
																self.fill_only("zero_optimization.stage3_prefetch_bucket_size"							,		0.9 * hidden_size * hidden_size  )
																self.fill_only("zero_optimization.stage3_param_persistence_threshold"							,		10 * hidden_size  )

        # scheduler
								self.fill_match("scheduler.params.total_num_steps"							,		lowercase_							,		"num_training_steps (calculated)"  )
								self.fill_match("scheduler.params.warmup_num_steps"							,		args.get_warmup_steps(lowercase_  )							,		"warmup_steps"  )

								if len(self.mismatches  ) > 0:
												SCREAMING_SNAKE_CASE__  				=					'''\n'''.join(self.mismatches  )
												raise ValueError(
												    "Please correct the following DeepSpeed config values that mismatch TrainingArguments"
												    f''' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.'''  )


_lowerCamelCase  =	None


def __lowercase  (   lowerCamelCase_	:   Optional[int]	):
				global _hf_deepspeed_config_weak_ref
				# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
				SCREAMING_SNAKE_CASE__  				=					weakref.ref(lowerCamelCase_	)


def __lowercase  (   ):
				global _hf_deepspeed_config_weak_ref
				SCREAMING_SNAKE_CASE__  				=					None


def __lowercase  (   ):






				if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
								return _hf_deepspeed_config_weak_ref().is_zeroa()
				else:
								return False


def __lowercase  (   ):






				if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
								return _hf_deepspeed_config_weak_ref().config
				else:
								return None


def __lowercase  (   lowerCamelCase_	:   Dict    ,       lowerCamelCase_	:   Tuple    ,       lowerCamelCase_	:   int    ,       lowerCamelCase_	:   Union[str, Any]    ,       lowerCamelCase_	:   List[str]	):
				from accelerate.utils import DummyOptim, DummyScheduler

				SCREAMING_SNAKE_CASE__  				=					hf_deepspeed_config.config

				# Optimizer + Scheduler
				# Currently supported combos:
				# 1. DS scheduler + DS optimizer: Yes
				# 2. HF scheduler + HF optimizer: Yes
				# 3. DS scheduler + HF optimizer: Yes
				# 4. HF scheduler + DS optimizer: No
				#
				# Unless Offload is enabled in which case it's:
				# 1. DS scheduler + DS optimizer: Yes
				# 2. HF scheduler + HF optimizer: Mostly*
				# 3. DS scheduler + HF optimizer: Mostly*
				# 4. HF scheduler + DS optimizer: No
				#
				# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

				SCREAMING_SNAKE_CASE__  				=					None
				if "optimizer" in config:
								if args.adafactor:
												raise ValueError(
												    "--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. "
												    "Only one optimizer can be configured."	)
								SCREAMING_SNAKE_CASE__  				=					DummyOptim(params=lowerCamelCase_	)
				else:
								if hf_deepspeed_config.is_offload():
												logger.info(
												    "Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"
												    " custom optimizer has both CPU and GPU implementation (except LAMB)"	)

								# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
								# But trainer uses AdamW by default.
								SCREAMING_SNAKE_CASE__  				=					trainer.create_optimizer()
								# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
								SCREAMING_SNAKE_CASE__  				=					True

				SCREAMING_SNAKE_CASE__  				=					None
				if "scheduler" in config:
								SCREAMING_SNAKE_CASE__  				=					DummyScheduler(lowerCamelCase_	)
				else:
								if isinstance(lowerCamelCase_    ,       lowerCamelCase_	):
												raise ValueError(
												    "Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. "
												    "Please configure a scheduler in the DeepSpeed config."	)
								SCREAMING_SNAKE_CASE__  				=					trainer.create_scheduler(num_training_steps=lowerCamelCase_    ,       optimizer=lowerCamelCase_	)

				return optimizer, lr_scheduler


def __lowercase  (   lowerCamelCase_	:   Any    ,       lowerCamelCase_	:   Optional[Any]    ,       lowerCamelCase_	:   Optional[Any]=False	):
				from deepspeed.utils import logger as ds_logger

				SCREAMING_SNAKE_CASE__  				=					trainer.model
				SCREAMING_SNAKE_CASE__  				=					trainer.args

				SCREAMING_SNAKE_CASE__  				=					trainer.accelerator.state.deepspeed_plugin.hf_ds_config

				# resume config update - some bits like `model` and `num_training_steps` only become available during train
				hf_deepspeed_config.trainer_config_finalize(lowerCamelCase_    ,       lowerCamelCase_    ,       lowerCamelCase_	)

				# set the Deepspeed log level consistent with the Trainer
				ds_logger.setLevel(args.get_process_log_level()	)

				if inference:
								# only Z3 makes sense for the inference
								if not hf_deepspeed_config.is_zeroa():
												raise ValueError("ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"	)

								# in case the training config is re-used for inference
								hf_deepspeed_config.del_config_sub_tree("optimizer"	)
								hf_deepspeed_config.del_config_sub_tree("lr_scheduler"	)
								SCREAMING_SNAKE_CASE__  				=					None, None
								SCREAMING_SNAKE_CASE__  				=					None
				else:
								SCREAMING_SNAKE_CASE__  				=					None  # important for when deepspeed_init is used as re-init
								SCREAMING_SNAKE_CASE__  				=					list(filter(lambda lowerCamelCase_	: p.requires_grad    ,       model.parameters()	)	)
								SCREAMING_SNAKE_CASE__  				=					deepspeed_optim_sched(
								    lowerCamelCase_    ,       lowerCamelCase_    ,       lowerCamelCase_    ,       lowerCamelCase_    ,       lowerCamelCase_	)

				# keep for quick debug:
				# from pprint import pprint; pprint(config)

				return optimizer, lr_scheduler


def __lowercase  (   lowerCamelCase_	:   Tuple    ,       lowerCamelCase_	:   List[Any]	):
				import glob

				SCREAMING_SNAKE_CASE__  				=					sorted(glob.glob(F'''{checkpoint_path}/global_step*'''	)	)

				if len(lowerCamelCase_	) > 0:
								logger.info(F'''Attempting to resume from {checkpoint_path}'''	)
								# this magically updates self.optimizer and self.lr_scheduler
								SCREAMING_SNAKE_CASE__  				=					deepspeed_engine.load_checkpoint(
								    lowerCamelCase_    ,       load_optimizer_states=lowerCamelCase_    ,       load_lr_scheduler_states=lowerCamelCase_	)
								if load_path is None:
												raise ValueError(F'''[deepspeed] failed to resume from checkpoint {checkpoint_path}'''	)
				else:
								raise ValueError(F'''Can\'t find a valid checkpoint at {checkpoint_path}'''	)



"""simple docstring"""





def 							_lowerCAmelCase   (    UpperCAmelCase  :				int   ,				UpperCAmelCase  :				int   ,				UpperCAmelCase  :				int					):


					'''simple docstring'''






					if power < 0:
										return -1
					base %= mod
					UpperCamelCase__		:     Any								=1

					while power > 0:
										if power & 1:
															UpperCamelCase__		:     Optional[Any]								=(result * base) % mod
										UpperCamelCase__		:     Union[str, Any]								=power >> 1
										UpperCamelCase__		:     Union[str, Any]								=(base * base) % mod

					return result



def 							_lowerCAmelCase   (    ):


					'''simple docstring'''






					print(modular_exponential(3   ,				200   ,				13					)					)


if __name__ == "__main__":
				import doctest

				doctest.testmod()

				main()


