




import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
 import torch

SCREAMING_SNAKE_CASE_    :       int       =							logging.get_logger(__name__)
def 							__lowerCAmelCase			():
    return importlib.util.find_spec("""deepspeed"""		) is not None


if is_accelerate_available() and is_deepspeed_available():
 from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
 # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
 # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
 from builtins import object as DeepSpeedConfig







class 		__UpperCAmelCase(		lowercase__			):





     """simple docstring"""



     def __init__(   self			:							List[Any]    ,					A			:							int     )      ->      Optional[int]:






         """simple docstring"""
         # set global weakref object
         set_hf_deepspeed_config(self     )
         dep_version_check("""accelerate"""     )
         dep_version_check("""deepspeed"""     )
         super().__init__(A     )







class 		__UpperCAmelCase(		lowercase__			):





     """simple docstring"""



     def __init__(   self			:							str    ,					A			:							List[Any]     )      ->      Optional[Any]:






         """simple docstring"""
         super().__init__(A     )
         a__  			=							None
         a__  			=							[]

     def 					__SCREAMING_SNAKE_CASE		(   self			:							Tuple     )      ->      List[str]:






         """simple docstring"""
         if self._dtype is None:
             raise ValueError("""trainer_config_process() wasn't called yet to tell dtype"""     )
         return self._dtype

     def 					__SCREAMING_SNAKE_CASE		(   self			:							Any    ,					A			:							Optional[Any]     )      ->      Dict:






         """simple docstring"""
         a__  			=							self.get_value(A     )
         if val is None:
             return False
         else:
             return val == "auto"

     def 					__SCREAMING_SNAKE_CASE		(   self			:							Dict    ,					A			:							Any    ,					A			:							int    ,					A			:							str=None    ,					A			:							List[Any]=True     )      ->      List[Any]:






         """simple docstring"""
         a__			,		a__  			=							self.find_config_node(A     )
         if config is None:
             return

         if config.get(A     ) == "auto":
             a__  			=							hf_val
             return

         if not must_match:
             return

         a__  			=							config.get(A     )
         if ds_val is not None and ds_val != hf_val:
             self.mismatches.append(f"""- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"""     )

     __UpperCAmelCase						:			List[Any]							=     partialmethod(lowercase__							,						must_match=lowercase__			)

     def 					__SCREAMING_SNAKE_CASE		(   self			:							Optional[int]    ,					A			:							Union[str, Any]     )      ->      Optional[int]:






         """simple docstring"""
         a__  			=							args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
         self.fill_match(
             """train_micro_batch_size_per_gpu"""    ,					args.per_device_train_batch_size    ,					"""per_device_train_batch_size"""     )
         self.fill_match("""gradient_accumulation_steps"""    ,					args.gradient_accumulation_steps    ,					"""gradient_accumulation_steps"""     )
         self.fill_match("""train_batch_size"""    ,					A    ,					"""train_batch_size (calculated)"""     )
         self.fill_match("""gradient_clipping"""    ,					args.max_grad_norm    ,					"""max_grad_norm"""     )

         self.fill_match("""optimizer.params.lr"""    ,					args.learning_rate    ,					"""learning_rate"""     )
         self.fill_match("""optimizer.params.betas"""    ,					[args.adam_betaa, args.adam_betaa]    ,					"""adam_beta1+adam_beta2"""     )
         self.fill_match("""optimizer.params.eps"""    ,					args.adam_epsilon    ,					"""adam_epsilon"""     )
         self.fill_match("""optimizer.params.weight_decay"""    ,					args.weight_decay    ,					"""weight_decay"""     )

         self.fill_only("""scheduler.params.warmup_min_lr"""    ,					0     )  # not a trainer arg
         self.fill_match("""scheduler.params.warmup_max_lr"""    ,					args.learning_rate    ,					"""learning_rate"""     )
         # total_num_steps - will get set in trainer_config_finalize

         # fp16
         if args.fpaa or args.fpaa_full_eval:
             a__  			=							"""apex""" if args.fpaa_backend == """apex""" else """amp"""
         else:
             a__  			=							None

         if args.save_on_each_node:
             # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
             a__  			=							self.config.get("""checkpoint"""    ,					{}     )
             a__  			=							args.save_on_each_node

         # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
         # any here unless the user did the work
         self.fill_match(
             """fp16.enabled"""    ,					((args.fpaa or args.fpaa_full_eval) and fpaa_backend == """amp""")    ,					"""fp16|fp16_full_eval+fp16_backend(amp)"""    ,					)

         # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
         # ZeRO features
         self.fill_match("""amp.enabled"""    ,					fpaa_backend == """apex"""    ,					"""fp16+fp16_backend(apex)"""     )
         self.fill_match("""amp.opt_level"""    ,					args.fpaa_opt_level    ,					"""fp16_opt_level"""     )

         self.fill_match("""bf16.enabled"""    ,					(args.bfaa or args.bfaa_full_eval)    ,					"""bf16|bf16_full_eval"""     )

         # deepspeed's default mode is fp16 unless there is a config that says differently
         if self.is_true("""bf16.enabled"""     ):
             a__  			=							torch.bfloataa
         elif self.is_false("""fp16.enabled"""     ):
             a__  			=							torch.floataa
         else:
             a__  			=							torch.floataa






     def 					__SCREAMING_SNAKE_CASE		(   self			:							Any    ,					A			:							str    ,					A			:							Tuple    ,					A			:							Union[str, Any]     )      ->      List[str]:






         """simple docstring"""
         a__  			=							[
             """zero_optimization.reduce_bucket_size""",
             """zero_optimization.stage3_prefetch_bucket_size""",
             """zero_optimization.stage3_param_persistence_threshold""",
         ]
         a__  			=							[x for x in hidden_size_based_keys if self.is_auto(A     )]

         if len(A     ) > 0:
             if hasattr(model.config    ,					"""hidden_size"""     ):
                 a__  			=							model.config.hidden_size
             elif hasattr(model.config    ,					"""hidden_sizes"""     ):
                 # if there are many hidden sizes pick the largest one
                 a__  			=							max(model.config.hidden_sizes     )
             else:
                 raise ValueError(
                     """The model's config file has neither `hidden_size` nor `hidden_sizes` entry, """
                     """therefore it's not possible to automatically fill out the following `auto` entries """
                     f"""in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing """
                     """`auto` values for these keys with an integer value of your choice."""     )

             self.fill_only("""zero_optimization.reduce_bucket_size"""    ,					hidden_size * hidden_size     )
             if self.is_zeroa():
                 # automatically assign the optimal config values based on model config
                 self.fill_only("""zero_optimization.stage3_prefetch_bucket_size"""    ,					0.9 * hidden_size * hidden_size     )
                 self.fill_only("""zero_optimization.stage3_param_persistence_threshold"""    ,					10 * hidden_size     )

        # scheduler
         self.fill_match("""scheduler.params.total_num_steps"""    ,					A    ,					"""num_training_steps (calculated)"""     )
         self.fill_match("""scheduler.params.warmup_num_steps"""    ,					args.get_warmup_steps(A     )    ,					"""warmup_steps"""     )

         if len(self.mismatches     ) > 0:
             a__  			=							"""\n""".join(self.mismatches     )
             raise ValueError(
                 """Please correct the following DeepSpeed config values that mismatch TrainingArguments"""
                 f""" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'."""     )






SCREAMING_SNAKE_CASE_    :       Tuple       =							None
def 							__lowerCAmelCase			(UpperCAmelCase_		):
    # this is a special weakref global object to allow us to get to Deepspeed config from APIs
    # that don't have an easy way to get to the Deepspeed config outside of the Trainer domain.
    global _hf_deepspeed_config_weak_ref
    # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
    a__  			=							weakref.ref(UpperCAmelCase_		)
def 							__lowerCAmelCase			():
    # useful for unit tests to ensure the global state doesn't leak - call from `tearDown` method
    global _hf_deepspeed_config_weak_ref
    a__  			=							None
def 							__lowerCAmelCase			():
    if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
        return _hf_deepspeed_config_weak_ref().is_zeroa()
    else:
        return False
def 							__lowerCAmelCase			():
    if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
        return _hf_deepspeed_config_weak_ref().config
    else:
        return None
def 							__lowerCAmelCase			(UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_		):
    from accelerate.utils import DummyOptim, DummyScheduler

    a__  			=							hf_deepspeed_config.config

    # Optimizer + Scheduler
    # Currently supported combos:
    # 1. DS scheduler + DS optimizer: Yes
    # 2. HF scheduler + HF optimizer: Yes
    # 3. DS scheduler + HF optimizer: Yes
    # 4. HF scheduler + DS optimizer: No
    #
    # Unless Offload is enabled in which case it's:
    # 1. DS scheduler + DS optimizer: Yes
    # 2. HF scheduler + HF optimizer: Mostly*
    # 3. DS scheduler + HF optimizer: Mostly*
    # 4. HF scheduler + DS optimizer: No
    #
    # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

    a__  			=							None
    if "optimizer" in config:
        if args.adafactor:
            raise ValueError(
                """--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. """
                """Only one optimizer can be configured."""		)
        a__  			=							DummyOptim(params=UpperCAmelCase_		)
    else:
        if hf_deepspeed_config.is_offload():
            logger.info(
                """Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"""
                """ custom optimizer has both CPU and GPU implementation (except LAMB)"""		)

        # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
        # But trainer uses AdamW by default.
        a__  			=							trainer.create_optimizer()
        # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
        a__  			=							True

    a__  			=							None
    if "scheduler" in config:
        a__  			=							DummyScheduler(UpperCAmelCase_		)
    else:
        if isinstance(UpperCAmelCase_ ,     UpperCAmelCase_		):
            raise ValueError(
                """Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. """
                """Please configure a scheduler in the DeepSpeed config."""		)
        a__  			=							trainer.create_scheduler(num_training_steps=UpperCAmelCase_ ,     optimizer=UpperCAmelCase_		)

    return optimizer, lr_scheduler
def 							__lowerCAmelCase			(UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_=False		):
    from deepspeed.utils import logger as ds_logger

    a__  			=							trainer.model
    a__  			=							trainer.args

    a__  			=							trainer.accelerator.state.deepspeed_plugin.hf_ds_config

    # resume config update - some bits like `model` and `num_training_steps` only become available during train
    hf_deepspeed_config.trainer_config_finalize(UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_		)

    # set the Deepspeed log level consistent with the Trainer
    ds_logger.setLevel(args.get_process_log_level()		)

    if inference:
        # only Z3 makes sense for the inference
        if not hf_deepspeed_config.is_zeroa():
            raise ValueError("""ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"""		)

        # in case the training config is re-used for inference
        hf_deepspeed_config.del_config_sub_tree("""optimizer"""		)
        hf_deepspeed_config.del_config_sub_tree("""lr_scheduler"""		)
        a__			,		a__  			=							None, None
        a__  			=							None
    else:
        a__  			=							None  # important for when deepspeed_init is used as re-init
        a__  			=							list(filter(lambda UpperCAmelCase_		: p.requires_grad ,     model.parameters()		)		)
        a__			,		a__  			=							deepspeed_optim_sched(
            UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_ ,     UpperCAmelCase_		)

    # keep for quick debug:
    # from pprint import pprint; pprint(config)

    return optimizer, lr_scheduler







def 							__lowerCAmelCase			(UpperCAmelCase_ ,     UpperCAmelCase_		):
    # it's possible that the user is trying to resume from model_path, which doesn't necessarily
    # contain a deepspeed checkpoint. e.g. examples just check if the dir exists and assume it's
    # a resume from a checkpoint and not just a local pretrained weight. So we check here if the
    # path contains what looks like a deepspeed checkpoint
    import glob

    a__  			=							sorted(glob.glob(F"""{checkpoint_path}/global_step*"""		)		)

    if len(UpperCAmelCase_		) > 0:
        logger.info(F"""Attempting to resume from {checkpoint_path}"""		)
        # this magically updates self.optimizer and self.lr_scheduler
        a__			,		a__  			=							deepspeed_engine.load_checkpoint(
            UpperCAmelCase_ ,     load_optimizer_states=UpperCAmelCase_ ,     load_lr_scheduler_states=UpperCAmelCase_		)
        if load_path is None:
            raise ValueError(F"""[deepspeed] failed to resume from checkpoint {checkpoint_path}"""		)
    else:
        raise ValueError(F"""Can't find a valid checkpoint at {checkpoint_path}"""		)












def 							__lowerCAmelCase			(UpperCAmelCase_		):
    return n * (2 * n - 1)
def 							__lowerCAmelCase			(UpperCAmelCase_		):
    a__  			=							(1 + 24 * n) ** 0.5
    return ((1 + root) / 6) % 1 == 0
def 							__lowerCAmelCase			(UpperCAmelCase_ = 144		):
    a__  			=							start
    a__  			=							hexagonal_num(UpperCAmelCase_		)
    while not is_pentagonal(UpperCAmelCase_		):
        n += 1
        a__  			=							hexagonal_num(UpperCAmelCase_		)
    return num


if __name__ == "__main__":
 print(F"{solution()} = ")






