


import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
       import torch

__UpperCamelCase							:							int    							=					logging.get_logger(__name__)

def       _a	(					):







 """simple docstring"""



 return importlib.util.find_spec('''deepspeed'''  ) is not None


if is_accelerate_available() and is_deepspeed_available():
       from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
       # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
       # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
       from builtins import object as DeepSpeedConfig

class    __magic_name__						(  lowercase__):


     def __init__(   self				:  List[Any]				,						lowerCamelCase__				:  int      )	->   Optional[int]:





      '''simple docstring'''






      set_hf_deepspeed_config(self      )
      dep_version_check('''accelerate'''      )
      dep_version_check('''deepspeed'''      )
      super().__init__(lowerCamelCase__      )



class    __magic_name__						(  lowercase__):


     def __init__(   self				:  str				,						lowerCamelCase__				:  List[Any]      )	->   Optional[Any]:





      '''simple docstring'''






      super().__init__(lowerCamelCase__      )
      UpperCamelCase__       :     Tuple 							= None
      UpperCamelCase__       :     Tuple 							= []


     def   UpperCAmelCase__						(   self				:  Tuple      )	->   List[str]:





      '''simple docstring'''






      if self._dtype is None:
       raise ValueError('''trainer_config_process() wasn\'t called yet to tell dtype'''      )
      return self._dtype


     def   UpperCAmelCase__						(   self				:  Any				,						lowerCamelCase__				:  Optional[Any]      )	->   Dict:





      '''simple docstring'''






      UpperCamelCase__       :     List[Any] 							= self.get_value(lowerCamelCase__      )
      if val is None:
       return False
      else:
       return val == "auto"


     def   UpperCAmelCase__						(   self				:  Dict				,						lowerCamelCase__				:  Any				,						lowerCamelCase__				:  int				,						lowerCamelCase__				:  str=None				,						lowerCamelCase__				:  List[Any]=True      )	->   List[Any]:





      '''simple docstring'''






      UpperCamelCase__							,					UpperCamelCase__       :     Dict 							= self.find_config_node(lowerCamelCase__      )
      if config is None:
       return

      if config.get(lowerCamelCase__      ) == "auto":
       UpperCamelCase__       :     List[str] 							= hf_val
       return

      if not must_match:
       return

      UpperCamelCase__       :     Tuple 							= config.get(lowerCamelCase__      )
      if ds_val is not None and ds_val != hf_val:
       self.mismatches.append(F"- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"      )

     A:		List[Any]								=   partialmethod(lowercase__       ,					must_match=lowercase__)


     def   UpperCAmelCase__						(   self				:  Optional[int]				,						lowerCamelCase__				:  Union[str, Any]      )	->   Optional[int]:





      '''simple docstring'''






      UpperCamelCase__       :     Any 							= args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
      self.fill_match(
          '''train_micro_batch_size_per_gpu'''				,						args.per_device_train_batch_size				,						'''per_device_train_batch_size'''      )
      self.fill_match('''gradient_accumulation_steps'''				,						args.gradient_accumulation_steps				,						'''gradient_accumulation_steps'''      )
      self.fill_match('''train_batch_size'''				,						lowerCamelCase__				,						'''train_batch_size (calculated)'''      )
      self.fill_match('''gradient_clipping'''				,						args.max_grad_norm				,						'''max_grad_norm'''      )

      self.fill_match('''optimizer.params.lr'''				,						args.learning_rate				,						'''learning_rate'''      )
      self.fill_match('''optimizer.params.betas'''				,						[args.adam_betaa, args.adam_betaa]				,						'''adam_beta1+adam_beta2'''      )
      self.fill_match('''optimizer.params.eps'''				,						args.adam_epsilon				,						'''adam_epsilon'''      )
      self.fill_match('''optimizer.params.weight_decay'''				,						args.weight_decay				,						'''weight_decay'''      )

      self.fill_only('''scheduler.params.warmup_min_lr'''				,						0      )  # not a trainer arg
      self.fill_match('''scheduler.params.warmup_max_lr'''				,						args.learning_rate				,						'''learning_rate'''      )
      # total_num_steps - will get set in trainer_config_finalize

      # fp16
      if args.fpaa or args.fpaa_full_eval:
       UpperCamelCase__       :     Any 							= '''apex''' if args.fpaa_backend == '''apex''' else '''amp'''
      else:
       UpperCamelCase__       :     Optional[Any] 							= None

      if args.save_on_each_node:
       # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
       UpperCamelCase__       :     Union[str, Any] 							= self.config.get('''checkpoint'''				,						{}      )
       UpperCamelCase__       :     int 							= args.save_on_each_node

      # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
      # any here unless the user did the work
      self.fill_match(
          '''fp16.enabled'''				,						((args.fpaa or args.fpaa_full_eval) and fpaa_backend == '''amp''')				,						'''fp16|fp16_full_eval+fp16_backend(amp)'''				,						)

      # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
      # ZeRO features
      self.fill_match('''amp.enabled'''				,						fpaa_backend == '''apex'''				,						'''fp16+fp16_backend(apex)'''      )
      self.fill_match('''amp.opt_level'''				,						args.fpaa_opt_level				,						'''fp16_opt_level'''      )

      self.fill_match('''bf16.enabled'''				,						(args.bfaa or args.bfaa_full_eval)				,						'''bf16|bf16_full_eval'''      )

      # deepspeed's default mode is fp16 unless there is a config that says differently
      if self.is_true('''bf16.enabled'''      ):
       UpperCamelCase__       :     Tuple 							= torch.bfloataa
      elif self.is_false('''fp16.enabled'''      ):
       UpperCamelCase__       :     str 							= torch.floataa
      else:
       UpperCamelCase__       :     Optional[Any] 							= torch.floataa


     def   UpperCAmelCase__						(   self				:  Any				,						lowerCamelCase__				:  str				,						lowerCamelCase__				:  Tuple				,						lowerCamelCase__				:  Union[str, Any]      )	->   List[str]:





      '''simple docstring'''






      UpperCamelCase__       :     Optional[Any] 							= [
          '''zero_optimization.reduce_bucket_size''',
          '''zero_optimization.stage3_prefetch_bucket_size''',
          '''zero_optimization.stage3_param_persistence_threshold''',
      ]
      UpperCamelCase__       :     int 							= [x for x in hidden_size_based_keys if self.is_auto(lowerCamelCase__      )]

      if len(lowerCamelCase__      ) > 0:
       if hasattr(model.config				,						'''hidden_size'''      ):
        UpperCamelCase__       :     str 							= model.config.hidden_size
       elif hasattr(model.config				,						'''hidden_sizes'''      ):
        # if there are many hidden sizes pick the largest one
        UpperCamelCase__       :     Optional[int] 							= max(model.config.hidden_sizes      )
       else:
        raise ValueError(
            '''The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '''
            '''therefore it\'s not possible to automatically fill out the following `auto` entries '''
            F"in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing "
            '''`auto` values for these keys with an integer value of your choice.'''      )

       self.fill_only('''zero_optimization.reduce_bucket_size'''				,						hidden_size * hidden_size      )
       if self.is_zeroa():
        # automatically assign the optimal config values based on model config
        self.fill_only('''zero_optimization.stage3_prefetch_bucket_size'''				,						0.9 * hidden_size * hidden_size      )
        self.fill_only('''zero_optimization.stage3_param_persistence_threshold'''				,						10 * hidden_size      )

        # scheduler
      self.fill_match('''scheduler.params.total_num_steps'''				,						lowerCamelCase__				,						'''num_training_steps (calculated)'''      )
      self.fill_match('''scheduler.params.warmup_num_steps'''				,						args.get_warmup_steps(lowerCamelCase__      )				,						'''warmup_steps'''      )

      if len(self.mismatches      ) > 0:
       UpperCamelCase__       :     Optional[Any] 							= '''\n'''.join(self.mismatches      )
       raise ValueError(
           '''Please correct the following DeepSpeed config values that mismatch TrainingArguments'''
           F" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'."      )



__UpperCamelCase							:							Tuple    							=					None

def       _a	(					SCREAMING_SNAKE_CASE						:					List[str]  ):







 """simple docstring"""



 global _hf_deepspeed_config_weak_ref
 # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
 UpperCamelCase__       :     Optional[Any] 							= weakref.ref(UpperCAmelCase_  )

def       _a	(					):







 """simple docstring"""



 global _hf_deepspeed_config_weak_ref
 UpperCamelCase__       :     Optional[int] 							= None

def       _a	(					):







 """simple docstring"""



 if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
  return _hf_deepspeed_config_weak_ref().is_zeroa()
 else:
  return False

def       _a	(					):







 """simple docstring"""



 if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
  return _hf_deepspeed_config_weak_ref().config
 else:
  return None

def       _a	(					SCREAMING_SNAKE_CASE						:					Dict     ,			SCREAMING_SNAKE_CASE						:					Dict     ,			SCREAMING_SNAKE_CASE						:					int     ,			SCREAMING_SNAKE_CASE						:					List[Any]     ,			SCREAMING_SNAKE_CASE						:					List[str]  ):







 """simple docstring"""



 from accelerate.utils import DummyOptim, DummyScheduler

 UpperCamelCase__       :     List[Any] 							= hf_deepspeed_config.config

 # Optimizer + Scheduler
 # Currently supported combos:
 # 1. DS scheduler + DS optimizer: Yes
 # 2. HF scheduler + HF optimizer: Yes
 # 3. DS scheduler + HF optimizer: Yes
 # 4. HF scheduler + DS optimizer: No
 #
 # Unless Offload is enabled in which case it's:
 # 1. DS scheduler + DS optimizer: Yes
 # 2. HF scheduler + HF optimizer: Mostly*
 # 3. DS scheduler + HF optimizer: Mostly*
 # 4. HF scheduler + DS optimizer: No
 #
 # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

 UpperCamelCase__       :     Any 							= None
 if "optimizer" in config:
  if args.adafactor:
   raise ValueError(
       '''--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '''
       '''Only one optimizer can be configured.'''  )
  UpperCamelCase__       :     Optional[int] 							= DummyOptim(params=UpperCAmelCase_  )
 else:
  if hf_deepspeed_config.is_offload():
   logger.info(
       '''Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'''
       ''' custom optimizer has both CPU and GPU implementation (except LAMB)'''  )

  # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
  # But trainer uses AdamW by default.
  UpperCamelCase__       :     Optional[int] 							= trainer.create_optimizer()
  # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
  UpperCamelCase__       :     List[str] 							= True

 UpperCamelCase__       :     List[str] 							= None
 if "scheduler" in config:
  UpperCamelCase__       :     str 							= DummyScheduler(UpperCAmelCase_  )
 else:
  if isinstance(UpperCAmelCase_     ,			UpperCAmelCase_  ):
   raise ValueError(
       '''Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '''
       '''Please configure a scheduler in the DeepSpeed config.'''  )
  UpperCamelCase__       :     Optional[int] 							= trainer.create_scheduler(num_training_steps=UpperCAmelCase_     ,			optimizer=UpperCAmelCase_  )

 return optimizer, lr_scheduler

def       _a	(					SCREAMING_SNAKE_CASE						:					List[Any]     ,			SCREAMING_SNAKE_CASE						:					Dict     ,			SCREAMING_SNAKE_CASE						:					Union[str, Any]=False  ):







 """simple docstring"""



 from deepspeed.utils import logger as ds_logger

 UpperCamelCase__       :     List[str] 							= trainer.model
 UpperCamelCase__       :     List[str] 							= trainer.args

 UpperCamelCase__       :     Tuple 							= trainer.accelerator.state.deepspeed_plugin.hf_ds_config

 # resume config update - some bits like `model` and `num_training_steps` only become available during train
 hf_deepspeed_config.trainer_config_finalize(UpperCAmelCase_     ,			UpperCAmelCase_     ,			UpperCAmelCase_  )

 # set the Deepspeed log level consistent with the Trainer
 ds_logger.setLevel(args.get_process_log_level()  )

 if inference:
  # only Z3 makes sense for the inference
  if not hf_deepspeed_config.is_zeroa():
   raise ValueError('''ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'''  )

  # in case the training config is re-used for inference
  hf_deepspeed_config.del_config_sub_tree('''optimizer'''  )
  hf_deepspeed_config.del_config_sub_tree('''lr_scheduler'''  )
  UpperCamelCase__							,					UpperCamelCase__       :     Optional[int] 							= None, None
  UpperCamelCase__       :     Union[str, Any] 							= None
 else:
  UpperCamelCase__       :     Dict 							= None  # important for when deepspeed_init is used as re-init
  UpperCamelCase__       :     int 							= list(filter(lambda SCREAMING_SNAKE_CASE  : p.requires_grad     ,			model.parameters()  )  )
  UpperCamelCase__							,					UpperCamelCase__       :     Optional[Any] 							= deepspeed_optim_sched(
      UpperCAmelCase_     ,			UpperCAmelCase_     ,			UpperCAmelCase_     ,			UpperCAmelCase_     ,			UpperCAmelCase_  )

 # keep for quick debug:
 # from pprint import pprint; pprint(config)

 return optimizer, lr_scheduler






def       _a	(					SCREAMING_SNAKE_CASE						:					Union[str, Any]     ,			SCREAMING_SNAKE_CASE						:					Any  ):







 """simple docstring"""



 import glob

 UpperCamelCase__       :     Dict 							= sorted(glob.glob(F"{checkpoint_path}/global_step*"  )  )

 if len(UpperCAmelCase_  ) > 0:
  logger.info(F"Attempting to resume from {checkpoint_path}"  )
  # this magically updates self.optimizer and self.lr_scheduler
  UpperCamelCase__							,					UpperCamelCase__       :     List[str] 							= deepspeed_engine.load_checkpoint(
      UpperCAmelCase_     ,			load_optimizer_states=UpperCAmelCase_     ,			load_lr_scheduler_states=UpperCAmelCase_  )
  if load_path is None:
   raise ValueError(F"[deepspeed] failed to resume from checkpoint {checkpoint_path}"  )
 else:
  raise ValueError(F"Can't find a valid checkpoint at {checkpoint_path}"  )








import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


SCREAMING_SNAKE_CASE_    :       Tuple       =							logging.get_logger(__name__)







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def __init__(   self			:							int     )      ->      Union[str, Any]:






         """simple docstring"""
         super().__init__()
         if version.parse(torch.__version__     ) < version.parse("""1.12.0"""     ):
             raise ImportError(
                 f"""You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use """
                 """PytorchGELUTanh. Please upgrade torch."""     )






     def 					__SCREAMING_SNAKE_CASE		(   self			:							str    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return nn.functional.gelu(A    ,					approximate="""tanh"""     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							str    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi     ) * (input + 0.0_4_4_7_1_5 * torch.pow(A    ,					3.0     ))     ))







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def __init__(   self			:							Optional[Any]    ,					A			:							bool = False     )      ->      Optional[Any]:






         """simple docstring"""
         super().__init__()
         if use_gelu_python:
             a__  			=							self._gelu_python
         else:
             a__  			=							nn.functional.gelu

     def 					__SCREAMING_SNAKE_CASE		(   self			:							Dict    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0     )     ))






     def 					__SCREAMING_SNAKE_CASE		(   self			:							Union[str, Any]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return self.act(A     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							Any    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return 0.5 * input * (1.0 + torch.tanh(input * 0.7_9_7_8_8_4_5_6_0_8 * (1.0 + 0.0_4_4_7_1_5 * input * input)     ))







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							List[Any]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return input * torch.sigmoid(1.7_0_2 * input     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def __init__(   self			:							Tuple    ,					A			:							float    ,					A			:							float     )      ->      Optional[Any]:






         """simple docstring"""
         if min > max:
             raise ValueError(f"""min should be < max (got min: {min}, max: {max})"""     )

         super().__init__()
         a__  			=							min
         a__  			=							max






     def 					__SCREAMING_SNAKE_CASE		(   self			:							Optional[int]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return torch.clip(gelu(A     )    ,					self.min    ,					self.max     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def __init__(   self			:							Tuple     )      ->      List[Any]:






         """simple docstring"""
         super().__init__()
         a__  			=							math.sqrt(2 / math.pi     )






     def 					__SCREAMING_SNAKE_CASE		(   self			:							int    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.0_4_4_7_1_5 * torch.pow(A    ,					3     ))     ))







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							Optional[int]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return nn.functional.silu(A     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def __init__(   self			:							Optional[Any]     )      ->      List[str]:






         """simple docstring"""
         super().__init__()
         if version.parse(torch.__version__     ) < version.parse("""1.9.0"""     ):
             a__  			=							self._mish_python
         else:
             a__  			=							nn.functional.mish

     def 					__SCREAMING_SNAKE_CASE		(   self			:							Any    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return input * torch.tanh(nn.functional.softplus(A     )     )






     def 					__SCREAMING_SNAKE_CASE		(   self			:							Union[str, Any]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return self.act(A     )







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							Optional[Any]    ,					A			:							Tensor     )      ->      Tensor:






         """simple docstring"""
         return input







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							List[Any]    ,					A			:							str    ,					A			:							Dict=0.7_0_7_1_0_7    ,					A			:							Union[str, Any]=0.2_8_2_0_9_5     )      ->      int:






         """simple docstring"""
         a__  			=							(input - mu).div(sigma * math.sqrt(2.0     )     )
         return 0.5 * (1.0 + torch.erf(A     ))







class 		__UpperCAmelCase(		nn.Module			):





     """simple docstring"""



     def 					__SCREAMING_SNAKE_CASE		(   self			:							List[str]    ,					A			:							List[Any]     )      ->      Tuple:






         """simple docstring"""
         a__  			=							nn.functional.relu(A     )
         a__  			=							torch.square(A     )
         return squared







class 		__UpperCAmelCase(		lowercase__			):





     """simple docstring"""



     def __getitem__(   self			:							List[str]    ,					A			:							int     )      ->      Any:






         """simple docstring"""
         a__  			=							super().__getitem__(A     )
         a__			,		a__  			=							content if isinstance(A    ,					A     ) else (content, {})
         return cls(**A     )






SCREAMING_SNAKE_CASE_    :       List[Any]       =							{
    '''gelu''': GELUActivation,
    '''gelu_10''': (ClippedGELUActivation, {'''min''': -10, '''max''': 10}),
    '''gelu_fast''': FastGELUActivation,
    '''gelu_new''': NewGELUActivation,
    '''gelu_python''': (GELUActivation, {'''use_gelu_python''': True}),
    '''gelu_pytorch_tanh''': PytorchGELUTanh,
    '''gelu_accurate''': AccurateGELUActivation,
    '''laplace''': LaplaceActivation,
    '''linear''': LinearActivation,
    '''mish''': MishActivation,
    '''quick_gelu''': QuickGELUActivation,
    '''relu''': nn.ReLU,
    '''relu2''': ReLUSquaredActivation,
    '''relu6''': nn.ReLUa,
    '''sigmoid''': nn.Sigmoid,
    '''silu''': SiLUActivation,
    '''swish''': SiLUActivation,
    '''tanh''': nn.Tanh,
}
SCREAMING_SNAKE_CASE_    :       Any       =							ClassInstantier(ACTaCLS)
def 							__lowerCAmelCase			(UpperCAmelCase_		):
    if activation_string in ACTaFN:
        return ACTaFN[activation_string]
    else:
        raise KeyError(F"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()		)}"""		)


# For backwards compatibility with: from activations import gelu_python
SCREAMING_SNAKE_CASE_    :       Optional[int]       =							get_activation('''gelu_python''')
SCREAMING_SNAKE_CASE_    :       Union[str, Any]       =							get_activation('''gelu_new''')
SCREAMING_SNAKE_CASE_    :       str       =							get_activation('''gelu''')
SCREAMING_SNAKE_CASE_    :       Optional[Any]       =							get_activation('''gelu_fast''')
SCREAMING_SNAKE_CASE_    :       Union[str, Any]       =							get_activation('''quick_gelu''')
SCREAMING_SNAKE_CASE_    :       Tuple       =							get_activation('''silu''')
SCREAMING_SNAKE_CASE_    :       Union[str, Any]       =							get_activation('''mish''')







SCREAMING_SNAKE_CASE_    :       Tuple       =							get_activation('''linear''')






