

import torch

from accelerate import Accelerator, DistributedType






class 				A_					:


     def __enter__(				self     :						List[Any]   )				->      Any:
      if self.enabled:
       __lowerCAmelCase:  int        = self.model.no_sync()
       self.model_sync_obj.__enter__()

      return self


     def __exit__(				self     :						List[str]     ,    UpperCAmelCase     :						Optional[Any]     ,    UpperCAmelCase     :						int     ,    UpperCAmelCase     :						Optional[int]   )				->      List[str]:
      if self.enabled:
       # Average all models on exit
       self._sync_and_avg_model_params()
       self.model_sync_obj.__exit__(UpperCAmelCase     ,    UpperCAmelCase     ,    UpperCAmelCase   )


     def __init__(				self     :						List[Any]     ,    UpperCAmelCase     :						Accelerator     ,    UpperCAmelCase     :						torch.nn.Module     ,    UpperCAmelCase     :						int     ,    UpperCAmelCase     :						bool = True   )				->      Dict:
      if accelerator.distributed_type not in [
          DistributedType.NO,
          DistributedType.MULTI_CPU,
          DistributedType.MULTI_GPU,
      ]:
       raise NotImplementedError('LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'   )
      __lowerCAmelCase:  Union[str, Any]        = enabled and accelerator.distributed_type != DistributedType.NO
      __lowerCAmelCase:  Union[str, Any]        = 0
      if self.enabled:
       __lowerCAmelCase:  Dict        = accelerator
       __lowerCAmelCase:  Optional[int]        = model
       __lowerCAmelCase:  Any        = local_sgd_steps


     def     UpperCAmelCase    (				self     :						List[Any]   )				->      Optional[int]:
      self.num_steps += 1
      if not self.enabled:
       return

      if self.num_steps % self.local_sgd_steps == 0:
       self._sync_and_avg_model_params()


     def     UpperCAmelCase    (				self     :						Optional[int]   )				->      Any:
      self.accelerator.wait_for_everyone()
      with self.accelerator.autocast():
       for param in self.model.parameters():
        __lowerCAmelCase:  List[str]        = self.accelerator.reduce(param.data     ,    reduction='mean'   )








import numpy as np







def _a					(			SCREAMING_SNAKE_CASE						:     Union[str, Any]						)       ->       List[str]:

 """simple docstring"""






 __lowerCAmelCase:  Union[str, Any]        = np.exp(SCREAMING_SNAKE_CASE						)

 # Add up the all the exponentials
 __lowerCAmelCase:  Dict        = np.sum(SCREAMING_SNAKE_CASE						)

 # Divide every exponent by the sum of all exponents
 __lowerCAmelCase:  int        = exponent_vector / sum_of_exponents

 return softmax_vector


if __name__ == "__main__":
  print(softmax((0,)))





