






"""simple docstring"""



import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
 import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


a  :						Optional[int]					       =							get_logger(__name__)




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE = None       ,	)    ->Union[str, Any]:
 # Model states
 for i, state in enumerate(_SCREAMING_SNAKE_CASE			):
  UpperCAmelCase__ 						=				F'''{MODEL_NAME}.bin''' if i == 0 else F'''{MODEL_NAME}_{i}.bin'''
  UpperCAmelCase__ 						=				os.path.join(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  save(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  logger.info(F'''Model weights saved in {output_model_file}'''			)
 # Optimizer states
 for i, opt in enumerate(_SCREAMING_SNAKE_CASE			):
  UpperCAmelCase__ 						=				opt.state_dict()
  UpperCAmelCase__ 						=				F'''{OPTIMIZER_NAME}.bin''' if i == 0 else F'''{OPTIMIZER_NAME}_{i}.bin'''
  UpperCAmelCase__ 						=				os.path.join(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  save(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  logger.info(F'''Optimizer state saved in {output_optimizer_file}'''			)
 # Scheduler states
 for i, scheduler in enumerate(_SCREAMING_SNAKE_CASE			):
  UpperCAmelCase__ 						=				scheduler.state_dict()
  UpperCAmelCase__ 						=				F'''{SCHEDULER_NAME}.bin''' if i == 0 else F'''{SCHEDULER_NAME}_{i}.bin'''
  UpperCAmelCase__ 						=				os.path.join(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  save(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  logger.info(F'''Scheduler state saved in {output_scheduler_file}'''			)
 # GradScaler state
 if scaler is not None:
  UpperCAmelCase__ 						=				scaler.state_dict()
  UpperCAmelCase__ 						=				os.path.join(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  torch.save(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  logger.info(F'''Gradient scaler state saved in {output_scaler_file}'''			)
 # Random number generator states
 UpperCAmelCase__ 						=				{}
 UpperCAmelCase__ 						=				F'''{RNG_STATE_NAME}_{process_index}.pkl'''
 UpperCAmelCase__ 						=				random.getstate()
 UpperCAmelCase__ 						=				np.random.get_state()
 UpperCAmelCase__ 						=				torch.get_rng_state()
 if is_xpu_available():
  UpperCAmelCase__ 						=				torch.xpu.get_rng_state_all()
 else:
  UpperCAmelCase__ 						=				torch.cuda.get_rng_state_all()
 if is_tpu_available():
  UpperCAmelCase__ 						=				xm.get_rng_state()
 UpperCAmelCase__ 						=				os.path.join(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
 torch.save(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
 logger.info(F'''Random states saved in {output_states_file}'''			)
 return output_dir




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None       ,	_SCREAMING_SNAKE_CASE=None       ,	**_SCREAMING_SNAKE_CASE       ,	)    ->Any:
 if map_location not in [None, "cpu", "on_device"]:
  raise TypeError(
      """Unsupported optimizer map location passed, please choose one of `None`, `'cpu'`, or `'on_device'`"""			)
 if map_location is None:
  UpperCAmelCase__ 						=				"""cpu"""
 elif map_location == "on_device":
  UpperCAmelCase__ 						=				PartialState().device
 # Model states
 for i, model in enumerate(_SCREAMING_SNAKE_CASE			):
  UpperCAmelCase__ 						=				F'''{MODEL_NAME}.bin''' if i == 0 else F'''{MODEL_NAME}_{i}.bin'''
  UpperCAmelCase__ 						=				os.path.join(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  models[i].load_state_dict(torch.load(_SCREAMING_SNAKE_CASE       ,	map_location=_SCREAMING_SNAKE_CASE			)       ,	**_SCREAMING_SNAKE_CASE			)
 logger.info("""All model weights loaded successfully"""			)

 # Optimizer states
 for i, opt in enumerate(_SCREAMING_SNAKE_CASE			):
  UpperCAmelCase__ 						=				F'''{OPTIMIZER_NAME}.bin''' if i == 0 else F'''{OPTIMIZER_NAME}_{i}.bin'''
  UpperCAmelCase__ 						=				os.path.join(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  UpperCAmelCase__ 						=				torch.load(_SCREAMING_SNAKE_CASE       ,	map_location=_SCREAMING_SNAKE_CASE			)
  optimizers[i].load_state_dict(_SCREAMING_SNAKE_CASE			)
 logger.info("""All optimizer states loaded successfully"""			)

 # Scheduler states
 for i, scheduler in enumerate(_SCREAMING_SNAKE_CASE			):
  UpperCAmelCase__ 						=				F'''{SCHEDULER_NAME}.bin''' if i == 0 else F'''{SCHEDULER_NAME}_{i}.bin'''
  UpperCAmelCase__ 						=				os.path.join(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  scheduler.load_state_dict(torch.load(_SCREAMING_SNAKE_CASE			)			)
 logger.info("""All scheduler states loaded successfully"""			)

 # GradScaler state
 if scaler is not None:
  UpperCAmelCase__ 						=				os.path.join(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE			)
  scaler.load_state_dict(torch.load(_SCREAMING_SNAKE_CASE			)			)
  logger.info("""GradScaler state loaded successfully"""			)

 # Random states
 try:
  UpperCAmelCase__ 						=				torch.load(os.path.join(_SCREAMING_SNAKE_CASE       ,	F'''{RNG_STATE_NAME}_{process_index}.pkl'''			)			)
  random.setstate(states["""random_state"""]			)
  np.random.set_state(states["""numpy_random_seed"""]			)
  torch.set_rng_state(states["""torch_manual_seed"""]			)
  if is_xpu_available():
   torch.xpu.set_rng_state_all(states["""torch_xpu_manual_seed"""]			)
  else:
   torch.cuda.set_rng_state_all(states["""torch_cuda_manual_seed"""]			)
  if is_tpu_available():
   xm.set_rng_state(states["""xm_seed"""]			)
  logger.info("""All random states loaded successfully"""			)
 except Exception:
  logger.info("""Could not load random states"""			)




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE = 0			)    ->Optional[Any]:
 UpperCAmelCase__ 						=				Path(_SCREAMING_SNAKE_CASE			) / F'''custom_checkpoint_{index}.pkl'''
 logger.info(F'''Saving the state of {get_pretty_name(_SCREAMING_SNAKE_CASE			)} to {save_location}'''			)
 torch.save(obj.state_dict()       ,	_SCREAMING_SNAKE_CASE			)




def       snake_case__		(   _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE = 0			)    ->str:
 UpperCAmelCase__ 						=				F'''{path}/custom_checkpoint_{index}.pkl'''
 logger.info(F'''Loading the state of {get_pretty_name(_SCREAMING_SNAKE_CASE			)} from {load_location}'''			)
 obj.load_state_dict(torch.load(_SCREAMING_SNAKE_CASE       ,	map_location="""cpu"""			)			)











"""simple docstring"""



import os
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
 import torch

 from transformers import GPTaLMHeadModel


@require_torch
@require_sentencepiece
@require_tokenizers
class 	_UpperCamelCase ( unittest.TestCase							):



    '''simple docstring'''


    @slow
    @unittest.skip("""Model is not available."""			)
    def 		A__      (      self			):
     UpperCAmelCase__ 						=				"""nvidia/megatron-gpt2-345m/"""
     if "MYDIR" in os.environ:
      UpperCAmelCase__ 						=				os.path.join(os.environ["""MYDIR"""]   ,			__lowercase			)
     UpperCAmelCase__ 						=				GPTaLMHeadModel.from_pretrained(__lowercase			)
     model.to(__lowercase			)
     model.half()

     UpperCAmelCase__ 						=				torch.tensor(
         [[101, 7110, 1005, 1056, 2023, 11333, 17413, 1029, 102]]   ,			device=__lowercase   ,			dtype=torch.long   ,			)

     with torch.no_grad():
      UpperCAmelCase__ 						=				model(__lowercase			).logits

     UpperCAmelCase__ 						=				torch.Size((1, 9, 50257)			)
     self.assertEqual(output.shape   ,			__lowercase			)

     UpperCAmelCase__ 						=				torch.tensor(
         [
             4.9_414,
             -0.2_920,
             -1.2_148,
             -4.0_273,
             -0.5_161,
             -5.2_109,
             -1.2_412,
             -1.8_301,
             -1.7_734,
             -4.7_148,
             -0.2_317,
             -1.0_811,
             -2.1_777,
             0.4_141,
             -3.7_969,
             -4.0_586,
             -2.5_332,
             -3.3_809,
             4.3_867,
         ]   ,			device=__lowercase   ,			dtype=torch.half   ,			)

     for i in range(19			):
      UpperCAmelCase__						,	UpperCAmelCase__ 						=				8 * i // 17, 2792 * i  # along the diagonal
      UpperCAmelCase__						,	UpperCAmelCase__ 						=				output[0, r, c], expected_diag[i]
      UpperCAmelCase__ 						=				F'''row={r} col={c} computed={computed} expected={expected}'''
      self.assertAlmostEqual(__lowercase   ,			__lowercase   ,			delta=1e-4   ,			msg=__lowercase			)



