




import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available



def    UpperCAmelCase__	(  lowerCamelCase,      lowerCamelCase,      lowerCamelCase,      lowerCamelCase=None ):
  lowercase     :List[Any]   						=    None
  # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
  if str(weight.dtype ) == "torch.bfloat16":
    # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
    lowercase     :Tuple   						=    weight.view(torch.intaa )
    lowercase     :Union[str, Any]   						=    "bfloat16"
  lowercase     :Union[str, Any]   						=    weight.cpu().numpy()
  lowercase     :Tuple   						=    os.path.join(lowerCamelCase,      F"{weight_name}.dat" )
  if index is not None:
    if dtype is None:
      lowercase     :Optional[Any]   						=    str(array.dtype )
    lowercase     :Any   						=    {"dtype": dtype, "shape": list(array.shape )}
  if array.ndim == 0:
    lowercase     :Tuple   						=    array[None]
  lowercase     :str   						=    np.memmap(lowerCamelCase,      dtype=array.dtype,      mode="w+",      shape=array.shape )
  lowercase     :List[str]   						=    array[:]
  file_array.flush()
  return index



def    UpperCAmelCase__	(  lowerCamelCase,      lowerCamelCase ):
  lowercase     :int   						=    tuple(weight_info["shape"] )
  if shape == ():
    # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
    lowercase     :Union[str, Any]   						=    (1,)

  lowercase     :List[str]   						=    weight_info["dtype"]
  if dtype == "bfloat16":
    # NumPy does not support bfloat16 so this was saved as a int16
    lowercase     :List[Any]   						=    "int16"

  lowercase     :str   						=    np.memmap(lowerCamelCase,      dtype=lowerCamelCase,      shape=lowerCamelCase,      mode="r" )

  if len(weight_info["shape"] ) == 0:
    lowercase     :Optional[int]   						=    weight[0]
  lowercase     :Optional[int]   						=    torch.tensor(lowerCamelCase )
  if weight_info["dtype"] == "bfloat16":
    lowercase     :Tuple   						=    weight.view(torch.bfloataa )

  return weight



def    UpperCAmelCase__	(  lowerCamelCase,      lowerCamelCase ):
  if index is None or len(lowerCamelCase ) == 0:
    # Nothing to save
    return

  lowercase     :List[Any]   						=    os.path.join(lowerCamelCase,      "index.json" )
  if os.path.isfile(lowerCamelCase ):
    with open(lowerCamelCase,      "r",      encoding="utf-8" ) as f:
      lowercase     :Dict   						=    json.load(lowerCamelCase )
  else:
    lowercase     :Tuple   						=    {}
  current_index.update(lowerCamelCase )

  with open(lowerCamelCase,      "w",      encoding="utf-8" ) as f:
    json.dump(lowerCamelCase,      lowerCamelCase,      indent=2 )



def    UpperCAmelCase__	(  lowerCamelCase,      lowerCamelCase ):
  os.makedirs(lowerCamelCase,      exist_ok=lowerCamelCase )
  lowercase     :List[Any]   						=    {}
  for name, parameter in state_dict.items():
    lowercase     :Optional[int]   						=    offload_weight(lowerCamelCase,      lowerCamelCase,      lowerCamelCase,      index=lowerCamelCase )

  # Update index
  save_offload_index(lowerCamelCase,      lowerCamelCase )




class  __lowerCAmelCase							(				lowerCAmelCase):






       def __init__( self:       Optional[int]							,       _lowerCAmelCase:       Mapping							,       _lowerCAmelCase:       str					):
         lowercase     :str   						=    dataset
         lowercase     :str   						=    prefix






       def __getitem__( self:       Optional[int]							,       _lowerCAmelCase:       str					):
         return self.dataset[F"{self.prefix}{key}"]






       def __iter__( self:       int					):
         return iter([key for key in self.dataset if key.startswith(self.prefix					)]					)






       def __len__( self:       int					):
         return len(self.dataset					)







class  __lowerCAmelCase							(				lowerCAmelCase):






       def __init__( self:       List[Any]							,       _lowerCAmelCase:       Dict[str, torch.Tensor] = None							,       _lowerCAmelCase:       Optional[Union[str, os.PathLike]] = None							,       _lowerCAmelCase:       Mapping = None							,       _lowerCAmelCase:       Optional[Any]=None							,       ):
         if state_dict is None and save_folder is None:
           raise ValueError("Need either a `state_dict` or a `save_folder` containing offloaded weights."					)

         lowercase     :str   						=    {} if state_dict is None else state_dict
         lowercase     :Optional[int]   						=    save_folder
         if index is None and save_folder is not None:
           with open(os.path.join(_lowerCAmelCase							,       "index.json"					)					) as f:
             lowercase     :int   						=    json.load(_lowerCAmelCase					)
         lowercase     :Tuple   						=    {} if index is None else index
         lowercase     :int   						=    list(self.state_dict.keys()					)
         self.all_keys.extend([key for key in self.index if key not in self.all_keys]					)
         lowercase     :int   						=    device






       def __getitem__( self:       Tuple							,       _lowerCAmelCase:       str					):
         # State dict gets priority
         if key in self.state_dict:
           return self.state_dict[key]
         lowercase     :Dict   						=    self.index[key]
         if weight_info.get("safetensors_file"					) is not None:
           if not is_safetensors_available():
             raise ImportError("These offloaded weights require the use of safetensors: `pip install safetensors`."					)

           from safetensors import safe_open

           lowercase     :List[str]   						=    "cpu" if self.device is None else self.device
           with safe_open(weight_info["safetensors_file"]							,       framework="pt"							,       device=_lowerCAmelCase					) as f:
             lowercase     :List[Any]   						=    f.get_tensor(weight_info.get("weight_name"							,       _lowerCAmelCase					)					)

           if "dtype" in weight_info:
             return tensor.to(getattr(_lowerCAmelCase							,       weight_info["dtype"]					)					)
           else:
             return tensor

         lowercase     :List[str]   						=    os.path.join(self.save_folder							,       F"{key}.dat"					)
         return load_offloaded_weight(_lowerCAmelCase							,       _lowerCAmelCase					)






       def __iter__( self:       Any					):
         return iter(self.all_keys					)






       def __len__( self:       List[Any]					):
         return len(self.all_keys					)







def    UpperCAmelCase__	(  lowerCamelCase,      lowerCamelCase ):
  lowercase     :Any   						=    {}
  for module_name in submodule_names:
    # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
    # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
    result.update(
        {
            key: param
            for key, param in state_dict.items()
            if key == module_name or key.startswith(module_name + "." )
        } )
  return result









import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only

from utils import save_json



def    UpperCAmelCase__	(  lowerCamelCase ):
  lowercase     :Optional[Any]   						=    filter(lambda lowerCamelCase : p.requires_grad,      model.parameters() )
  lowercase     :Optional[int]   						=    sum([np.prod(p.size() ) for p in model_parameters] )
  return params


_UpperCAmelCase						:    Optional[Any]      =  logging.getLogger(__name__)




class  __lowerCAmelCase							(				pl.Callback):






       def        SCREAMING_SNAKE_CASE     ( self:       Union[str, Any]							,       _lowerCAmelCase:       List[Any]							,       _lowerCAmelCase:       Union[str, Any]					):
         lowercase     :Dict   						=    {F"lr_group_{i}": param["lr"] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups					)}
         pl_module.logger.log_metrics(_lowerCAmelCase					)






       @rank_zero_only
       def        SCREAMING_SNAKE_CASE     ( self:       Optional[Any]							,       _lowerCAmelCase:       pl.Trainer							,       _lowerCAmelCase:       pl.LightningModule							,       _lowerCAmelCase:       str							,       _lowerCAmelCase:       List[str]=True					):
         logger.info(F"***** {type_path} results at step {trainer.global_step:05d} *****"					)
         lowercase     :Tuple   						=    trainer.callback_metrics
         trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ["log", "progress_bar", "preds"]}					)
         # Log results
         lowercase     :Optional[int]   						=    Path(pl_module.hparams.output_dir					)
         if type_path == "test":
           lowercase     :List[Any]   						=    od / "test_results.txt"
           lowercase     :Optional[int]   						=    od / "test_generations.txt"
         else:
           # this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
           # If people want this it will be easy enough to add back.
           lowercase     :int   						=    od / F"{type_path}_results/{trainer.global_step:05d}.txt"
           lowercase     :Tuple   						=    od / F"{type_path}_generations/{trainer.global_step:05d}.txt"
           results_file.parent.mkdir(exist_ok=_lowerCAmelCase					)
           generations_file.parent.mkdir(exist_ok=_lowerCAmelCase					)
         with open(_lowerCAmelCase							,       "a+"					) as writer:
           for key in sorted(_lowerCAmelCase					):
             if key in ["log", "progress_bar", "preds"]:
               continue
             lowercase     :int   						=    metrics[key]
             if isinstance(_lowerCAmelCase							,       torch.Tensor					):
               lowercase     :int   						=    val.item()
             lowercase     :Optional[Any]   						=    F"{key}: {val:.6f}\n"
             writer.write(_lowerCAmelCase					)

         if not save_generations:
           return

         if "preds" in metrics:
           lowercase     :str   						=    "\n".join(metrics["preds"]					)
           generations_file.open("w+"					).write(_lowerCAmelCase					)






       @rank_zero_only
       def        SCREAMING_SNAKE_CASE     ( self:       Optional[int]							,       _lowerCAmelCase:       Tuple							,       _lowerCAmelCase:       Tuple					):
         try:
           lowercase     :Optional[int]   						=    pl_module.model.model.num_parameters()
         except AttributeError:
           lowercase     :Dict   						=    pl_module.model.num_parameters()

         lowercase     :Any   						=    count_trainable_parameters(_lowerCAmelCase					)
         # mp stands for million parameters
         trainer.logger.log_metrics({"n_params": npars, "mp": npars / 1e6, "grad_mp": n_trainable_pars / 1e6}					)






       @rank_zero_only
       def        SCREAMING_SNAKE_CASE     ( self:       Union[str, Any]							,       _lowerCAmelCase:       pl.Trainer							,       _lowerCAmelCase:       pl.LightningModule					):
         save_json(pl_module.metrics							,       pl_module.metrics_save_path					)
         return self._write_logs(_lowerCAmelCase							,       _lowerCAmelCase							,       "test"					)






       @rank_zero_only
       def        SCREAMING_SNAKE_CASE     ( self:       List[str]							,       _lowerCAmelCase:       pl.Trainer							,       _lowerCAmelCase:       Dict					):
         save_json(pl_module.metrics							,       pl_module.metrics_save_path					)
         # Uncommenting this will save val generations
         # return self._write_logs(trainer, pl_module, "valid")







def    UpperCAmelCase__	(  lowerCamelCase,      lowerCamelCase,      lowerCamelCase=1,      lowerCamelCase=False ):
  if metric == "rouge2":
    lowercase     :str   						=    "{val_avg_rouge2:.4f}-{step_count}"
  elif metric == "bleu":
    lowercase     :int   						=    "{val_avg_bleu:.4f}-{step_count}"
  elif metric == "loss":
    lowercase     :List[str]   						=    "{val_avg_loss:.4f}-{step_count}"
  else:
    raise NotImplementedError(
        F"seq2seq callbacks only support rouge2, bleu and loss, got {metric}, You can make your own by adding to"
        " this function." )

  lowercase     :int   						=    ModelCheckpoint(
      dirpath=lowerCamelCase,      filename=lowerCamelCase,      monitor=F"val_{metric}",      mode="min" if "loss" in metric else "max",      save_top_k=lowerCamelCase,      )
  return checkpoint_callback







def    UpperCAmelCase__	(  lowerCamelCase,      lowerCamelCase ):
  return EarlyStopping(
      monitor=F"val_{metric}",      mode="min" if "loss" in metric else "max",      patience=lowerCamelCase,      verbose=lowerCamelCase,      )



