

import collections

from .utils import ExplicitEnum, is_torch_available, logging


if is_torch_available():
  import torch


UpperCAmelCase__			:  List[Any]					     =     logging.get_logger(__name__)







class 							a__							:







  """simple docstring"""




  def __init__(      self     :			List[str]					,					UpperCAmelCase__     :			str					,					UpperCAmelCase__     :			Optional[Any]=2_1					,					UpperCAmelCase__     :			Dict=[]					,					UpperCAmelCase__     :			Union[str, Any]=None		)       ->Optional[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Dict	  = model
        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = trace_batch_nums
        SCREAMING_SNAKE_CASE			:   Tuple	  = abort_after_batch_num

        # keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence
        SCREAMING_SNAKE_CASE			:   Any	  = collections.deque([]					,					UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE			:   str	  = []
        SCREAMING_SNAKE_CASE			:   Any	  = 0
        SCREAMING_SNAKE_CASE			:   Tuple	  = 0
        SCREAMING_SNAKE_CASE			:   Any	  = False
        SCREAMING_SNAKE_CASE			:   Dict	  = """                 """

        self.analyse_model()

        self.register_forward_hook()


  def 			_lowercase   (      self     :			int					,					UpperCAmelCase__     :			Tuple=None		)       ->Optional[int]:
        """simple docstring"""







        if frame is not None:
              self.expand_frame(UpperCAmelCase__		)
        self.frames.append("""\n""".join(self.frame		)		)
        SCREAMING_SNAKE_CASE			:   Dict	  = []  # start a new frame


  def 			_lowercase   (      self     :			Tuple					,					UpperCAmelCase__     :			List[str]		)       ->Optional[Any]:
        """simple docstring"""







        self.frame.append(UpperCAmelCase__		)


  def 			_lowercase   (      self     :			Dict		)       ->List[Any]:
        """simple docstring"""







        print("""\n""".join(self.frames		)		)
        SCREAMING_SNAKE_CASE			:   Optional[int]	  = []


  def 			_lowercase   (      self     :			Tuple		)       ->str:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Optional[int]	  = []


  def 			_lowercase   (      self     :			Any		)       ->List[Any]:
        """simple docstring"""







        print(f"\nDetected inf/nan during batch_number={self.batch_number}"		)
        print(f"Last {len(self.frames		)} forward frames:"		)
        print(f"{'abs min':8} {'abs max':8} metadata"		)
        print("""\n""".join(self.frames		)		)
        print("""\n\n"""		)
        SCREAMING_SNAKE_CASE			:   str	  = []


  def 			_lowercase   (      self     :			int		)       ->Tuple:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Any	  = {m: name for name, m in self.model.named_modules()}
        # self.longest_module_name = max(len(v) for v in self.module_names.values())


  def 			_lowercase   (      self     :			Tuple					,					UpperCAmelCase__     :			Dict					,					UpperCAmelCase__     :			str		)       ->Any:
        """simple docstring"""







        if torch.is_tensor(UpperCAmelCase__		):
              self.expand_frame(get_abs_min_max(UpperCAmelCase__					,					UpperCAmelCase__		)		)
              if detect_overflow(UpperCAmelCase__					,					UpperCAmelCase__		):
                    SCREAMING_SNAKE_CASE			:   List[str]	  = True
        elif var is None:
              self.expand_frame(f"{'None':>17} {ctx}"		)
        else:
              self.expand_frame(f"{'not a tensor':>17} {ctx}"		)


  def 			_lowercase   (      self     :			List[Any]		)       ->Union[str, Any]:
        """simple docstring"""







        self.expand_frame(f"\n\n{self.prefix} *** Starting batch number={self.batch_number} ***"		)
        self.expand_frame(f"{'abs min':8} {'abs max':8} metadata"		)


  def 			_lowercase   (      self     :			int		)       ->Optional[int]:
        """simple docstring"""







        self.expand_frame(f"{self.prefix} *** Finished batch number={self.batch_number-1} ***\n\n"		)


  def 			_lowercase   (      self     :			Dict					,					UpperCAmelCase__     :			Tuple					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			Optional[Any]		)       ->str:
        """simple docstring"""







        self.expand_frame(f"{self.prefix} {self.module_names[module]} {module.__class__.__name__}"		)

        # params
        for name, p in module.named_parameters(recurse=UpperCAmelCase__		):
              self.analyse_variable(UpperCAmelCase__					,					UpperCAmelCase__		)

        # inputs
        if isinstance(UpperCAmelCase__					,					UpperCAmelCase__		):
              for i, x in enumerate(UpperCAmelCase__		):
                    self.analyse_variable(UpperCAmelCase__					,					f"input[{i}]"		)
        else:
              self.analyse_variable(UpperCAmelCase__					,					"""input"""		)

        # outputs
        if isinstance(UpperCAmelCase__					,					UpperCAmelCase__		):
              for i, x in enumerate(UpperCAmelCase__		):
                    # possibly a tuple of tuples
                    if isinstance(UpperCAmelCase__					,					UpperCAmelCase__		):
                          for j, y in enumerate(UpperCAmelCase__		):
                                self.analyse_variable(UpperCAmelCase__					,					f"output[{i}][{j}]"		)
                    else:
                          self.analyse_variable(UpperCAmelCase__					,					f"output[{i}]"		)
        else:
              self.analyse_variable(UpperCAmelCase__					,					"""output"""		)

        self.save_frame()


  def 			_lowercase   (      self     :			List[str]		)       ->List[str]:
        """simple docstring"""







        self.model.apply(self._register_forward_hook		)


  def 			_lowercase   (      self     :			Optional[Any]					,					UpperCAmelCase__     :			int		)       ->Any:
        """simple docstring"""







        module.register_forward_hook(self.forward_hook		)




  def 			_lowercase   (      self     :			List[Any]					,					UpperCAmelCase__     :			Tuple					,					UpperCAmelCase__     :			List[Any]					,					UpperCAmelCase__     :			List[str]		)       ->Any:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = False

        SCREAMING_SNAKE_CASE			:   Optional[int]	  = True if self.batch_number in self.trace_batch_nums else False
        if trace_mode:
              self.reset_saved_frames()

        if self.total_calls == 0:
              self.batch_start_frame()
        self.total_calls += 1

        # count batch numbers - the very first forward hook of the batch will be called when the
        # batch completes - i.e. it gets called very last - we know this batch has finished
        if module == self.model:
              self.batch_number += 1
              SCREAMING_SNAKE_CASE			:   List[Any]	  = True

        self.create_frame(UpperCAmelCase__					,					UpperCAmelCase__					,					UpperCAmelCase__		)

        # if last_frame_of_batch:
        #     self.batch_end_frame()

        if trace_mode:
              self.trace_frames()

        if last_frame_of_batch:
              self.batch_start_frame()

        if self.detected_overflow and not trace_mode:
              self.dump_saved_frames()

              # now we can abort, as it's pointless to continue running
              raise ValueError(
                  """DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. """
                  """Please scroll up above this traceback to see the activation values prior to this event."""		)

        # abort after certain batch if requested to do so
        if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num:
              raise ValueError(
                  f"DebugUnderflowOverflow: aborting after {self.batch_number} batches due to"
                  f" `abort_after_batch_num={self.abort_after_batch_num}` arg"		)






def        __lowercase   (  _A    ,     _A      )  ->    int:
      SCREAMING_SNAKE_CASE			:   int	  = var.abs()
      return F"{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}"






def        __lowercase   (  _A    ,     _A      )  ->    Dict:
      SCREAMING_SNAKE_CASE			:   List[str]	  = False
      if torch.isnan(_A      ).any().item():
            SCREAMING_SNAKE_CASE			:   List[Any]	  = True
            print(F"{ctx} has nans"      )
      if torch.isinf(_A      ).any().item():
            SCREAMING_SNAKE_CASE			:   int	  = True
            print(F"{ctx} has infs"      )

      # if needed to monitor large elements can enable the following
      if 0:  # and detected:
            SCREAMING_SNAKE_CASE			:   List[Any]	  = var[torch.ge(var.abs()    ,     100      )]
            if naaa.numel() > 0:
                  print(F"{ctx}:  n100={naaa.numel()}"      )
            SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = var[torch.ge(var.abs()    ,     1000      )]
            if naaaa.numel() > 0:
                  print(F"{ctx}: n1000={naaaa.numel()}"      )
            SCREAMING_SNAKE_CASE			:   Optional[Any]	  = var[torch.ge(var.abs()    ,     10000      )]
            if naaaaa.numel() > 0:
                  print(F"{ctx}: n10000={naaaaa.numel()}"      )

      if 0:
            print(F"min={var.min():9.2e} max={var.max():9.2e}"      )

      if 0:
            print(F"min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})"      )

      return detected







class 							a__							( UpperCAmelCase							):







  """simple docstring"""




  UpperCAmelCase__   :			str      							="""underflow_overflow"""
  UpperCAmelCase__   :			Optional[Any]      							="""tpu_metrics_debug"""







import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only

from utils import save_json






def        __lowercase   (  _A      )  ->    Dict:
      SCREAMING_SNAKE_CASE			:   Dict	  = filter(lambda _A      : p.requires_grad    ,     model.parameters()      )
      SCREAMING_SNAKE_CASE			:   Optional[int]	  = sum([np.prod(p.size()      ) for p in model_parameters]      )
      return params


UpperCAmelCase__			:  List[Any]					     =     logging.getLogger(__name__)







class 							a__							( pl.Callback							):







  """simple docstring"""




  def 			_lowercase   (      self     :			List[Any]					,					UpperCAmelCase__     :			str					,					UpperCAmelCase__     :			Any		)       ->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   List[str]	  = {f"lr_group_{i}": param["""lr"""] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups		)}
        pl_module.logger.log_metrics(UpperCAmelCase__		)


  @rank_zero_only
  def 			_lowercase   (      self     :			List[Any]					,					UpperCAmelCase__     :			pl.Trainer					,					UpperCAmelCase__     :			pl.LightningModule					,					UpperCAmelCase__     :			str					,					UpperCAmelCase__     :			Tuple=True		)       ->None:
        """simple docstring"""







        logger.info(f"***** {type_path} results at step {trainer.global_step:05d} *****"		)
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = trainer.callback_metrics
        trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ["""log""", """progress_bar""", """preds"""]}		)
        # Log results
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = Path(pl_module.hparams.output_dir		)
        if type_path == "test":
              SCREAMING_SNAKE_CASE			:   Optional[int]	  = od / """test_results.txt"""
              SCREAMING_SNAKE_CASE			:   int	  = od / """test_generations.txt"""
        else:
              # this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
              # If people want this it will be easy enough to add back.
              SCREAMING_SNAKE_CASE			:   Any	  = od / f"{type_path}_results/{trainer.global_step:05d}.txt"
              SCREAMING_SNAKE_CASE			:   int	  = od / f"{type_path}_generations/{trainer.global_step:05d}.txt"
              results_file.parent.mkdir(exist_ok=UpperCAmelCase__		)
              generations_file.parent.mkdir(exist_ok=UpperCAmelCase__		)
        with open(UpperCAmelCase__					,					"""a+"""		) as writer:
              for key in sorted(UpperCAmelCase__		):
                    if key in ["log", "progress_bar", "preds"]:
                          continue
                    SCREAMING_SNAKE_CASE			:   Optional[int]	  = metrics[key]
                    if isinstance(UpperCAmelCase__					,					torch.Tensor		):
                          SCREAMING_SNAKE_CASE			:   Optional[int]	  = val.item()
                    SCREAMING_SNAKE_CASE			:   List[str]	  = f"{key}: {val:.6f}\n"
                    writer.write(UpperCAmelCase__		)

        if not save_generations:
              return

        if "preds" in metrics:
              SCREAMING_SNAKE_CASE			:   Optional[int]	  = """\n""".join(metrics["""preds"""]		)
              generations_file.open("""w+"""		).write(UpperCAmelCase__		)


  @rank_zero_only
  def 			_lowercase   (      self     :			Optional[int]					,					UpperCAmelCase__     :			Optional[int]					,					UpperCAmelCase__     :			Union[str, Any]		)       ->int:
        """simple docstring"""







        try:
              SCREAMING_SNAKE_CASE			:   int	  = pl_module.model.model.num_parameters()
        except AttributeError:
              SCREAMING_SNAKE_CASE			:   Optional[Any]	  = pl_module.model.num_parameters()

        SCREAMING_SNAKE_CASE			:   Dict	  = count_trainable_parameters(UpperCAmelCase__		)
        # mp stands for million parameters
        trainer.logger.log_metrics({"""n_params""": npars, """mp""": npars / 1e6, """grad_mp""": n_trainable_pars / 1e6}		)


  @rank_zero_only
  def 			_lowercase   (      self     :			str					,					UpperCAmelCase__     :			pl.Trainer					,					UpperCAmelCase__     :			pl.LightningModule		)       ->Optional[int]:
        """simple docstring"""







        save_json(pl_module.metrics					,					pl_module.metrics_save_path		)
        return self._write_logs(UpperCAmelCase__					,					UpperCAmelCase__					,					"""test"""		)




  @rank_zero_only
  def 			_lowercase   (      self     :			List[Any]					,					UpperCAmelCase__     :			pl.Trainer					,					UpperCAmelCase__     :			List[Any]		)       ->Dict:
        """simple docstring"""







        save_json(pl_module.metrics					,					pl_module.metrics_save_path		)
        # Uncommenting this will save val generations
        # return self._write_logs(trainer, pl_module, "valid")






def        __lowercase   (  _A    ,     _A    ,     _A=1    ,     _A=False      )  ->    Tuple:
      if metric == "rouge2":
            SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = """{val_avg_rouge2:.4f}-{step_count}"""
      elif metric == "bleu":
            SCREAMING_SNAKE_CASE			:   Tuple	  = """{val_avg_bleu:.4f}-{step_count}"""
      elif metric == "loss":
            SCREAMING_SNAKE_CASE			:   Any	  = """{val_avg_loss:.4f}-{step_count}"""
      else:
            raise NotImplementedError(
                F"seq2seq callbacks only support rouge2, bleu and loss, got {metric}, You can make your own by adding to"
                """ this function."""      )

      SCREAMING_SNAKE_CASE			:   Tuple	  = ModelCheckpoint(
          dirpath=_A    ,     filename=_A    ,     monitor=F"val_{metric}"    ,     mode="""min""" if """loss""" in metric else """max"""    ,     save_top_k=_A    ,     )
      return checkpoint_callback







def        __lowercase   (  _A    ,     _A      )  ->    Optional[int]:
      return EarlyStopping(
          monitor=F"val_{metric}"    ,     mode="""min""" if """loss""" in metric else """max"""    ,     patience=_A    ,     verbose=_A    ,     )




