"""simple docstring"""







import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
     import torch

     from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
     import pyanvml.pyanvml as nvml


_SCREAMING_SNAKE_CASE	:		int   =   logging.get_logger(__name__)






class      _SCREAMING_SNAKE_CASE							(			UpperCAmelCase				):



   '''simple docstring'''



   __UpperCAmelCase	:							PyTorchBenchmarkArguments
   __UpperCAmelCase	:							PretrainedConfig
   __UpperCAmelCase	:							str    =    "PyTorch"







   @property
   def    _A				(	self	):


      '''simple docstring'''

      return torch.__version__







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					self._prepare_inference_func(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)
      return self._measure_speed(_inference	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					self._prepare_inference_func(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)
      return self._measure_memory(_inference	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					self._prepare_train_func(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)
      return self._measure_speed(_train	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					self._prepare_train_func(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)
      return self._measure_memory(_train	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					self.config_dict[model_name]

      if self.args.torchscript:
         __lowerCamelCase    		=					True

      __lowerCamelCase    		=					(
          hasattr(_SCREAMING_SNAKE_CASE ,				'architectures'	)
          and isinstance(config.architectures ,				_SCREAMING_SNAKE_CASE	)
          and len(config.architectures	) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
         try:
            __lowerCamelCase    		=					config.architectures[0]
            __lowerCamelCase    		=					__import__('transformers' ,				fromlist=[model_class]	)
            __lowerCamelCase    		=					getattr(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)
            __lowerCamelCase    		=					model_cls(_SCREAMING_SNAKE_CASE	)
         except ImportError:
            raise ImportError(
                F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'	)
      else:
         __lowerCamelCase    		=					MODEL_MAPPING[config.__class__](_SCREAMING_SNAKE_CASE	)

      model.eval()
      model.to(self.args.device	)

      # encoder-decoder has vocab size saved differently
      __lowerCamelCase    		=					config.vocab_size if hasattr(_SCREAMING_SNAKE_CASE ,				'vocab_size'	) else config.encoder.vocab_size
      __lowerCamelCase    		=					torch.randint(_SCREAMING_SNAKE_CASE ,				(batch_size, sequence_length) ,				dtype=torch.long ,				device=self.args.device	)

      if self.args.fpaa:
         logger.info('Running training in Mixed Precision...'	)
         if not self.args.is_gpu:
            raise ValueError('Mixed precision is possible only for GPU.'	)
         # amp seems to have memory leaks so that memory usage
         # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
         model.half()

      if self.args.torchscript:
         with torch.no_grad():
            __lowerCamelCase    		=					torch.jit.trace(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)
      else:
         __lowerCamelCase    		=					model

      def encoder_decoder_forward():
         with torch.no_grad():
            __lowerCamelCase    		=					inference_model(_SCREAMING_SNAKE_CASE ,				decoder_input_ids=_SCREAMING_SNAKE_CASE	)
         return outputs

      def encoder_forward():
         with torch.no_grad():
            __lowerCamelCase    		=					inference_model(_SCREAMING_SNAKE_CASE	)
         return outputs

      __lowerCamelCase    		=					encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
      return _forward







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					self.config_dict[model_name]

      __lowerCamelCase    		=					(
          hasattr(_SCREAMING_SNAKE_CASE ,				'architectures'	)
          and isinstance(config.architectures ,				_SCREAMING_SNAKE_CASE	)
          and len(config.architectures	) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
         try:
            __lowerCamelCase    		=					config.architectures[0]
            __lowerCamelCase    		=					__import__('transformers' ,				fromlist=[model_class]	)
            __lowerCamelCase    		=					getattr(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)
            __lowerCamelCase    		=					model_cls(_SCREAMING_SNAKE_CASE	)
         except ImportError:
            raise ImportError(
                F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'	)
      else:
         __lowerCamelCase    		=					MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_SCREAMING_SNAKE_CASE	)

      if self.args.torchscript:
         raise NotImplementedError('Training for torchscript is currently not implemented'	)
      else:
         __lowerCamelCase    		=					model

      model.train()
      model.to(self.args.device	)

      # encoder-decoder has vocab size saved differently
      __lowerCamelCase    		=					config.vocab_size if hasattr(_SCREAMING_SNAKE_CASE ,				'vocab_size'	) else config.encoder.vocab_size
      __lowerCamelCase    		=					torch.randint(_SCREAMING_SNAKE_CASE ,				(batch_size, sequence_length) ,				dtype=torch.long ,				device=self.args.device	)

      if self.args.fpaa:
         logger.info('Running training in Mixed Precision...'	)
         if not self.args.is_gpu:
            raise ValueError('Mixed precision is possible only for GPU.'	)

         # amp seems to have memory leaks so that memory usage
         # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
         model.half()

      def compute_loss_and_backprob_encoder():
         __lowerCamelCase    		=					train_model(_SCREAMING_SNAKE_CASE ,				labels=_SCREAMING_SNAKE_CASE	)[0]
         loss.backward()
         return loss

      def compute_loss_and_backprob_encoder_decoder():
         __lowerCamelCase    		=					train_model(_SCREAMING_SNAKE_CASE ,				decoder_input_ids=_SCREAMING_SNAKE_CASE ,				labels=_SCREAMING_SNAKE_CASE	)[0]
         loss.backward()
         return loss

      __lowerCamelCase    		=					(
          compute_loss_and_backprob_encoder_decoder
          if config.is_encoder_decoder
          else compute_loss_and_backprob_encoder
      )
      return _train







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      try:
         if self.args.is_tpu or self.args.torchscript:
            # run additional 10 times to stabilize compilation for tpu and torchscript
            logger.info('Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'	)
            timeit.repeat(
                _SCREAMING_SNAKE_CASE ,				repeat=1 ,				number=5 ,				)

         # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
         __lowerCamelCase    		=					timeit.repeat(
             _SCREAMING_SNAKE_CASE ,				repeat=self.args.repeat ,				number=10 ,				)

         if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
            import torch_xla.debug.metrics as met

            self.print_fn(met.metrics_report()	)

         return min(_SCREAMING_SNAKE_CASE	) / 10.0
      except RuntimeError as e:
         self.print_fn(F'''Doesn\'t fit on GPU. {e}'''	)
         return "N/A"







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      try:
         if self.args.trace_memory_line_by_line:
            __lowerCamelCase    		=					start_memory_tracing('transformers'	)

         if self.args.is_tpu:
            # tpu
            raise NotImplementedError(
                'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'
                ' `--no-memory` or `args.memory=False`'	)
         elif self.args.is_gpu:
            if not is_pyanvml_available():
               logger.warning(
                   'py3nvml not installed, we won\'t log GPU memory usage. '
                   'Install py3nvml (pip install py3nvml) to log information about GPU.'	)
               __lowerCamelCase    		=					'N/A'
            else:
               logger.info(
                   'Measuring total GPU usage on GPU device. Make sure to not have additional processes running'
                   ' on the same GPU.'	)
               # init nvml
               nvml.nvmlInit()
               func()
               __lowerCamelCase    		=					nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx	)
               __lowerCamelCase    		=					nvml.nvmlDeviceGetMemoryInfo(_SCREAMING_SNAKE_CASE	)
               __lowerCamelCase    		=					meminfo.used
               __lowerCamelCase    		=					Memory(_SCREAMING_SNAKE_CASE	)
               # shutdown nvml
               nvml.nvmlShutdown()
         else:
            # cpu
            __lowerCamelCase    		=					measure_peak_memory_cpu(_SCREAMING_SNAKE_CASE	)
            __lowerCamelCase    		=					Memory(_SCREAMING_SNAKE_CASE	) if isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	) else memory_bytes

         if self.args.trace_memory_line_by_line:
            __lowerCamelCase    		=					stop_memory_tracing(_SCREAMING_SNAKE_CASE	)
         else:
            __lowerCamelCase    		=					None

         return memory, summary
      except RuntimeError as e:
         self.print_fn(F'''Doesn\'t fit on GPU. {e}'''	)
         return "N/A", None







"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE	:		str   =   logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE	:		Any   =   {
    '''facebook/opt-125m''': '''https://huggingface.co/facebook/opt-125m/blob/main/config.json''',
    '''facebook/opt-350m''': '''https://huggingface.co/facebook/opt-350m/blob/main/config.json''',
    '''facebook/opt-1.3b''': '''https://huggingface.co/facebook/opt-1.3b/blob/main/config.json''',
    '''facebook/opt-2.7b''': '''https://huggingface.co/facebook/opt-2.7b/blob/main/config.json''',
    '''facebook/opt-6.7b''': '''https://huggingface.co/facebook/opt-6.7b/blob/main/config.json''',
    '''facebook/opt-13b''': '''https://huggingface.co/facebook/opt-13b/blob/main/config.json''',
}






class      _SCREAMING_SNAKE_CASE							(			UpperCAmelCase				):



   '''simple docstring'''



   __UpperCAmelCase	:							Tuple    =    "opt"
   __UpperCAmelCase	:							Any    =    ["past_key_values"]







   def __init__(	self ,				_SCREAMING_SNAKE_CASE=50272 ,				_SCREAMING_SNAKE_CASE=768 ,				_SCREAMING_SNAKE_CASE=12 ,				_SCREAMING_SNAKE_CASE=3072 ,				_SCREAMING_SNAKE_CASE=2048 ,				_SCREAMING_SNAKE_CASE=True ,				_SCREAMING_SNAKE_CASE=False ,				_SCREAMING_SNAKE_CASE=None ,				_SCREAMING_SNAKE_CASE=0.1 ,				_SCREAMING_SNAKE_CASE=0.0 ,				_SCREAMING_SNAKE_CASE=12 ,				_SCREAMING_SNAKE_CASE="relu" ,				_SCREAMING_SNAKE_CASE=0.0 ,				_SCREAMING_SNAKE_CASE=0.02 ,				_SCREAMING_SNAKE_CASE=True ,				_SCREAMING_SNAKE_CASE=1 ,				_SCREAMING_SNAKE_CASE=2 ,				_SCREAMING_SNAKE_CASE=2 ,				_SCREAMING_SNAKE_CASE=True ,				_SCREAMING_SNAKE_CASE=True ,				**_SCREAMING_SNAKE_CASE ,				):


      '''simple docstring'''

      super().__init__(
          pad_token_id=_SCREAMING_SNAKE_CASE ,				bos_token_id=_SCREAMING_SNAKE_CASE ,				eos_token_id=_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE ,				)
      __lowerCamelCase    		=					vocab_size
      __lowerCamelCase    		=					max_position_embeddings
      __lowerCamelCase    		=					num_attention_heads
      __lowerCamelCase    		=					word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
      __lowerCamelCase    		=					ffn_dim
      __lowerCamelCase    		=					hidden_size
      __lowerCamelCase    		=					num_hidden_layers
      __lowerCamelCase    		=					dropout
      __lowerCamelCase    		=					attention_dropout
      __lowerCamelCase    		=					activation_function
      __lowerCamelCase    		=					init_std
      __lowerCamelCase    		=					layerdrop
      __lowerCamelCase    		=					use_cache
      __lowerCamelCase    		=					do_layer_norm_before
      # We keep these variables at `True` for backward compatibility.
      __lowerCamelCase    		=					enable_bias
      __lowerCamelCase    		=					layer_norm_elementwise_affine

      # Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
      # with checkpoints that have been fine-tuned before transformers v4.20.1
      # see https://github.com/facebookresearch/metaseq/pull/164
      __lowerCamelCase    		=					_remove_final_layer_norm






