






"""simple docstring"""



import argparse
import time

import torch
import transformers
from measures_util import end_measure, log_measures, start_measure
from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeqaSeqLM, AutoTokenizer

from accelerate.utils import compute_module_sizes


A_     =				{
    '''gpt-j-6b''': {'''is_causal''': True, '''model''': '''sgugger/sharded-gpt-j-6B''', '''tokenizer''': '''EleutherAI/gpt-j-6B'''},
    '''gpt-neox''': {'''is_causal''': True, '''model''': '''EleutherAI/gpt-neox-20b'''},
    '''opt''': {'''is_causal''': True, '''model''': '''facebook/opt-30b'''},
    '''T0pp''': {'''is_causal''': False, '''model''': '''bigscience/T0pp''', '''model_revision''': '''sharded'''},
}

A_     =				[
    '''Hello, my name is''',
    '''Are unicorns real? Unicorns are''',
    '''For the first time in several years,''',
    '''My name is Julien and I am''',
    '''The goal of life is''',
    '''Whenever I\'m sad, I like to''',
]




def   _lowerCAmelCase		(				)		->str:
						A__    :	List[str]			=	argparse.ArgumentParser(description="""Run and time generations on a big model using Accelerate."""   )
						parser.add_argument("""model_name""",    type=_UpperCAmelCase,    default=_UpperCAmelCase,    help="""The name of the model to try."""   )
						parser.add_argument(
						    """--tokenizer_name""",    type=_UpperCAmelCase,    default=_UpperCAmelCase,    help="""The name of the tokenizer (if different from the model."""   )
						parser.add_argument("""--is_causal""",    type=_UpperCAmelCase,    default=_UpperCAmelCase,    help="""Whether or not the model is causal."""   )
						parser.add_argument(
						    """--model_revision""",    type=_UpperCAmelCase,    default=_UpperCAmelCase,    help="""The revision to use for the model checkpoint."""   )
						parser.add_argument("""--torch_dtype""",    type=_UpperCAmelCase,    default=_UpperCAmelCase,    help="""The dtype for the model."""   )
						parser.add_argument("""--disk_offload""",    action="""store_true"""   )

						A__    :	Any			=	parser.parse_args()

						# Sanitize args
						if args.model_name in DEFAULT_MODELS:
												A__    :	Optional[int]			=	DEFAULT_MODELS[args.model_name]
												A__    :	Dict			=	defaults['model']
												if args.tokenizer_name is None:
																		A__    :	Union[str, Any]			=	defaults.get("""tokenizer""",    args.model_name   )
												if args.is_causal is None:
																		A__    :	Union[str, Any]			=	defaults['is_causal']
												if args.model_revision is None:
																		A__    :	Optional[int]			=	defaults.get("""model_revision""",    """main"""   )

						if args.is_causal is None:
												raise ValueError("""Could not infer the default for `--is_causal`, pass either True or False for it."""   )
						if args.tokenizer_name is None:
												A__    :	Tuple			=	args.model_name
						if args.model_revision is None:
												A__    :	Tuple			=	'main'

						return args




def   _lowerCAmelCase		(				)		->List[Any]:
						transformers.utils.logging.set_verbosity_error()
						A__    :	Tuple			=	parse_args()

						if args.torch_dtype is None:
												A__    :	Optional[Any]			=	AutoConfig.from_pretrained(args.model_name   )
												A__    :	str			=	getattr(_UpperCAmelCase,    """torch_dtype""",    torch.floataa   )
						else:
												A__    :	Union[str, Any]			=	getattr(_UpperCAmelCase,    args.torch_dtype   )
						A__    :	List[Any]			=	AutoModelForCausalLM if args.is_causal else AutoModelForSeqaSeqLM
						A__    :	List[str]			=	{
						    'torch_dtype': torch_dtype,
						    'revision': args.model_revision,
						}
						if args.disk_offload:
												A__    :	List[Any]			=	'tmp_offload'
												A__    :	Tuple			=	True

						A__    :	int			=	start_measure()
						A__    :	List[str]			=	model_cls.from_pretrained(args.model_name,    device_map="""auto""",    **_UpperCAmelCase   )
						A__    :	List[str]			=	end_measure(_UpperCAmelCase   )
						log_measures(_UpperCAmelCase,    """Model loading"""   )

						A__    :	List[Any]			=	compute_module_sizes(_UpperCAmelCase   )
						A__    :	List[str]			=	{v: 0 for v in model.hf_device_map.values()}
						for module, device in model.hf_device_map.items():
												device_size[device] += module_sizes[module]
						A__    :	Dict			=	'\n'.join([f'- {device}: {size // 2**2_0}MiB' for device, size in device_size.items()]   )
						print(f'\nTheoretical use:\n{message}'   )

						A__    :	int			=	AutoTokenizer.from_pretrained(args.tokenizer_name   )

						A__    :	Tuple			=	start_measure()
						A__    :	Union[str, Any]			=	[]
						A__    :	str			=	[]
						A__    :	Optional[int]			=	[]
						for prompt in PROMPTS:
												A__    :	str			=	tokenizer(_UpperCAmelCase,    return_tensors="""pt"""   ).to(0   )
												A__    :	Any			=	inputs['input_ids'][0].tolist()
												A__    :	str			=	time.time()
												A__    :	int			=	model.generate(inputs["""input_ids"""]   )
												A__    :	int			=	time.time()
												A__    :	str			=	outputs[0].tolist()
												A__    :	Optional[Any]			=	len(_UpperCAmelCase   ) if outputs[: len(_UpperCAmelCase   )] != tokens else len(_UpperCAmelCase   ) - len(_UpperCAmelCase   )
												A__    :	str			=	after_generate - before_generate

												A__    :	Optional[Any]			=	tokenizer.decode(_UpperCAmelCase,    skip_special_tokens=_UpperCAmelCase   )
												texts_outs.append(_UpperCAmelCase   )
												generation_times.append(_UpperCAmelCase   )
												gen_tokens.append(_UpperCAmelCase   )
												print(f'Prompt: {prompt}\nGeneration {text_out}\nIn {generation_time:.2f}s for {num_gen_tokens} tokens\n'   )

						A__    :	List[str]			=	end_measure(_UpperCAmelCase   )
						log_measures(_UpperCAmelCase,    """Model generation"""   )

						A__    :	Optional[int]			=	[gen / tok for gen, tok in zip(_UpperCAmelCase,    _UpperCAmelCase   )]
						A__    :	Dict			=	sum(_UpperCAmelCase   ) / len(_UpperCAmelCase   )
						print(f'Average time of generation per token: {avg_gen:.2f}s'   )
						print(f'First generation (avg time per token): {generation_times_per_token[0]:.2f}s'   )
						A__    :	Union[str, Any]			=	sum(generation_times_per_token[1:]   ) / (len(_UpperCAmelCase   ) - 1)
						print(f'Average time of generation per token (excluding the first): {avg_gen:.2f}s'   )


if __name__ == "__main__":
						main()













'''simple docstring'''


import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
    import torch

    from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
    import pyanvml.pyanvml as nvml


A_				     =							logging.get_logger(__name__)


class 		UpperCamelCase__     (       a							):
      '''simple docstring'''






      _snake_case           =				42
      _snake_case           =				42
      _snake_case           =				"PyTorch"




      @property
      def 	snake_case  (     self )				->  Dict:
        return torch.__version__




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )				->  float:
        __lowerCAmelCase		:    Any			     =  self._prepare_inference_func(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )
        return self._measure_speed(_inference )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )				->  [Memory, Optional[MemorySummary]]:
        __lowerCAmelCase		:    Dict			     =  self._prepare_inference_func(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )
        return self._measure_memory(_inference )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )				->  float:
        __lowerCAmelCase		:    Optional[Any]			     =  self._prepare_train_func(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )
        return self._measure_speed(_train )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )				->  [Memory, Optional[MemorySummary]]:
        __lowerCAmelCase		:    Optional[int]			     =  self._prepare_train_func(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )
        return self._measure_memory(_train )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )				->  Callable[[], None]:
        __lowerCAmelCase		:    str			     =  self.config_dict[model_name]

        if self.args.torchscript:
          __lowerCAmelCase		:    Tuple			     =  True

        __lowerCAmelCase		:    Union[str, Any]			     =  (
            hasattr(SCREAMING_SNAKE_CASE     , 'architectures' )
            and isinstance(config.architectures     , SCREAMING_SNAKE_CASE )
            and len(config.architectures ) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
          try:
            __lowerCAmelCase		:    Dict			     =  config.architectures[0]
            __lowerCAmelCase		:    List[str]			     =  __import__('transformers'     , fromlist=[model_class] )
            __lowerCAmelCase		:    Tuple			     =  getattr(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )
            __lowerCAmelCase		:    Dict			     =  model_cls(SCREAMING_SNAKE_CASE )
          except ImportError:
            raise ImportError(
                F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
                ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.' )
        else:
          __lowerCAmelCase		:    List[Any]			     =  MODEL_MAPPING[config.__class__](SCREAMING_SNAKE_CASE )

        model.eval()
        model.to(self.args.device )

        # encoder-decoder has vocab size saved differently
        __lowerCAmelCase		:    List[str]			     =  config.vocab_size if hasattr(SCREAMING_SNAKE_CASE     , 'vocab_size' ) else config.encoder.vocab_size
        __lowerCAmelCase		:    Optional[Any]			     =  torch.randint(SCREAMING_SNAKE_CASE     , (batch_size, sequence_length)     , dtype=torch.long     , device=self.args.device )

        if self.args.fpaa:
          logger.info('Running training in Mixed Precision...' )
          if not self.args.is_gpu:
            raise ValueError('Mixed precision is possible only for GPU.' )
          # amp seems to have memory leaks so that memory usage
          # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
          model.half()

        if self.args.torchscript:
          with torch.no_grad():
            __lowerCAmelCase		:    List[Any]			     =  torch.jit.trace(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )
        else:
          __lowerCAmelCase		:    str			     =  model

        def encoder_decoder_forward():
          with torch.no_grad():
            __lowerCAmelCase		:    str			     =  inference_model(SCREAMING_SNAKE_CASE     , decoder_input_ids=SCREAMING_SNAKE_CASE )
          return outputs

        def encoder_forward():
          with torch.no_grad():
            __lowerCAmelCase		:    str			     =  inference_model(SCREAMING_SNAKE_CASE )
          return outputs

        __lowerCAmelCase		:    Optional[Any]			     =  encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
        return _forward




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )				->  Callable[[], None]:
        __lowerCAmelCase		:    int			     =  self.config_dict[model_name]

        __lowerCAmelCase		:    Optional[int]			     =  (
            hasattr(SCREAMING_SNAKE_CASE     , 'architectures' )
            and isinstance(config.architectures     , SCREAMING_SNAKE_CASE )
            and len(config.architectures ) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
          try:
            __lowerCAmelCase		:    List[Any]			     =  config.architectures[0]
            __lowerCAmelCase		:    List[str]			     =  __import__('transformers'     , fromlist=[model_class] )
            __lowerCAmelCase		:    Tuple			     =  getattr(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )
            __lowerCAmelCase		:    Any			     =  model_cls(SCREAMING_SNAKE_CASE )
          except ImportError:
            raise ImportError(
                F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
                ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.' )
        else:
          __lowerCAmelCase		:    List[str]			     =  MODEL_WITH_LM_HEAD_MAPPING[config.__class__](SCREAMING_SNAKE_CASE )

        if self.args.torchscript:
          raise NotImplementedError('Training for torchscript is currently not implemented' )
        else:
          __lowerCAmelCase		:    int			     =  model

        model.train()
        model.to(self.args.device )

        # encoder-decoder has vocab size saved differently
        __lowerCAmelCase		:    Tuple			     =  config.vocab_size if hasattr(SCREAMING_SNAKE_CASE     , 'vocab_size' ) else config.encoder.vocab_size
        __lowerCAmelCase		:    Optional[int]			     =  torch.randint(SCREAMING_SNAKE_CASE     , (batch_size, sequence_length)     , dtype=torch.long     , device=self.args.device )

        if self.args.fpaa:
          logger.info('Running training in Mixed Precision...' )
          if not self.args.is_gpu:
            raise ValueError('Mixed precision is possible only for GPU.' )

          # amp seems to have memory leaks so that memory usage
          # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
          model.half()

        def compute_loss_and_backprob_encoder():
          __lowerCAmelCase		:    List[str]			     =  train_model(SCREAMING_SNAKE_CASE     , labels=SCREAMING_SNAKE_CASE )[0]
          loss.backward()
          return loss

        def compute_loss_and_backprob_encoder_decoder():
          __lowerCAmelCase		:    List[Any]			     =  train_model(SCREAMING_SNAKE_CASE     , decoder_input_ids=SCREAMING_SNAKE_CASE     , labels=SCREAMING_SNAKE_CASE )[0]
          loss.backward()
          return loss

        __lowerCAmelCase		:    Dict			     =  (
            compute_loss_and_backprob_encoder_decoder
            if config.is_encoder_decoder
            else compute_loss_and_backprob_encoder
        )
        return _train




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  float:
        try:
          if self.args.is_tpu or self.args.torchscript:
            # run additional 10 times to stabilize compilation for tpu and torchscript
            logger.info('Do inference on TPU or torchscript. Running model 5 times to stabilize compilation' )
            timeit.repeat(
                SCREAMING_SNAKE_CASE     , repeat=1     , number=5     , )

          # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
          __lowerCAmelCase		:    Tuple			     =  timeit.repeat(
              SCREAMING_SNAKE_CASE     , repeat=self.args.repeat     , number=10     , )

          if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
            import torch_xla.debug.metrics as met

            self.print_fn(met.metrics_report() )

          return min(SCREAMING_SNAKE_CASE ) / 1_0.0
        except RuntimeError as e:
          self.print_fn(F"""Doesn't fit on GPU. {e}""" )
          return "N/A"







      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  [Memory, MemorySummary]:
        try:
          if self.args.trace_memory_line_by_line:
            __lowerCAmelCase		:    Tuple			     =  start_memory_tracing('transformers' )

          if self.args.is_tpu:
            # tpu
            raise NotImplementedError(
                'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'
                ' `--no-memory` or `args.memory=False`' )
          elif self.args.is_gpu:
            if not is_pyanvml_available():
              logger.warning(
                  'py3nvml not installed, we won\'t log GPU memory usage. '
                  'Install py3nvml (pip install py3nvml) to log information about GPU.' )
              __lowerCAmelCase		:    Union[str, Any]			     =  'N/A'
            else:
              logger.info(
                  'Measuring total GPU usage on GPU device. Make sure to not have additional processes running'
                  ' on the same GPU.' )
              # init nvml
              nvml.nvmlInit()
              func()
              __lowerCAmelCase		:    int			     =  nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx )
              __lowerCAmelCase		:    List[str]			     =  nvml.nvmlDeviceGetMemoryInfo(SCREAMING_SNAKE_CASE )
              __lowerCAmelCase		:    Tuple			     =  meminfo.used
              __lowerCAmelCase		:    Dict			     =  Memory(SCREAMING_SNAKE_CASE )
              # shutdown nvml
              nvml.nvmlShutdown()
          else:
            # cpu
            __lowerCAmelCase		:    Tuple			     =  measure_peak_memory_cpu(SCREAMING_SNAKE_CASE )
            __lowerCAmelCase		:    str			     =  Memory(SCREAMING_SNAKE_CASE ) if isinstance(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE ) else memory_bytes

          if self.args.trace_memory_line_by_line:
            __lowerCAmelCase		:    int			     =  stop_memory_tracing(SCREAMING_SNAKE_CASE )
          else:
            __lowerCAmelCase		:    Any			     =  None

          return memory, summary
        except RuntimeError as e:
          self.print_fn(F"""Doesn't fit on GPU. {e}""" )
          return "N/A", None





