






"""simple docstring"""




import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available






def 							lowerCAmelCase			(   UpperCamelCase_:				Dict  ,			UpperCamelCase_:				List[Any]  ,			UpperCamelCase_:				Union[str, Any]  ,			UpperCamelCase_:				str=None       )							->				Any:







  '''simple docstring'''







  _a					      =						None
  # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
  if str(weight.dtype       ) == "torch.bfloat16":
    # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
    _a					      =						weight.view(torch.intaa       )
    _a					      =						"bfloat16"
  _a					      =						weight.cpu().numpy()
  _a					      =						os.path.join(UpperCamelCase_  ,			f'''{weight_name}.dat'''       )
  if index is not None:
    if dtype is None:
      _a					      =						str(array.dtype       )
    _a					      =						{"dtype": dtype, "shape": list(array.shape       )}
  if array.ndim == 0:
    _a					      =						array[None]
  _a					      =						np.memmap(UpperCamelCase_  ,			dtype=array.dtype  ,			mode="w+"  ,			shape=array.shape       )
  _a					      =						array[:]
  file_array.flush()
  return index






def 							lowerCAmelCase			(   UpperCamelCase_:				Optional[int]  ,			UpperCamelCase_:				Tuple       )							->				List[Any]:







  '''simple docstring'''







  _a					      =						tuple(weight_info["shape"]       )
  if shape == ():
    # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
    _a					      =						(1,)

  _a					      =						weight_info["dtype"]
  if dtype == "bfloat16":
    # NumPy does not support bfloat16 so this was saved as a int16
    _a					      =						"int16"

  _a					      =						np.memmap(UpperCamelCase_  ,			dtype=UpperCamelCase_  ,			shape=UpperCamelCase_  ,			mode="r"       )

  if len(weight_info["shape"]       ) == 0:
    _a					      =						weight[0]
  _a					      =						torch.tensor(UpperCamelCase_       )
  if weight_info["dtype"] == "bfloat16":
    _a					      =						weight.view(torch.bfloataa       )

  return weight






def 							lowerCAmelCase			(   UpperCamelCase_:				Dict  ,			UpperCamelCase_:				List[Any]       )							->				str:







  '''simple docstring'''







  if index is None or len(UpperCamelCase_       ) == 0:
    # Nothing to save
    return

  _a					      =						os.path.join(UpperCamelCase_  ,			"index.json"       )
  if os.path.isfile(UpperCamelCase_       ):
    with open(UpperCamelCase_  ,			"r"  ,			encoding="utf-8"       ) as f:
      _a					      =						json.load(UpperCamelCase_       )
  else:
    _a					      =						{}
  current_index.update(UpperCamelCase_       )

  with open(UpperCamelCase_  ,			"w"  ,			encoding="utf-8"       ) as f:
    json.dump(UpperCamelCase_  ,			UpperCamelCase_  ,			indent=2       )






def 							lowerCAmelCase			(   UpperCamelCase_:				Union[str, os.PathLike]  ,			UpperCamelCase_:				Dict[str, torch.Tensor]       )							->				Optional[Any]:







  '''simple docstring'''







  os.makedirs(UpperCamelCase_  ,			exist_ok=UpperCamelCase_       )
  _a					      =						{}
  for name, parameter in state_dict.items():
    _a					      =						offload_weight(UpperCamelCase_  ,			UpperCamelCase_  ,			UpperCamelCase_  ,			index=UpperCamelCase_       )

  # Update index
  save_offload_index(UpperCamelCase_  ,			UpperCamelCase_       )




class  lowercase_     (_UpperCAmelCase  ):






      def __init__(     self  ,  a_  ,  a_						)						->Dict:







        '''simple docstring'''


        _a					      =						dataset
        _a					      =						prefix






      def __getitem__(     self  ,  a_						)						->Any:







        '''simple docstring'''


        return self.dataset[f'''{self.prefix}{key}''']






      def __iter__(     self						)						->List[Any]:







        '''simple docstring'''


        return iter([key for key in self.dataset if key.startswith(self.prefix						)]						)






      def __len__(     self						)						->Tuple:







        '''simple docstring'''


        return len(self.dataset						)







class  lowercase_     (_UpperCAmelCase  ):






      def __init__(     self  ,  a_ = None  ,  a_ = None  ,  a_ = None  ,  a_=None  ,  )						->List[str]:







        '''simple docstring'''


        if state_dict is None and save_folder is None:
          raise ValueError("Need either a `state_dict` or a `save_folder` containing offloaded weights."						)

        _a					      =						{} if state_dict is None else state_dict
        _a					      =						save_folder
        if index is None and save_folder is not None:
          with open(os.path.join(a_  ,  "index.json"						)						) as f:
            _a					      =						json.load(a_						)
        _a					      =						{} if index is None else index
        _a					      =						list(self.state_dict.keys()						)
        self.all_keys.extend([key for key in self.index if key not in self.all_keys]						)
        _a					      =						device






      def __getitem__(     self  ,  a_						)						->Union[str, Any]:







        '''simple docstring'''


        if key in self.state_dict:
          return self.state_dict[key]
        _a					      =						self.index[key]
        if weight_info.get("safetensors_file"						) is not None:
          if not is_safetensors_available():
            raise ImportError("These offloaded weights require the use of safetensors: `pip install safetensors`."						)

          from safetensors import safe_open

          _a					      =						"cpu" if self.device is None else self.device
          with safe_open(weight_info["safetensors_file"]  ,  framework="pt"  ,  device=a_						) as f:
            _a					      =						f.get_tensor(weight_info.get("weight_name"  ,  a_						)						)

          if "dtype" in weight_info:
            return tensor.to(getattr(a_  ,  weight_info["dtype"]						)						)
          else:
            return tensor

        _a					      =						os.path.join(self.save_folder  ,  f'''{key}.dat'''						)
        return load_offloaded_weight(a_  ,  a_						)






      def __iter__(     self						)						->int:







        '''simple docstring'''


        return iter(self.all_keys						)






      def __len__(     self						)						->Any:







        '''simple docstring'''


        return len(self.all_keys						)







def 							lowerCAmelCase			(   UpperCamelCase_:				Dict[str, torch.Tensor]  ,			UpperCamelCase_:				List[str]       )							->				Any:







  '''simple docstring'''







  _a					      =						{}
  for module_name in submodule_names:
    # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
    # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
    result.update(
        {
            key: param
            for key, param in state_dict.items()
            if key == module_name or key.startswith(module_name + "."       )
        }       )
  return result










"""simple docstring"""




import argparse
import time

import torch
import transformers
from measures_util import end_measure, log_measures, start_measure
from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeqaSeqLM, AutoTokenizer

from accelerate.utils import compute_module_sizes


UpperCamelCase											=     {
    """gpt-j-6b""": {"""is_causal""": True, """model""": """sgugger/sharded-gpt-j-6B""", """tokenizer""": """EleutherAI/gpt-j-6B"""},
    """gpt-neox""": {"""is_causal""": True, """model""": """EleutherAI/gpt-neox-20b"""},
    """opt""": {"""is_causal""": True, """model""": """facebook/opt-30b"""},
    """T0pp""": {"""is_causal""": False, """model""": """bigscience/T0pp""", """model_revision""": """sharded"""},
}

UpperCamelCase											=     [
    """Hello, my name is""",
    """Are unicorns real? Unicorns are""",
    """For the first time in several years,""",
    """My name is Julien and I am""",
    """The goal of life is""",
    """Whenever I'm sad, I like to""",
]






def 							lowerCAmelCase			(   )							->				Optional[int]:







  '''simple docstring'''







  _a					      =						argparse.ArgumentParser(description="Run and time generations on a big model using Accelerate."       )
  parser.add_argument("model_name"  ,			type=UpperCamelCase_  ,			default=UpperCamelCase_  ,			help="The name of the model to try."       )
  parser.add_argument(
      "--tokenizer_name"  ,			type=UpperCamelCase_  ,			default=UpperCamelCase_  ,			help="The name of the tokenizer (if different from the model."       )
  parser.add_argument("--is_causal"  ,			type=UpperCamelCase_  ,			default=UpperCamelCase_  ,			help="Whether or not the model is causal."       )
  parser.add_argument(
      "--model_revision"  ,			type=UpperCamelCase_  ,			default=UpperCamelCase_  ,			help="The revision to use for the model checkpoint."       )
  parser.add_argument("--torch_dtype"  ,			type=UpperCamelCase_  ,			default=UpperCamelCase_  ,			help="The dtype for the model."       )
  parser.add_argument("--disk_offload"  ,			action="store_true"       )

  _a					      =						parser.parse_args()

  # Sanitize args
  if args.model_name in DEFAULT_MODELS:
    _a					      =						DEFAULT_MODELS[args.model_name]
    _a					      =						defaults["model"]
    if args.tokenizer_name is None:
      _a					      =						defaults.get("tokenizer"  ,			args.model_name       )
    if args.is_causal is None:
      _a					      =						defaults["is_causal"]
    if args.model_revision is None:
      _a					      =						defaults.get("model_revision"  ,			"main"       )

  if args.is_causal is None:
    raise ValueError("Could not infer the default for `--is_causal`, pass either True or False for it."       )
  if args.tokenizer_name is None:
    _a					      =						args.model_name
  if args.model_revision is None:
    _a					      =						"main"

  return args






def 							lowerCAmelCase			(   )							->				str:







  '''simple docstring'''







  transformers.utils.logging.set_verbosity_error()
  _a					      =						parse_args()

  if args.torch_dtype is None:
    _a					      =						AutoConfig.from_pretrained(args.model_name       )
    _a					      =						getattr(UpperCamelCase_  ,			"torch_dtype"  ,			torch.floataa       )
  else:
    _a					      =						getattr(UpperCamelCase_  ,			args.torch_dtype       )
  _a					      =						AutoModelForCausalLM if args.is_causal else AutoModelForSeqaSeqLM
  _a					      =						{
      "torch_dtype": torch_dtype,
      "revision": args.model_revision,
  }
  if args.disk_offload:
    _a					      =						"tmp_offload"
    _a					      =						True

  _a					      =						start_measure()
  _a					      =						model_cls.from_pretrained(args.model_name  ,			device_map="auto"  ,			**UpperCamelCase_       )
  _a					      =						end_measure(UpperCamelCase_       )
  log_measures(UpperCamelCase_  ,			"Model loading"       )

  _a					      =						compute_module_sizes(UpperCamelCase_       )
  _a					      =						{v: 0 for v in model.hf_device_map.values()}
  for module, device in model.hf_device_map.items():
    device_size[device] += module_sizes[module]
  _a					      =						"\n".join([f'''- {device}: {size // 2**20}MiB''' for device, size in device_size.items()]       )
  print(f'''\nTheoretical use:\n{message}'''       )

  _a					      =						AutoTokenizer.from_pretrained(args.tokenizer_name       )

  _a					      =						start_measure()
  _a					      =						[]
  _a					      =						[]
  _a					      =						[]
  for prompt in PROMPTS:
    _a					      =						tokenizer(UpperCamelCase_  ,			return_tensors="pt"       ).to(0       )
    _a					      =						inputs["input_ids"][0].tolist()
    _a					      =						time.time()
    _a					      =						model.generate(inputs["input_ids"]       )
    _a					      =						time.time()
    _a					      =						outputs[0].tolist()
    _a					      =						len(UpperCamelCase_       ) if outputs[: len(UpperCamelCase_       )] != tokens else len(UpperCamelCase_       ) - len(UpperCamelCase_       )
    _a					      =						after_generate - before_generate

    _a					      =						tokenizer.decode(UpperCamelCase_  ,			skip_special_tokens=UpperCamelCase_       )
    texts_outs.append(UpperCamelCase_       )
    generation_times.append(UpperCamelCase_       )
    gen_tokens.append(UpperCamelCase_       )
    print(f'''Prompt: {prompt}\nGeneration {text_out}\nIn {generation_time:.2f}s for {num_gen_tokens} tokens\n'''       )

  _a					      =						end_measure(UpperCamelCase_       )
  log_measures(UpperCamelCase_  ,			"Model generation"       )

  _a					      =						[gen / tok for gen, tok in zip(UpperCamelCase_  ,			UpperCamelCase_       )]
  _a					      =						sum(UpperCamelCase_       ) / len(UpperCamelCase_       )
  print(f'''Average time of generation per token: {avg_gen:.2f}s'''       )
  print(f'''First generation (avg time per token): {generation_times_per_token[0]:.2f}s'''       )
  _a					      =						sum(generation_times_per_token[1:]       ) / (len(UpperCamelCase_       ) - 1)
  print(f'''Average time of generation per token (excluding the first): {avg_gen:.2f}s'''       )


if __name__ == "__main__":
     main()


