'''simple docstring'''


import argparse
import time

import torch
import transformers
from measures_util import end_measure, log_measures, start_measure
from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeqaSeqLM, AutoTokenizer

from accelerate.utils import compute_module_sizes


snake_case__	:     Union[str, Any]      =    {
    '''gpt-j-6b''': {'''is_causal''': True, '''model''': '''sgugger/sharded-gpt-j-6B''', '''tokenizer''': '''EleutherAI/gpt-j-6B'''},
    '''gpt-neox''': {'''is_causal''': True, '''model''': '''EleutherAI/gpt-neox-20b'''},
    '''opt''': {'''is_causal''': True, '''model''': '''facebook/opt-30b'''},
    '''T0pp''': {'''is_causal''': False, '''model''': '''bigscience/T0pp''', '''model_revision''': '''sharded'''},
}

snake_case__	:     Any      =    [
    '''Hello, my name is''',
    '''Are unicorns real? Unicorns are''',
    '''For the first time in several years,''',
    '''My name is Julien and I am''',
    '''The goal of life is''',
    '''Whenever I\'m sad, I like to''',
]


def    _lowerCamelCase       ( ):



     """simple docstring"""





     UpperCAmelCase_				:  Tuple  = argparse.ArgumentParser(description='Run and time generations on a big model using Accelerate.'  )
     parser.add_argument('model_name'      ,	type=lowerCamelCase_      ,	default=lowerCamelCase_      ,	help='The name of the model to try.'  )
     parser.add_argument(
         '--tokenizer_name'      ,	type=lowerCamelCase_      ,	default=lowerCamelCase_      ,	help='The name of the tokenizer (if different from the model.'  )
     parser.add_argument('--is_causal'      ,	type=lowerCamelCase_      ,	default=lowerCamelCase_      ,	help='Whether or not the model is causal.'  )
     parser.add_argument(
         '--model_revision'      ,	type=lowerCamelCase_      ,	default=lowerCamelCase_      ,	help='The revision to use for the model checkpoint.'  )
     parser.add_argument('--torch_dtype'      ,	type=lowerCamelCase_      ,	default=lowerCamelCase_      ,	help='The dtype for the model.'  )
     parser.add_argument('--disk_offload'      ,	action='store_true'  )

     UpperCAmelCase_				:  Union[str, Any]  = parser.parse_args()

     # Sanitize args
     if args.model_name in DEFAULT_MODELS:
          UpperCAmelCase_				:  Dict  = DEFAULT_MODELS[args.model_name]
          UpperCAmelCase_				:  Tuple  = defaults['model']
          if args.tokenizer_name is None:
               UpperCAmelCase_				:  Dict  = defaults.get('tokenizer'      ,	args.model_name  )
          if args.is_causal is None:
               UpperCAmelCase_				:  Union[str, Any]  = defaults['is_causal']
          if args.model_revision is None:
               UpperCAmelCase_				:  Optional[Any]  = defaults.get('model_revision'      ,	'main'  )

     if args.is_causal is None:
          raise ValueError('Could not infer the default for `--is_causal`, pass either True or False for it.'  )
     if args.tokenizer_name is None:
          UpperCAmelCase_				:  str  = args.model_name
     if args.model_revision is None:
          UpperCAmelCase_				:  Tuple  = 'main'

     return args


def    _lowerCamelCase       ( ):



     """simple docstring"""





     transformers.utils.logging.set_verbosity_error()
     UpperCAmelCase_				:  Optional[Any]  = parse_args()

     if args.torch_dtype is None:
          UpperCAmelCase_				:  Optional[Any]  = AutoConfig.from_pretrained(args.model_name  )
          UpperCAmelCase_				:  List[Any]  = getattr(lowerCamelCase_      ,	'torch_dtype'      ,	torch.floataa  )
     else:
          UpperCAmelCase_				:  Any  = getattr(lowerCamelCase_      ,	args.torch_dtype  )
     UpperCAmelCase_				:  Union[str, Any]  = AutoModelForCausalLM if args.is_causal else AutoModelForSeqaSeqLM
     UpperCAmelCase_				:  Any  = {
         'torch_dtype': torch_dtype,
         'revision': args.model_revision,
     }
     if args.disk_offload:
          UpperCAmelCase_				:  List[str]  = 'tmp_offload'
          UpperCAmelCase_				:  str  = True

     UpperCAmelCase_				:  Union[str, Any]  = start_measure()
     UpperCAmelCase_				:  Optional[int]  = model_cls.from_pretrained(args.model_name      ,	device_map='auto'      ,	**lowerCamelCase_  )
     UpperCAmelCase_				:  Any  = end_measure(lowerCamelCase_  )
     log_measures(lowerCamelCase_      ,	'Model loading'  )

     UpperCAmelCase_				:  int  = compute_module_sizes(lowerCamelCase_  )
     UpperCAmelCase_				:  Tuple  = {v: 0 for v in model.hf_device_map.values()}
     for module, device in model.hf_device_map.items():
          device_size[device] += module_sizes[module]
     UpperCAmelCase_				:  Any  = '\n'.join([F'''- {device}: {size // 2**20}MiB''' for device, size in device_size.items()]  )
     print(F'''\nTheoretical use:\n{message}'''  )

     UpperCAmelCase_				:  int  = AutoTokenizer.from_pretrained(args.tokenizer_name  )

     UpperCAmelCase_				:  List[str]  = start_measure()
     UpperCAmelCase_				:  Optional[Any]  = []
     UpperCAmelCase_				:  List[str]  = []
     UpperCAmelCase_				:  List[Any]  = []
     for prompt in PROMPTS:
          UpperCAmelCase_				:  Union[str, Any]  = tokenizer(lowerCamelCase_      ,	return_tensors='pt'  ).to(0  )
          UpperCAmelCase_				:  Any  = inputs['input_ids'][0].tolist()
          UpperCAmelCase_				:  Union[str, Any]  = time.time()
          UpperCAmelCase_				:  Dict  = model.generate(inputs['input_ids']  )
          UpperCAmelCase_				:  int  = time.time()
          UpperCAmelCase_				:  Optional[int]  = outputs[0].tolist()
          UpperCAmelCase_				:  Any  = len(lowerCamelCase_  ) if outputs[: len(lowerCamelCase_  )] != tokens else len(lowerCamelCase_  ) - len(lowerCamelCase_  )
          UpperCAmelCase_				:  List[str]  = after_generate - before_generate

          UpperCAmelCase_				:  Tuple  = tokenizer.decode(lowerCamelCase_      ,	skip_special_tokens=lowerCamelCase_  )
          texts_outs.append(lowerCamelCase_  )
          generation_times.append(lowerCamelCase_  )
          gen_tokens.append(lowerCamelCase_  )
          print(F'''Prompt: {prompt}\nGeneration {text_out}\nIn {generation_time:.2f}s for {num_gen_tokens} tokens\n'''  )

     UpperCAmelCase_				:  List[str]  = end_measure(lowerCamelCase_  )
     log_measures(lowerCamelCase_      ,	'Model generation'  )

     UpperCAmelCase_				:  Any  = [gen / tok for gen, tok in zip(lowerCamelCase_      ,	lowerCamelCase_  )]
     UpperCAmelCase_				:  int  = sum(lowerCamelCase_  ) / len(lowerCamelCase_  )
     print(F'''Average time of generation per token: {avg_gen:.2f}s'''  )
     print(F'''First generation (avg time per token): {generation_times_per_token[0]:.2f}s'''  )
     UpperCAmelCase_				:  int  = sum(generation_times_per_token[1:]  ) / (len(lowerCamelCase_  ) - 1)
     print(F'''Average time of generation per token (excluding the first): {avg_gen:.2f}s'''  )


if __name__ == "__main__":
 main()







'''simple docstring'''


def    _lowerCamelCase       ( lowerCamelCase_   :				list  ):



     """simple docstring"""





     UpperCAmelCase_				:  Optional[int]  = len(lowerCamelCase_  )

     # Set the gap value to be decreased by a factor of 1.3
     # after each iteration
     UpperCAmelCase_				:  Optional[Any]  = 1.3

     # Continue sorting until the gap is 1
     while gap > 1:
          # Decrease the gap value
          UpperCAmelCase_				:  Optional[Any]  = int(gap / shrink  )

          # Sort the elements using insertion sort
          for i in range(lowerCamelCase_      ,	len(lowerCamelCase_  )  ):
               UpperCAmelCase_				:  Optional[int]  = collection[i]
               UpperCAmelCase_				:  Tuple  = i
               while j >= gap and collection[j - gap] > temp:
                    UpperCAmelCase_				:  Tuple  = collection[j - gap]
                    j -= gap
               UpperCAmelCase_				:  Optional[int]  = temp

     return collection


if __name__ == "__main__":
 import doctest

 doctest.testmod()






