


from manim import *







class       lowerCamelCase__   (       lowerCAmelCase):





     def      __A					(self       )     ->   List[str]:
      _lowercase							      =Rectangle(height=0.5		,     width=0.5       )
      _lowercase							      =Rectangle(height=0.46		,     width=0.46       ).set_stroke(width=0       )

      _lowercase							      =Rectangle(height=0.25		,     width=0.25       )

      _lowercase							      =[mem.copy() for i in range(6       )]
      _lowercase							      =[mem.copy() for i in range(6       )]
      _lowercase							      =VGroup(*UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0       )
      _lowercase							      =VGroup(*UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0       )
      _lowercase							      =VGroup(UpperCAmelCase		,     UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0       )
      _lowercase							      =Text('''CPU'''		,     font_size=2_4       )
      _lowercase							      =Group(UpperCAmelCase		,     UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0.5		,     aligned_edge=UpperCAmelCase       )
      cpu.move_to([-2.5, -0.5, 0]       )
      self.add(UpperCAmelCase       )

      _lowercase							      =[mem.copy() for i in range(4       )]
      _lowercase							      =VGroup(*UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0       )
      _lowercase							      =Text('''GPU'''		,     font_size=2_4       )
      _lowercase							      =Group(UpperCAmelCase		,     UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0.5		,     aligned_edge=UpperCAmelCase       )
      gpu.move_to([-1, -1, 0]       )
      self.add(UpperCAmelCase       )

      _lowercase							      =[mem.copy() for i in range(6       )]
      _lowercase							      =VGroup(*UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0       )

      _lowercase							      =Text('''Model'''		,     font_size=2_4       )
      _lowercase							      =Group(UpperCAmelCase		,     UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0.5		,     aligned_edge=UpperCAmelCase       )
      model.move_to([3, -1.0, 0]       )
      self.add(UpperCAmelCase       )

      _lowercase							      =[]
      _lowercase							      =[]

      for i, rect in enumerate(UpperCAmelCase       ):
       _lowercase							      =fill.copy().set_fill(UpperCAmelCase		,     opacity=0.8       )
       target.move_to(UpperCAmelCase       )
       model_arr.append(UpperCAmelCase       )

       _lowercase							      =Rectangle(height=0.46		,     width=0.46       ).set_stroke(width=0.0       ).set_fill(UpperCAmelCase		,     opacity=0.8       )
       cpu_target.move_to(cpu_left_col_base[i]       )
       model_cpu_arr.append(UpperCAmelCase       )

      self.add(*UpperCAmelCase		,     *UpperCAmelCase       )

      _lowercase							      =[meta_mem.copy() for i in range(6       )]
      _lowercase							      =[meta_mem.copy() for i in range(6       )]
      _lowercase							      =VGroup(*UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0       )
      _lowercase							      =VGroup(*UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0       )
      _lowercase							      =VGroup(UpperCAmelCase		,     UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0       )
      _lowercase							      =Text('''Disk'''		,     font_size=2_4       )
      _lowercase							      =Group(UpperCAmelCase		,     UpperCAmelCase       ).arrange(UpperCAmelCase		,     buff=0.5		,     aligned_edge=UpperCAmelCase       )
      disk.move_to([-4, -1.25, 0]       )
      self.add(UpperCAmelCase		,     UpperCAmelCase       )

      _lowercase							      =Square(side_length=2.2       )
      key.move_to([-5, 2, 0]       )

      _lowercase							      =MarkupText(
          f"<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"		,     font_size=1_8		,     )

      key_text.move_to([-5, 2.4, 0]       )

      self.add(UpperCAmelCase		,     UpperCAmelCase       )

      _lowercase							      =MarkupText(
          f"<span fgcolor='{BLUE}'>●</span> Checkpoint"		,     font_size=1_8		,     )

      blue_text.next_to(UpperCAmelCase		,     DOWN * 2.4		,     aligned_edge=key_text.get_left()       )
      self.add(UpperCAmelCase       )

      _lowercase							      =MarkupText(
          f"Now watch as an input is passed through the model\nand how the memory is utilized and handled."		,     font_size=2_4		,     )
      step_a.move_to([2, 2, 0]       )

      self.play(Write(UpperCAmelCase       )       )

      _lowercase							      =Square(0.3       )
      input.set_fill(UpperCAmelCase		,     opacity=1.0       )
      input.set_stroke(width=0.0       )
      input.next_to(model_base[0]		,     UpperCAmelCase		,     buff=0.5       )

      self.play(Write(UpperCAmelCase       )       )

      input.generate_target()
      input.target.next_to(model_arr[0]		,     direction=UpperCAmelCase		,     buff=0.02       )
      self.play(MoveToTarget(UpperCAmelCase       )       )

      self.play(FadeOut(UpperCAmelCase       )       )

      _lowercase							      =Arrow(start=UpperCAmelCase		,     end=UpperCAmelCase		,     color=UpperCAmelCase		,     buff=0.5       )
      a.next_to(model_arr[0].get_left()		,     UpperCAmelCase		,     buff=0.2       )

      model_cpu_arr[0].generate_target()
      model_cpu_arr[0].target.move_to(gpu_rect[0]       )

      _lowercase							      =MarkupText(
          f"As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back."		,     font_size=2_4		,     )
      step_a.move_to([2, 2, 0]       )

      self.play(Write(UpperCAmelCase		,     run_time=3       )       )

      _lowercase							      ={'''run_time''': 1, '''fade_in''': True, '''fade_out''': True, '''buff''': 0.02}

      self.play(
          Write(UpperCAmelCase       )		,     Circumscribe(model_arr[0]		,     color=UpperCAmelCase		,     **UpperCAmelCase       )		,     Circumscribe(model_cpu_arr[0]		,     color=UpperCAmelCase		,     **UpperCAmelCase       )		,     Circumscribe(gpu_rect[0]		,     color=UpperCAmelCase		,     **UpperCAmelCase       )		,     )
      self.play(MoveToTarget(model_cpu_arr[0]       )       )

      _lowercase							      =a.copy()
      for i in range(6       ):
       a_c.next_to(model_arr[i].get_right() + 0.02		,     UpperCAmelCase		,     buff=0.2       )

       input.generate_target()
       input.target.move_to(model_arr[i].get_right() + 0.02       )

       _lowercase							      =AnimationGroup(
           FadeOut(UpperCAmelCase		,     run_time=0.5       )		,     MoveToTarget(UpperCAmelCase		,     run_time=0.5       )		,     FadeIn(UpperCAmelCase		,     run_time=0.5       )		,     lag_ratio=0.2       )

       self.play(UpperCAmelCase       )

       model_cpu_arr[i].generate_target()
       model_cpu_arr[i].target.move_to(cpu_left_col_base[i]       )

       if i < 5:
        model_cpu_arr[i + 1].generate_target()
        model_cpu_arr[i + 1].target.move_to(gpu_rect[0]       )
        if i >= 1:
         _lowercase							      =0.7

        self.play(
            Circumscribe(model_arr[i]		,     **UpperCAmelCase       )		,     Circumscribe(cpu_left_col_base[i]		,     **UpperCAmelCase       )		,     Circumscribe(cpu_left_col_base[i + 1]		,     color=UpperCAmelCase		,     **UpperCAmelCase       )		,     Circumscribe(gpu_rect[0]		,     color=UpperCAmelCase		,     **UpperCAmelCase       )		,     Circumscribe(model_arr[i + 1]		,     color=UpperCAmelCase		,     **UpperCAmelCase       )		,     )
        if i < 1:
         self.play(
             MoveToTarget(model_cpu_arr[i]       )		,     MoveToTarget(model_cpu_arr[i + 1]       )		,     )
        else:
         self.play(
             MoveToTarget(model_cpu_arr[i]		,     run_time=0.7       )		,     MoveToTarget(model_cpu_arr[i + 1]		,     run_time=0.7       )		,     )
       else:
        model_cpu_arr[i].generate_target()
        model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]       )
        input.generate_target()
        input.target.next_to(model_arr[-1].get_right()		,     RIGHT + 0.02		,     buff=0.2       )

        self.play(
            Circumscribe(model_arr[-1]		,     color=UpperCAmelCase		,     **UpperCAmelCase       )		,     Circumscribe(cpu_left_col_base[-1]		,     color=UpperCAmelCase		,     **UpperCAmelCase       )		,     Circumscribe(gpu_rect[0]		,     color=UpperCAmelCase		,     **UpperCAmelCase       )		,     )

        self.play(MoveToTarget(model_cpu_arr[i]       )       )

       _lowercase							      =a_c
       _lowercase							      =a_c.copy()

      input.generate_target()
      input.target.next_to(model_base[-1]		,     RIGHT + 0.02		,     buff=0.5       )
      self.play(
          FadeOut(UpperCAmelCase       )		,     FadeOut(UpperCAmelCase		,     run_time=0.5       )		,     )

      _lowercase							      =MarkupText(f"Inference on a model too large for GPU memory\nis successfully completed."		,     font_size=2_4       )
      step_a.move_to([2, 2, 0]       )

      self.play(Write(UpperCAmelCase		,     run_time=3       )		,     MoveToTarget(UpperCAmelCase       )       )

      self.wait()





import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


UpperCAmelCase__   	=							['''\nclass''', '''\ndef''', '''\n#''', '''\n@''', '''\nprint''', '''\nif''']







class       lowerCamelCase__   (       lowerCAmelCase):





     def __init__(self		,     UpperCAmelCase		,     UpperCAmelCase		,     UpperCAmelCase=None		,     UpperCAmelCase=1       )     ->   Dict:
      _lowercase							      =tokenizer
      _lowercase							      =dataset
      _lowercase							      =len(UpperCAmelCase       ) if n_tasks is None else n_tasks
      _lowercase							      =n_copies





     def __iter__(self       )     ->   Optional[Any]:
      _lowercase							      =[]
      for task in range(self.n_tasks       ):
       # without strip, the model generate commented codes ...
       prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip()       )
      _lowercase							      =self.tokenizer(UpperCAmelCase		,     padding=UpperCAmelCase		,     return_tensors='''pt'''       )
      for task in range(self.n_tasks       ):
       for _ in range(self.n_copies       ):
        yield {
            "ids": outputs.input_ids[task],
            "task_id": task,
            "input_len": outputs.attention_mask[task].sum(),
        }







class       lowerCamelCase__   (       lowerCAmelCase):





     def __init__(self		,     UpperCAmelCase		,     UpperCAmelCase		,     UpperCAmelCase       )     ->   Optional[Any]:
      _lowercase							      =start_length
      _lowercase							      =eof_strings
      _lowercase							      =tokenizer





     def __call__(self		,     UpperCAmelCase		,     UpperCAmelCase		,     **UpperCAmelCase       )     ->   Dict:
      _lowercase							      =self.tokenizer.batch_decode(input_ids[:, self.start_length :]       )
      _lowercase							      =[]
      for decoded_generation in decoded_generations:
       done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings       )       )
      return all(UpperCAmelCase       )



def 				UpperCAmelCase_					(							__snake_case						)     ->			Optional[Any]:







 """simple docstring"""


 _lowercase							      =re.split('''(%s)''' % '''|'''.join(__snake_case						)     ,						__snake_case						)
 # last string should be ""
 return "".join(string_list[:-2]						)
def 				UpperCAmelCase_					(							__snake_case     ,						__snake_case     ,						__snake_case     ,						__snake_case     ,						__snake_case     ,						__snake_case=20     ,						**__snake_case						)     ->			Tuple:







 """simple docstring"""


 _lowercase							      =defaultdict(__snake_case						)  # dict of list of generated tokens
 for step, batch in tqdm(enumerate(__snake_case						)						):
  with torch.no_grad():
   _lowercase							      =batch['''ids'''].shape[-1]
   _lowercase							      =accelerator.unwrap_model(__snake_case						).generate(
       input_ids=batch['''ids'''][:, : batch['''input_len''']]     ,						num_return_sequences=__snake_case     ,						**__snake_case						)
   # each task is generated batch_size times
   _lowercase							      =batch['''task_id'''].repeat(__snake_case						)
   _lowercase							      =accelerator.pad_across_processes(
       __snake_case     ,						dim=1     ,						pad_index=tokenizer.pad_token_id						)

   _lowercase      ,    _lowercase							      =accelerator.gather((generated_tokens, generated_tasks)						)
   _lowercase							      =generated_tokens.cpu().numpy()
   _lowercase							      =generated_tasks.cpu().numpy()

   for task, generated_tokens in zip(__snake_case     ,						__snake_case						):
    gen_token_dict[task].append(__snake_case						)

 _lowercase							      =[[] for _ in range(__snake_case						)]
 for task, generated_tokens in gen_token_dict.items():
  for s in generated_tokens:
   _lowercase							      =tokenizer.decode(__snake_case     ,						skip_special_tokens=__snake_case     ,						clean_up_tokenization_spaces=__snake_case						)
   code_gens[task].append(remove_last_block(__snake_case						)						)
 return code_gens
def 				UpperCAmelCase_					(							)     ->			str:







 """simple docstring"""


 _lowercase							      =HfArgumentParser(__snake_case						)
 _lowercase							      =parser.parse_args()

 transformers.logging.set_verbosity_error()
 # enables code execution in code_eval metric
 _lowercase							      =args.HF_ALLOW_CODE_EVAL
 # make sure tokenizer plays nice with multiprocessing
 _lowercase							      ='''false'''

 if args.num_workers is None:
  _lowercase							      =multiprocessing.cpu_count()

 # Use dataset load to feed to accelerate
 _lowercase							      =Accelerator()
 set_seed(args.seed     ,						device_specific=__snake_case						)

 # Load model and tokenizer
 _lowercase							      =AutoTokenizer.from_pretrained(args.model_ckpt						)
 _lowercase							      =tokenizer.eos_token
 _lowercase							      =AutoModelForCausalLM.from_pretrained(args.model_ckpt						)

 # Generation settings
 _lowercase							      ={
     '''do_sample''': args.do_sample,
     '''temperature''': args.temperature,
     '''max_new_tokens''': args.max_new_tokens,
     '''top_p''': args.top_p,
     '''top_k''': args.top_k,
     '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0     ,						__snake_case     ,						__snake_case						)]						),
 }

 # Load evaluation dataset and metric
 _lowercase							      =load_dataset('''openai_humaneval'''						)
 _lowercase							      =load_metric('''code_eval'''						)

 _lowercase							      =args.num_tasks if args.num_tasks is not None else len(human_eval['''test''']						)
 _lowercase							      =args.n_samples // args.batch_size

 _lowercase							      =TokenizedDataset(__snake_case     ,						human_eval['''test''']     ,						n_copies=__snake_case     ,						n_tasks=__snake_case						)
 # do not confuse args.batch_size, which is actually the num_return_sequences
 _lowercase							      =DataLoader(__snake_case     ,						batch_size=1						)

 # Run a quick test to see if code evaluation is enabled
 try:
  _lowercase							      =code_eval_metric.compute(references=['''''']     ,						predictions=[['''''']]						)
 except ValueError as exception:
  print(
      '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
      ''' flag to enable code evaluation.'''						)
  raise exception

 _lowercase      ,    _lowercase							      =accelerator.prepare(__snake_case     ,						__snake_case						)

 _lowercase							      =complete_code(
     __snake_case     ,						__snake_case     ,						__snake_case     ,						__snake_case     ,						n_tasks=__snake_case     ,						batch_size=args.batch_size     ,						**__snake_case     ,						)

 if accelerator.is_main_process:
  _lowercase							      =[]

  for task in tqdm(range(__snake_case						)						):
   _lowercase							      =human_eval['''test'''][task]['''test''']
   _lowercase							      =F"check({human_eval['test'][task]['entry_point']})"
   references.append('''\n''' + test_func + '''\n''' + entry_point						)

  # Evaluate completions with "code_eval" metric
  _lowercase      ,    _lowercase							      =code_eval_metric.compute(
      references=__snake_case     ,						predictions=__snake_case     ,						num_workers=args.num_workers						)
  print(F"Results: {pass_at_k}"						)

  # Save results to json file
  with open(args.output_file     ,						'''w'''						) as fp:
   json.dump(__snake_case     ,						__snake_case						)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
  main()

