import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__lowercase							 =					['''\nclass''', '''\ndef''', '''\n#''', '''\n@''', '''\nprint''', '''\nif''']


class 	lowerCamelCase_     (   UpperCAmelCase_ ):






    '''simple docstring'''






    def __init__(	self , __lowercase , __lowercase , __lowercase=None , __lowercase=1)       ->	List[Any]:
        __UpperCamelCase     :List[Any]			  =		tokenizer
        __UpperCamelCase     :Tuple			  =		dataset
        __UpperCamelCase     :Optional[Any]			  =		len(__lowercase) if n_tasks is None else n_tasks
        __UpperCamelCase     :List[Any]			  =		n_copies







    def __iter__(	self)       ->	List[Any]:
        __UpperCamelCase     :Tuple			  =		[]
        for task in range(self.n_tasks):
            # without strip, the model generate commented codes ...
            prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip())
        __UpperCamelCase     :List[str]			  =		self.tokenizer(__lowercase , padding=__lowercase , return_tensors='''pt''')
        for task in range(self.n_tasks):
            for _ in range(self.n_copies):
                yield {
                    "ids": outputs.input_ids[task],
                    "task_id": task,
                    "input_len": outputs.attention_mask[task].sum(),
                }


class 	lowerCamelCase_     (   UpperCAmelCase_ ):






    '''simple docstring'''






    def __init__(	self , __lowercase , __lowercase , __lowercase)       ->	Any:
        __UpperCamelCase     :Optional[Any]			  =		start_length
        __UpperCamelCase     :Any			  =		eof_strings
        __UpperCamelCase     :int			  =		tokenizer







    def __call__(	self , __lowercase , __lowercase , **__lowercase)       ->	Tuple:
        __UpperCamelCase     :Union[str, Any]			  =		self.tokenizer.batch_decode(input_ids[:, self.start_length :])
        __UpperCamelCase     :Tuple			  =		[]
        for decoded_generation in decoded_generations:
            done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings))
        return all(__lowercase)



def lowerCamelCase  (   SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    __UpperCamelCase     :Dict			  =		re.split('''(%s)''' % '''|'''.join(SCREAMING_SNAKE_CASE	)  ,			SCREAMING_SNAKE_CASE	)
    # last string should be ""
    return "".join(string_list[:-2]	)



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE=20  ,			**SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    __UpperCamelCase     :Dict			  =		defaultdict(SCREAMING_SNAKE_CASE	)  # dict of list of generated tokens
    for step, batch in tqdm(enumerate(SCREAMING_SNAKE_CASE	)	):
        with torch.no_grad():
            __UpperCamelCase     :str			  =		batch['''ids'''].shape[-1]
            __UpperCamelCase     :List[str]			  =		accelerator.unwrap_model(SCREAMING_SNAKE_CASE	).generate(
                input_ids=batch['''ids'''][:, : batch['''input_len''']]  ,			num_return_sequences=SCREAMING_SNAKE_CASE  ,			**SCREAMING_SNAKE_CASE	)
            # each task is generated batch_size times
            __UpperCamelCase     :int			  =		batch['''task_id'''].repeat(SCREAMING_SNAKE_CASE	)
            __UpperCamelCase     :List[str]			  =		accelerator.pad_across_processes(
                SCREAMING_SNAKE_CASE  ,			dim=1  ,			pad_index=tokenizer.pad_token_id	)

            __UpperCamelCase  ,       __UpperCamelCase     :Union[str, Any]			  =		accelerator.gather((generated_tokens, generated_tasks)	)
            __UpperCamelCase     :Tuple			  =		generated_tokens.cpu().numpy()
            __UpperCamelCase     :Dict			  =		generated_tasks.cpu().numpy()

            for task, generated_tokens in zip(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	):
                gen_token_dict[task].append(SCREAMING_SNAKE_CASE	)

    __UpperCamelCase     :str			  =		[[] for _ in range(SCREAMING_SNAKE_CASE	)]
    for task, generated_tokens in gen_token_dict.items():
        for s in generated_tokens:
            __UpperCamelCase     :Any			  =		tokenizer.decode(SCREAMING_SNAKE_CASE  ,			skip_special_tokens=SCREAMING_SNAKE_CASE  ,			clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE	)
            code_gens[task].append(remove_last_block(SCREAMING_SNAKE_CASE	)	)
    return code_gens



def lowerCamelCase  (   ):
    '''simple docstring'''







    __UpperCamelCase     :Optional[Any]			  =		HfArgumentParser(SCREAMING_SNAKE_CASE	)
    __UpperCamelCase     :Any			  =		parser.parse_args()

    transformers.logging.set_verbosity_error()
    # enables code execution in code_eval metric
    __UpperCamelCase     :Union[str, Any]			  =		args.HF_ALLOW_CODE_EVAL
    # make sure tokenizer plays nice with multiprocessing
    __UpperCamelCase     :Optional[int]			  =		'''false'''

    if args.num_workers is None:
        __UpperCamelCase     :str			  =		multiprocessing.cpu_count()

    # Use dataset load to feed to accelerate
    __UpperCamelCase     :int			  =		Accelerator()
    set_seed(args.seed  ,			device_specific=SCREAMING_SNAKE_CASE	)

    # Load model and tokenizer
    __UpperCamelCase     :int			  =		AutoTokenizer.from_pretrained(args.model_ckpt	)
    __UpperCamelCase     :Dict			  =		tokenizer.eos_token
    __UpperCamelCase     :int			  =		AutoModelForCausalLM.from_pretrained(args.model_ckpt	)

    # Generation settings
    __UpperCamelCase     :Union[str, Any]			  =		{
        '''do_sample''': args.do_sample,
        '''temperature''': args.temperature,
        '''max_new_tokens''': args.max_new_tokens,
        '''top_p''': args.top_p,
        '''top_k''': args.top_k,
        '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)]	),
    }

    # Load evaluation dataset and metric
    __UpperCamelCase     :Union[str, Any]			  =		load_dataset('''openai_humaneval'''	)
    __UpperCamelCase     :List[Any]			  =		load_metric('''code_eval'''	)

    __UpperCamelCase     :str			  =		args.num_tasks if args.num_tasks is not None else len(human_eval['''test''']	)
    __UpperCamelCase     :int			  =		args.n_samples // args.batch_size

    __UpperCamelCase     :int			  =		TokenizedDataset(SCREAMING_SNAKE_CASE  ,			human_eval['''test''']  ,			n_copies=SCREAMING_SNAKE_CASE  ,			n_tasks=SCREAMING_SNAKE_CASE	)
    # do not confuse args.batch_size, which is actually the num_return_sequences
    __UpperCamelCase     :Optional[Any]			  =		DataLoader(SCREAMING_SNAKE_CASE  ,			batch_size=1	)

    # Run a quick test to see if code evaluation is enabled
    try:
        __UpperCamelCase     :int			  =		code_eval_metric.compute(references=['''''']  ,			predictions=[['''''']]	)
    except ValueError as exception:
        print(
            '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
            ''' flag to enable code evaluation.'''	)
        raise exception

    __UpperCamelCase  ,       __UpperCamelCase     :List[Any]			  =		accelerator.prepare(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)

    __UpperCamelCase     :Any			  =		complete_code(
        SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			n_tasks=SCREAMING_SNAKE_CASE  ,			batch_size=args.batch_size  ,			**SCREAMING_SNAKE_CASE  ,			)

    if accelerator.is_main_process:
        __UpperCamelCase     :Optional[int]			  =		[]

        for task in tqdm(range(SCREAMING_SNAKE_CASE	)	):
            __UpperCamelCase     :List[Any]			  =		human_eval['''test'''][task]['''test''']
            __UpperCamelCase     :Any			  =		f"""check({human_eval['test'][task]['entry_point']})"""
            references.append('''\n''' + test_func + '''\n''' + entry_point	)

        # Evaluate completions with "code_eval" metric
        __UpperCamelCase  ,       __UpperCamelCase     :Optional[Any]			  =		code_eval_metric.compute(
            references=SCREAMING_SNAKE_CASE  ,			predictions=SCREAMING_SNAKE_CASE  ,			num_workers=args.num_workers	)
        print(f"""Results: {pass_at_k}"""	)

        # Save results to json file
        with open(args.output_file  ,			'''w'''	) as fp:
            json.dump(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
    main()






import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


__lowercase							 =					logging.get_logger(__name__)



def lowerCamelCase  (   SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    __UpperCamelCase     :Tuple			  =		R'''\w+[.]\d+'''
    __UpperCamelCase     :List[str]			  =		re.findall(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)
    for pat in pats:
        __UpperCamelCase     :int			  =		key.replace(SCREAMING_SNAKE_CASE  ,			'''_'''.join(pat.split('''.'''	)	)	)
    return key



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    __UpperCamelCase     :Tuple			  =		pt_tuple_key[:-1] + ('''scale''',)
    if (
        any('''norm''' in str_ for str_ in pt_tuple_key	)
        and (pt_tuple_key[-1] == "bias")
        and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
        and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
    ):
        __UpperCamelCase     :str			  =		pt_tuple_key[:-1] + ('''scale''',)
        return renamed_pt_tuple_key, pt_tensor
    elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
        __UpperCamelCase     :Any			  =		pt_tuple_key[:-1] + ('''scale''',)
        return renamed_pt_tuple_key, pt_tensor

    # embedding
    if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
        __UpperCamelCase     :str			  =		pt_tuple_key[:-1] + ('''embedding''',)
        return renamed_pt_tuple_key, pt_tensor

    # conv layer
    __UpperCamelCase     :List[str]			  =		pt_tuple_key[:-1] + ('''kernel''',)
    if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
        __UpperCamelCase     :List[Any]			  =		pt_tensor.transpose(2  ,			3  ,			1  ,			0	)
        return renamed_pt_tuple_key, pt_tensor

    # linear layer
    __UpperCamelCase     :List[str]			  =		pt_tuple_key[:-1] + ('''kernel''',)
    if pt_tuple_key[-1] == "weight":
        __UpperCamelCase     :Any			  =		pt_tensor.T
        return renamed_pt_tuple_key, pt_tensor

    # old PyTorch layer norm weight
    __UpperCamelCase     :int			  =		pt_tuple_key[:-1] + ('''weight''',)
    if pt_tuple_key[-1] == "gamma":
        return renamed_pt_tuple_key, pt_tensor

    # old PyTorch layer norm bias
    __UpperCamelCase     :int			  =		pt_tuple_key[:-1] + ('''bias''',)
    if pt_tuple_key[-1] == "beta":
        return renamed_pt_tuple_key, pt_tensor

    return pt_tuple_key, pt_tensor



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE=42	):
    '''simple docstring'''







    __UpperCamelCase     :Union[str, Any]			  =		{k: v.numpy() for k, v in pt_state_dict.items()}

    # Step 2: Since the model is stateless, get random Flax params
    __UpperCamelCase     :str			  =		flax_model.init_weights(PRNGKey(SCREAMING_SNAKE_CASE	)	)

    __UpperCamelCase     :int			  =		flatten_dict(SCREAMING_SNAKE_CASE	)
    __UpperCamelCase     :List[Any]			  =		{}

    # Need to change some parameters name to match Flax names
    for pt_key, pt_tensor in pt_state_dict.items():
        __UpperCamelCase     :List[Any]			  =		rename_key(SCREAMING_SNAKE_CASE	)
        __UpperCamelCase     :List[Any]			  =		tuple(renamed_pt_key.split('''.'''	)	)

        # Correctly rename weight parameters
        __UpperCamelCase  ,       __UpperCamelCase     :Any			  =		rename_key_and_reshape_tensor(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)

        if flax_key in random_flax_state_dict:
            if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                raise ValueError(
                    f"""PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape """
                    f"""{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."""	)

        # also add unexpected weight so that warning is thrown
        __UpperCamelCase     :str			  =		jnp.asarray(SCREAMING_SNAKE_CASE	)

    return unflatten_dict(SCREAMING_SNAKE_CASE	)





