def run_metafeature_benchmark(benchmark_name, iters=<NUM_LIT:100>):
with open(METADATA_PATH, "<STR_LIT:r>") as f:<EOL><INDENT>dataset_descriptions = json.load(f)<EOL><DEDENT>benchmark_data = {}<EOL>for dataset_metadata in dataset_descriptions:<EOL><INDENT>print(dataset_metadata["<STR_LIT:filename>"])<EOL>X, Y, column_types = read_dataset(dataset_metadata)<EOL>init_times = []<EOL>total_compute_times = []<EOL>metafeature_compute_times = {<EOL>mf_id: [] for mf_id in Metafeatures.IDS<EOL>}<EOL>for i in range(iters):<EOL><INDENT>print(f"<STR_LIT>")<EOL>start_timestamp = time.time()<EOL>mf = Metafeatures()<EOL>init_timestamp = time.time()<EOL>computed_mfs = mf.compute(<EOL>X=X, Y=Y, column_types=column_types, seed=CORRECTNESS_SEED<EOL>)<EOL>compute_timestamp = time.time()<EOL>init_times.append(init_timestamp - start_timestamp)<EOL>total_compute_times.append(compute_timestamp - init_timestamp)<EOL>for mf_id, result in computed_mfs.items():<EOL><INDENT>metafeature_compute_times[mf_id].append(<EOL>result[Metafeatures.COMPUTE_TIME_KEY]<EOL>)<EOL><DEDENT><DEDENT>benchmark_data[dataset_metadata["<STR_LIT:filename>"]] = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": np.mean(init_times),<EOL>"<STR_LIT>": np.std(init_times)<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": np.mean(total_compute_times),<EOL>"<STR_LIT>": np.std(total_compute_times)<EOL>},<EOL>"<STR_LIT>": {<EOL>mf_id: {<EOL>"<STR_LIT>": np.mean(mf_times),<EOL>"<STR_LIT>": np.std(mf_times)<EOL>} for mf_id, mf_times in metafeature_compute_times.items()<EOL>}<EOL>}<EOL><DEDENT>write_benchmark_data(benchmark_name, benchmark_data)<EOL>
Computes metafeatures `iters` times over the test datasets and stores
comparable information in ./<benchmark_name>.json.