def run_experiment(methods, data, n_classes, true_labels, n_runs=<NUM_LIT:10>, use_purity=True, use_nmi=False, use_ari=False, use_nne=False, consensus=False):
results = []<EOL>names = []<EOL>clusterings = {}<EOL>other_results = {}<EOL>other_results['<STR_LIT>'] = {}<EOL>other_results['<STR_LIT>'] = {}<EOL>if use_purity:<EOL><INDENT>purity_method = purity<EOL><DEDENT>elif use_nmi:<EOL><INDENT>purity_method = nmi<EOL><DEDENT>elif use_ari:<EOL><INDENT>purity_method = ari<EOL><DEDENT>elif use_nne:<EOL><INDENT>purity_method = nne<EOL><DEDENT>for i in range(n_runs):<EOL><INDENT>print('<STR_LIT>'.format(i))<EOL>purities = []<EOL>r = <NUM_LIT:0><EOL>method_index = <NUM_LIT:0><EOL>for preproc, cluster in methods:<EOL><INDENT>t0 = time.time()<EOL>if isinstance(preproc, Preprocess):<EOL><INDENT>preprocessed, ll = preproc.run(data)<EOL>output_names = preproc.output_names<EOL><DEDENT>else:<EOL><INDENT>p1 = data<EOL>output_names = ['<STR_LIT>']<EOL>for p in preproc:<EOL><INDENT>p1, ll = p.run(p1)<EOL>p1 = p1[<NUM_LIT:0>]<EOL>if output_names[<NUM_LIT:0>] != '<STR_LIT>':<EOL><INDENT>output_names[<NUM_LIT:0>] = output_names[<NUM_LIT:0>] + '<STR_LIT:_>' + p.output_names[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>output_names[<NUM_LIT:0>] = p.output_names[<NUM_LIT:0>]<EOL><DEDENT><DEDENT>preprocessed = [p1]<EOL><DEDENT>t1 = time.time() - t0<EOL>for name, pre in zip(output_names, preprocessed):<EOL><INDENT>starting_index = method_index<EOL>if isinstance(cluster, Cluster):<EOL><INDENT>t0 = time.time()<EOL>labels = cluster.run(pre)<EOL>t2 = t1 + time.time() - t0<EOL>if use_nne:<EOL><INDENT>purities.append(purity_method(pre, true_labels))<EOL><DEDENT>else:<EOL><INDENT>purities.append(purity_method(labels, true_labels))<EOL><DEDENT>if i==<NUM_LIT:0>:<EOL><INDENT>names.append(name + '<STR_LIT:_>' + cluster.name)<EOL>clusterings[names[-<NUM_LIT:1>]] = []<EOL>other_results['<STR_LIT>'][names[-<NUM_LIT:1>]] = []<EOL><DEDENT>print(names[r])<EOL>clusterings[names[r]].append(labels)<EOL>print('<STR_LIT>' + str(t2))<EOL>other_results['<STR_LIT>'][names[r]].append(t2)<EOL>print(purities[-<NUM_LIT:1>])<EOL>r += <NUM_LIT:1><EOL>method_index += <NUM_LIT:1><EOL><DEDENT>elif type(cluster) == list:<EOL><INDENT>for c in cluster:<EOL><INDENT>if isinstance(c, list):<EOL><INDENT>t2 = t1<EOL>name2 = name<EOL>sub_data = pre.copy()<EOL>for subproc in c[:-<NUM_LIT:1>]:<EOL><INDENT>t0 = time.time()<EOL>subproc_out, ll = subproc.run(sub_data)<EOL>sub_data = subproc_out[<NUM_LIT:0>]<EOL>name2 = name2 + '<STR_LIT:_>' + subproc.output_names[<NUM_LIT:0>]<EOL>t2 += time.time() - t0<EOL><DEDENT>t0 = time.time()<EOL>labels = c[-<NUM_LIT:1>].run(sub_data)<EOL>t2 += time.time() - t0<EOL>if use_nne:<EOL><INDENT>purities.append(purity_method(sub_data, true_labels))<EOL><DEDENT>else:<EOL><INDENT>purities.append(purity_method(labels, true_labels))<EOL><DEDENT>if i==<NUM_LIT:0>:<EOL><INDENT>names.append(name2 + '<STR_LIT:_>' + c[-<NUM_LIT:1>].name)<EOL>clusterings[names[-<NUM_LIT:1>]] = []<EOL>other_results['<STR_LIT>'][names[-<NUM_LIT:1>]] = []<EOL><DEDENT>print(names[r])<EOL>clusterings[names[r]].append(labels)<EOL>other_results['<STR_LIT>'][names[r]].append(t2)<EOL>print('<STR_LIT>' + str(t2))<EOL>print(purities[-<NUM_LIT:1>])<EOL>r += <NUM_LIT:1><EOL>method_index += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>t0 = time.time()<EOL>labels = c.run(pre)<EOL>t2 = t1 + time.time() - t0<EOL>if i==<NUM_LIT:0>:<EOL><INDENT>names.append(name + '<STR_LIT:_>' + c.name)<EOL>clusterings[names[-<NUM_LIT:1>]] = []<EOL>other_results['<STR_LIT>'][names[-<NUM_LIT:1>]] = []<EOL><DEDENT>if use_nne:<EOL><INDENT>purities.append(purity_method(pre, true_labels))<EOL><DEDENT>else:<EOL><INDENT>purities.append(purity_method(labels, true_labels))<EOL><DEDENT>print(names[r])<EOL>clusterings[names[r]].append(labels)<EOL>other_results['<STR_LIT>'][names[r]].append(t2)<EOL>print('<STR_LIT>' + str(t2))<EOL>print(purities[-<NUM_LIT:1>])<EOL>r += <NUM_LIT:1><EOL>method_index += <NUM_LIT:1><EOL><DEDENT>except:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT><DEDENT>num_clustering_results = method_index - starting_index<EOL>clustering_results = purities[-num_clustering_results:]<EOL>if i > <NUM_LIT:0> and len(clustering_results) > <NUM_LIT:0>:<EOL><INDENT>old_clustering_results = results[-<NUM_LIT:1>][starting_index:method_index]<EOL>if max(old_clustering_results) < max(clustering_results):<EOL><INDENT>other_results['<STR_LIT>'][name] = pre<EOL><DEDENT><DEDENT>else:<EOL><INDENT>other_results['<STR_LIT>'][name] = pre<EOL><DEDENT><DEDENT><DEDENT>print('<STR_LIT:\t>'.join(names))<EOL>print('<STR_LIT>' + '<STR_LIT:\t>'.join(map(str, purities)))<EOL>results.append(purities)<EOL><DEDENT>consensus_purities = []<EOL>if consensus:<EOL><INDENT>other_results['<STR_LIT>'] = {}<EOL>k = len(np.unique(true_labels))<EOL>for name, clusts in clusterings.items():<EOL><INDENT>print(name)<EOL>clusts = np.vstack(clusts)<EOL>consensus_clust = CE.cluster_ensembles(clusts, verbose=False, N_clusters_max=k)<EOL>other_results['<STR_LIT>'][name] = consensus_clust<EOL>if use_purity:<EOL><INDENT>consensus_purity = purity(consensus_clust.flatten(), true_labels)<EOL>print('<STR_LIT>' + str(consensus_purity))<EOL>consensus_purities.append(consensus_purity)<EOL><DEDENT>if use_nmi:<EOL><INDENT>consensus_nmi = nmi(true_labels, consensus_clust)<EOL>print('<STR_LIT>' + str(consensus_nmi))<EOL>consensus_purities.append(consensus_nmi)<EOL><DEDENT>if use_ari:<EOL><INDENT>consensus_ari = ari(true_labels, consensus_clust)<EOL>print('<STR_LIT>' + str(consensus_ari))<EOL>consensus_purities.append(consensus_ari)<EOL><DEDENT><DEDENT>print('<STR_LIT>' + '<STR_LIT:\t>'.join(map(str, consensus_purities)))<EOL><DEDENT>other_results['<STR_LIT>'] = clusterings<EOL>return results, names, other_results<EOL>
runs a pre-processing + clustering experiment...

exactly one of use_purity, use_nmi, or use_ari can be true

Args:
    methods: list of 2-tuples. The first element is either a single Preprocess object or a list of Preprocess objects, to be applied in sequence to the data. The second element is either a single Cluster object, a list of Cluster objects, or a list of lists, where each list is a sequence of Preprocess objects with the final element being a Cluster object.
    data: genes x cells array
    true_labels: 1d array of length cells
    consensus: if true, runs a consensus on cluster results for each method at the very end.
    use_purity, use_nmi, use_ari, use_nne: which error metric to use (at most one can be True)

Returns:
    purities (list of lists)
    names (list of lists)
    other (dict): keys: timing, preprocessing, clusterings