def main():
ROOT_LOGGER.setLevel(logging.DEBUG)<EOL>file_handler = logging.StreamHandler(LOG_PATH.open("<STR_LIT>"))<EOL>formatter = logging.Formatter(LOG_FORMAT)<EOL>file_handler.setFormatter(formatter)<EOL>file_handler.setLevel(logging.DEBUG)<EOL>ROOT_LOGGER.addHandler(file_handler)<EOL>terminal_handler = logging.StreamHandler(stdout)<EOL>terminal_handler.setFormatter(formatter)<EOL>terminal_handler.setLevel(logging.INFO)<EOL>ROOT_LOGGER.addHandler(terminal_handler)<EOL>LOGGER.debug("<STR_LIT>")<EOL>parser = ArgumentParser(<EOL>description="""<STR_LIT>""")<EOL>parser.add_argument(<EOL>"<STR_LIT>", nargs='<STR_LIT:+>', required=False,<EOL>type=lambda s: LabelledPath(s.split('<STR_LIT:=>', <NUM_LIT:1>)[<NUM_LIT:0>], Path(s.split('<STR_LIT:=>', <NUM_LIT:1>)[<NUM_LIT:1>])), help="""<STR_LIT>""")<EOL>parser.add_argument(<EOL>"<STR_LIT>", nargs='<STR_LIT:+>', required=False,<EOL>type=lambda s: (LabelledPath.labels[s.split('<STR_LIT::>', <NUM_LIT:1>)[<NUM_LIT:0>]], Path(s.split('<STR_LIT::>', <NUM_LIT:1>)[<NUM_LIT:1>])), help="""<STR_LIT>""")<EOL>parser.add_argument(<EOL>"<STR_LIT>", type=Path, nargs='<STR_LIT:+>', help="""<STR_LIT>""")<EOL>parser.add_argument(<EOL>"<STR_LIT>", type=Path, default=POSITIONS_ALL_PATH, help="""<STR_LIT>""")<EOL>parser.add_argument(<EOL>"<STR_LIT>", type=Path, default=ESTIMATES_PATH, help="""<STR_LIT>""")<EOL>parser.add_argument(<EOL>"<STR_LIT>", type=int, default=<NUM_LIT:1>, help="""<STR_LIT>""")<EOL>args = parser.parse_args()<EOL>LOGGER.debug("<STR_LIT>")<EOL>if args.datasets:<EOL><INDENT>for dataset in args.datasets:<EOL><INDENT>assert dataset.path.exists() and dataset.path.is_dir(),"<STR_LIT>" % dataset.path<EOL><DEDENT>assert args.judgements, "<STR_LIT>"<EOL>for _, judgement_path in args.judgements:<EOL><INDENT>assert judgement_path.exists() and judgement_path.is_file(),"<STR_LIT>" % judgement_path<EOL><DEDENT>assert not args.estimates.exists() or args.estimates.is_file(),"<STR_LIT>" % args.estimates<EOL>if args.positions.exists():<EOL><INDENT>LOGGER.warning("<STR_LIT>", args.positions.name)<EOL><DEDENT>if args.estimates.exists():<EOL><INDENT>LOGGER.warning("<STR_LIT>", args.estimates.name)<EOL><DEDENT><DEDENT>if args.plots:<EOL><INDENT>assert args.datasets or args.estimates,"<STR_LIT>"<EOL>if not args.datasets:<EOL><INDENT>assert args.estimates.exists() and args.estimates.is_file(),"<STR_LIT>" % args.estimates<EOL><DEDENT>for plot in args.plots:<EOL><INDENT>assert plot.parents[<NUM_LIT:0>].exists() and plot.parents[<NUM_LIT:0>].is_dir(),"<STR_LIT>" %plot.parents[<NUM_LIT:0>]<EOL>if plot.exists():<EOL><INDENT>LOGGER.warning("<STR_LIT>", plot)<EOL><DEDENT><DEDENT><DEDENT>assert args.positions.parents[<NUM_LIT:0>].exists() and args.positions.parents[<NUM_LIT:0>].is_dir(),"<STR_LIT>" %args.positions.parents[<NUM_LIT:0>]<EOL>assert args.estimates.parents[<NUM_LIT:0>].exists() and args.estimates.parents[<NUM_LIT:0>].is_dir(),"<STR_LIT>" %args.estimates.parents[<NUM_LIT:0>]<EOL>assert args.num_workers > <NUM_LIT:0>, "<STR_LIT>"<EOL>if args.datasets:<EOL><INDENT>identifiers_judged = {}<EOL>identifiers_relevant = {}<EOL>for dataset_path, judgement_path in args.judgements:<EOL><INDENT>LOGGER.info(<EOL>"<STR_LIT>", judgement_path.name)<EOL>if dataset_path not in identifiers_judged:<EOL><INDENT>identifiers_judged[dataset_path] = set()<EOL><DEDENT>if dataset_path not in identifiers_relevant:<EOL><INDENT>identifiers_relevant[dataset_path] = set()<EOL><DEDENT>with judgement_path.open("<STR_LIT>") as f:<EOL><INDENT>for identifier, score in get_judged_identifiers(f):<EOL><INDENT>identifiers_judged[dataset_path].add(identifier)<EOL>if score > MIN_RELEVANT_SCORE:<EOL><INDENT>identifiers_relevant[dataset_path].add(identifier)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>identifiers_all = {}<EOL>positions_all = {}<EOL>positions_relevant = {}<EOL>for dataset in args.datasets:<EOL><INDENT>LOGGER.info(<EOL>"<STR_LIT>", dataset.path.name)<EOL>identifiers_all[dataset.path] = []<EOL>positions_all[dataset.path] = []<EOL>positions_relevant[dataset.path] = []<EOL>for directory, identifier, position in get_all_positions(<EOL>dataset.path, args.num_workers):<EOL><INDENT>identifiers_all[dataset.path].append((directory, identifier))<EOL>positions_all[dataset.path].append(position)<EOL>if identifier in identifiers_relevant[dataset.path]:<EOL><INDENT>positions_relevant[dataset.path].append(position)<EOL><DEDENT><DEDENT><DEDENT>for dataset in args.datasets:<EOL><INDENT>LOGGER.info(<EOL>"<STR_LIT>",<EOL>len(identifiers_relevant[dataset.path]), len(identifiers_judged[dataset.path]),<EOL>len(identifiers_all[dataset.path]), dataset.path.name)<EOL><DEDENT>LOGGER.info("<STR_LIT>", args.positions.name)<EOL>with gzip.open(args.positions.open("<STR_LIT:wb>"), "<STR_LIT:wb>") as f:<EOL><INDENT>pickle.dump({<EOL>dataset.path: {<EOL>identifier: position for position, (_, identifier)<EOL>in zip(positions_all[dataset.path], identifiers_all[dataset.path])<EOL>} for dataset in args.datasets}, f)<EOL><DEDENT>LOGGER.info("<STR_LIT>")<EOL>estimators = get_estimators(positions_all, positions_relevant)<EOL>LOGGER.info("<STR_LIT>")<EOL>estimates = get_estimates(estimators, SAMPLES, args.num_workers)<EOL>LOGGER.info("<STR_LIT>", args.estimates.name)<EOL>with gzip.open(args.estimates.open("<STR_LIT:wb>"), "<STR_LIT:wb>") as f:<EOL><INDENT>pickle.dump(estimates, f)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGGER.info("<STR_LIT>", args.estimates.name)<EOL>with gzip.open(args.estimates.open("<STR_LIT:rb>"), "<STR_LIT:rb>") as f:<EOL><INDENT>estimates = pickle.load(f)<EOL><DEDENT><DEDENT>if args.plots:<EOL><INDENT>figure = plot_estimates(SAMPLES, estimates)<EOL>for plot_path in args.plots:<EOL><INDENT>LOGGER.info("<STR_LIT>", plot_path.name)<EOL>figure.savefig(str(plot_path))<EOL><DEDENT><DEDENT>
Main entry point of the app