def entity_level_f1(<EOL>candidates, gold_file, attribute=None, corpus=None, parts_by_doc=None<EOL>):
docs = [(doc.name).upper() for doc in corpus] if corpus else None<EOL>val_on = attribute is not None<EOL>gold_set = get_gold_dict(<EOL>gold_file,<EOL>docs=docs,<EOL>doc_on=True,<EOL>part_on=True,<EOL>val_on=val_on,<EOL>attribute=attribute,<EOL>)<EOL>if len(gold_set) == <NUM_LIT:0>:<EOL><INDENT>logger.info(f"<STR_LIT>")<EOL>logger.error("<STR_LIT>")<EOL>return<EOL><DEDENT>logger.info("<STR_LIT>")<EOL>entities = set()<EOL>for i, c in enumerate(tqdm(candidates)):<EOL><INDENT>part = c[<NUM_LIT:0>].context.get_span()<EOL>doc = c[<NUM_LIT:0>].context.sentence.document.name.upper()<EOL>if attribute:<EOL><INDENT>val = c[<NUM_LIT:1>].context.get_span()<EOL><DEDENT>for p in get_implied_parts(part, doc, parts_by_doc):<EOL><INDENT>if attribute:<EOL><INDENT>entities.add((doc, p, val))<EOL><DEDENT>else:<EOL><INDENT>entities.add((doc, p))<EOL><DEDENT><DEDENT><DEDENT>(TP_set, FP_set, FN_set) = entity_confusion_matrix(entities, gold_set)<EOL>TP = len(TP_set)<EOL>FP = len(FP_set)<EOL>FN = len(FN_set)<EOL>prec = TP / (TP + FP) if TP + FP > <NUM_LIT:0> else float("<STR_LIT>")<EOL>rec = TP / (TP + FN) if TP + FN > <NUM_LIT:0> else float("<STR_LIT>")<EOL>f1 = <NUM_LIT:2> * (prec * rec) / (prec + rec) if prec + rec > <NUM_LIT:0> else float("<STR_LIT>")<EOL>logger.info("<STR_LIT>")<EOL>logger.info("<STR_LIT>")<EOL>logger.info("<STR_LIT>")<EOL>logger.info(f"<STR_LIT>")<EOL>logger.info(f"<STR_LIT>")<EOL>logger.info(f"<STR_LIT>")<EOL>logger.info("<STR_LIT>")<EOL>logger.info(f"<STR_LIT>")<EOL>logger.info("<STR_LIT>")<EOL>return [sorted(list(x)) for x in [TP_set, FP_set, FN_set]]<EOL>
Checks entity-level recall of candidates compared to gold.

    Turns a CandidateSet into a normal set of entity-level tuples
    (doc, part, [attribute_value])
    then compares this to the entity-level tuples found in the gold.

    Example Usage:
        from hardware_utils import entity_level_total_recall
        candidates = # CandidateSet of all candidates you want to consider
        gold_file = 'tutorials/tables/data/hardware/hardware_gold.csv'
        entity_level_total_recall(candidates, gold_file, 'stg_temp_min')