def supervise_until_complete(monitor_data, dependencies, args, recovery_file):
project = args['<STR_LIT>']<EOL>workspace = args['<STR_LIT>']<EOL>namespace = args['<STR_LIT>']<EOL>sample_sets = args['<STR_LIT>']<EOL>recovery_data = {'<STR_LIT:args>': args}<EOL>if not validate_monitor_tasks(dependencies, args):<EOL><INDENT>logging.error("<STR_LIT>")<EOL>return<EOL><DEDENT>while True:<EOL><INDENT>running = <NUM_LIT:0><EOL>waiting = <NUM_LIT:0><EOL>completed = <NUM_LIT:0><EOL>r = fapi.list_submissions(project, workspace)<EOL>sub_list = r.json()<EOL>sub_lookup = {s["<STR_LIT>"]: s for s in sub_list}<EOL>for n in dependencies:<EOL><INDENT>for sset in sample_sets:<EOL><INDENT>task_data = monitor_data[n][sset]<EOL>if task_data['<STR_LIT:state>'] == "<STR_LIT>":<EOL><INDENT>upstream_evaluated = True<EOL>for dep in dependencies[n]:<EOL><INDENT>upstream_task_data = monitor_data[dep['<STR_LIT>']][sset]<EOL>if not upstream_task_data.get('<STR_LIT>'):<EOL><INDENT>upstream_evaluated = False<EOL><DEDENT><DEDENT>if upstream_evaluated:<EOL><INDENT>should_run = True<EOL>for dep in dependencies[n]:<EOL><INDENT>upstream_task_data = monitor_data[dep['<STR_LIT>']][sset]<EOL>mode = dep['<STR_LIT>']<EOL>if mode == '<STR_LIT>' and not upstream_task_data['<STR_LIT>']:<EOL><INDENT>should_run = False<EOL><DEDENT><DEDENT>if should_run:<EOL><INDENT>fc_config = n<EOL>logging.info("<STR_LIT>" + fc_config + "<STR_LIT>" + sset)<EOL>for retry in range(<NUM_LIT:3>):<EOL><INDENT>r = fapi.create_submission(<EOL>project, workspace, namespace, fc_config,<EOL>sset, etype="<STR_LIT>", expression=None<EOL>)<EOL>if r.status_code == <NUM_LIT>:<EOL><INDENT>task_data['<STR_LIT>'] = r.json()['<STR_LIT>']<EOL>task_data['<STR_LIT:state>'] = "<STR_LIT>"<EOL>running += <NUM_LIT:1><EOL>break<EOL><DEDENT>else:<EOL><INDENT>logging.debug("<STR_LIT>" + fc_config<EOL>+ "<STR_LIT>" + sset + "<STR_LIT>"<EOL>+ r.content + "<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logging.error("<STR_LIT>")<EOL>task_data['<STR_LIT:state>'] = '<STR_LIT>'<EOL>task_data['<STR_LIT>'] = True<EOL>task_data['<STR_LIT>'] = False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>task_data['<STR_LIT:state>'] = "<STR_LIT>"<EOL>task_data['<STR_LIT>'] = True<EOL>completed += <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>waiting += <NUM_LIT:1><EOL><DEDENT><DEDENT>elif task_data['<STR_LIT:state>'] == "<STR_LIT>":<EOL><INDENT>submission = sub_lookup[task_data['<STR_LIT>']]<EOL>status = submission['<STR_LIT:status>']<EOL>if status == "<STR_LIT>":<EOL><INDENT>logging.info("<STR_LIT>" + n + "<STR_LIT>" + sset)<EOL>success = '<STR_LIT>' not in submission['<STR_LIT>']<EOL>task_data['<STR_LIT>'] = True<EOL>task_data['<STR_LIT>'] = success<EOL>task_data['<STR_LIT:state>'] = "<STR_LIT>"<EOL>completed += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>running += <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>completed += <NUM_LIT:1><EOL><DEDENT>recovery_data['<STR_LIT>'] = monitor_data<EOL>recovery_data['<STR_LIT>'] = dependencies<EOL>with open(recovery_file, '<STR_LIT:w>') as rf:<EOL><INDENT>json.dump(recovery_data, rf)<EOL><DEDENT><DEDENT><DEDENT>logging.info("<STR_LIT>".format(waiting, running, completed))<EOL>if all(monitor_data[n][sset]['<STR_LIT>']<EOL>for n in monitor_data for sset in monitor_data[n]):<EOL><INDENT>logging.info("<STR_LIT>")<EOL>break<EOL><DEDENT>time.sleep(<NUM_LIT:30>)<EOL><DEDENT>
Supervisor loop. Loop forever until all tasks are evaluated or completed