def failure_recovery(self, fail_info):
<EOL>LOG.info("<STR_LIT>")<EOL>projs = self.get_fialed_projects_entries(constants.CREATE_FAIL)<EOL>for proj in projs:<EOL><INDENT>LOG.debug("<STR_LIT>", (<EOL>{'<STR_LIT:name>': proj.name}))<EOL>try:<EOL><INDENT>self.dcnm_client.create_project(self.cfg.dcnm.orchestrator_id,<EOL>proj.name,<EOL>self.cfg.dcnm.<EOL>default_partition_name,<EOL>proj.dci_id)<EOL><DEDENT>except dexc.DfaClientRequestFailed as e:<EOL><INDENT>LOG.error("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>{'<STR_LIT>': proj.name, '<STR_LIT>': str(e)})<EOL><DEDENT>else:<EOL><INDENT>self.update_project_info_cache(proj.id, dci_id=proj.dci_id,<EOL>name=proj.name,<EOL>opcode='<STR_LIT>')<EOL>LOG.debug('<STR_LIT>'<EOL>'<STR_LIT>', {'<STR_LIT:name>': proj.name})<EOL><DEDENT><DEDENT>projs = self.get_fialed_projects_entries(constants.UPDATE_FAIL)<EOL>for proj in projs:<EOL><INDENT>LOG.debug("<STR_LIT>", (<EOL>{'<STR_LIT:name>': proj.name}))<EOL>try:<EOL><INDENT>self.dcnm_client.update_project(proj.name,<EOL>self.cfg.dcnm.<EOL>default_partition_name,<EOL>proj.dci_id)<EOL><DEDENT>except dexc.DfaClientRequestFailed as exc:<EOL><INDENT>LOG.error("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>{'<STR_LIT>': proj.name, '<STR_LIT>': str(exc)})<EOL><DEDENT>else:<EOL><INDENT>self.update_project_info_cache(proj.id,<EOL>dci_id=proj.dci_id,<EOL>name=proj.name,<EOL>opcode='<STR_LIT>')<EOL>LOG.debug('<STR_LIT>'<EOL>'<STR_LIT>', {'<STR_LIT:name>': proj.name})<EOL><DEDENT><DEDENT>nets = self.get_all_networks()<EOL>for net in nets:<EOL><INDENT>if (net.result == constants.CREATE_FAIL and<EOL>net.source.lower() == '<STR_LIT>'):<EOL><INDENT>net_id = net.network_id<EOL>try:<EOL><INDENT>subnets = self.neutron_event.nclient.list_subnets(<EOL>network_id=net_id).get('<STR_LIT>')<EOL><DEDENT>except dexc.ConnectionFailed:<EOL><INDENT>LOG.exception('<STR_LIT>')<EOL>continue<EOL><DEDENT>for subnet in subnets:<EOL><INDENT>tenant_name = self.get_project_name(subnet['<STR_LIT>'])<EOL>snet = utils.Dict2Obj(subnet)<EOL>try:<EOL><INDENT>if not net.config_profile:<EOL><INDENT>cfgp, fwd_mod = (<EOL>self.dcnm_client.<EOL>get_config_profile_for_network(net.name))<EOL>net.config_profile = cfgp<EOL>net.fwd_mod = fwd_mod<EOL><DEDENT>self.dcnm_client.create_network(tenant_name, net, snet,<EOL>self.dcnm_dhcp)<EOL><DEDENT>except dexc.DfaClientRequestFailed:<EOL><INDENT>LOG.error('<STR_LIT>',<EOL>{'<STR_LIT>': net.name})<EOL><DEDENT>else:<EOL><INDENT>params = dict(<EOL>columns=dict(config_profile=net.config_profile,<EOL>fwd_mod=net.fwd_mod,<EOL>result=constants.RESULT_SUCCESS))<EOL>self.update_network(net_id, **params)<EOL>LOG.debug("<STR_LIT>"<EOL>"<STR_LIT>", {'<STR_LIT>': net.name})<EOL><DEDENT><DEDENT><DEDENT><DEDENT>instances = self.get_vms()<EOL>for vm in instances:<EOL><INDENT>vm_info = dict(status=vm.status,<EOL>vm_mac=vm.mac,<EOL>segmentation_id=vm.segmentation_id,<EOL>host=vm.host,<EOL>port_uuid=vm.port_id,<EOL>net_uuid=vm.network_id,<EOL>oui=dict(ip_addr=vm.ip,<EOL>vm_name=vm.name,<EOL>vm_uuid=vm.instance_id,<EOL>gw_mac=vm.gw_mac,<EOL>fwd_mod=vm.fwd_mod,<EOL>oui_id='<STR_LIT>'))<EOL>if vm.result == constants.CREATE_FAIL:<EOL><INDENT>try:<EOL><INDENT>self.neutron_event.send_vm_info(str(vm.host), str(vm_info))<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOG.error('<STR_LIT>'<EOL>'<STR_LIT>', str(e))<EOL><DEDENT>else:<EOL><INDENT>params = dict(columns=dict(<EOL>result=constants.RESULT_SUCCESS))<EOL>self.update_vm_db(vm.port_id, **params)<EOL>LOG.info('<STR_LIT>', {'<STR_LIT>': vm.name})<EOL><DEDENT><DEDENT><DEDENT>for vm in instances:<EOL><INDENT>if vm.result == constants.DELETE_FAIL:<EOL><INDENT>vm_info['<STR_LIT:status>'] = '<STR_LIT>'<EOL>try:<EOL><INDENT>self.neutron_event.send_vm_info(str(vm.host), str(vm_info))<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOG.error('<STR_LIT>'<EOL>'<STR_LIT>', str(e))<EOL><DEDENT>else:<EOL><INDENT>self.delete_vm_db(vm.port_id)<EOL>LOG.info('<STR_LIT>',<EOL>{'<STR_LIT>': vm.name})<EOL><DEDENT><DEDENT><DEDENT>for net in nets:<EOL><INDENT>if (net.result == constants.DELETE_FAIL and<EOL>net.source.lower() == '<STR_LIT>'):<EOL><INDENT>net_id = net.network_id<EOL>segid = net.segmentation_id<EOL>tenant_name = self.get_project_name(net.tenant_id)<EOL>try:<EOL><INDENT>self.dcnm_client.delete_network(tenant_name, net)<EOL><DEDENT>except dexc.DfaClientRequestFailed:<EOL><INDENT>LOG.error('<STR_LIT>',<EOL>{'<STR_LIT>': net.name})<EOL><DEDENT>else:<EOL><INDENT>self.delete_network_db(net_id)<EOL>self.segmentation_pool.add(segid)<EOL>LOG.debug("<STR_LIT>"<EOL>"<STR_LIT>", {'<STR_LIT>': net.name})<EOL><DEDENT><DEDENT><DEDENT>projs = self.get_fialed_projects_entries(constants.DELETE_FAIL)<EOL>for proj in projs:<EOL><INDENT>LOG.debug("<STR_LIT>", (<EOL>{'<STR_LIT:name>': proj.name}))<EOL>try:<EOL><INDENT>self.dcnm_client.delete_project(proj.name,<EOL>self.cfg.dcnm.<EOL>default_partition_name)<EOL><DEDENT>except dexc.DfaClientRequestFailed as e:<EOL><INDENT>LOG.error("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>{'<STR_LIT>': proj.name, '<STR_LIT>': str(e)})<EOL><DEDENT>else:<EOL><INDENT>self.update_project_info_cache(proj.id, opcode='<STR_LIT>')<EOL>LOG.debug("<STR_LIT>"<EOL>"<STR_LIT>", {'<STR_LIT>': proj.name})<EOL><DEDENT><DEDENT>self.fw_retry_failures()<EOL>if self.need_dhcp_check():<EOL><INDENT>nets = self.get_all_networks()<EOL>for net in nets:<EOL><INDENT>net_id = net.network_id<EOL>LOG.debug("<STR_LIT>", net_id)<EOL>self.correct_dhcp_ports(net_id)<EOL><DEDENT>self.decrement_dhcp_check()<EOL><DEDENT>LOG.info("<STR_LIT>")<EOL>
Failure recovery task.

        In case of failure in projects, network and VM create/delete, this
        task goes through all failure cases and try the request.