def xml_import(self,<EOL>xml_fname=None,<EOL>xml_content=None,<EOL>ns_mapping=None,<EOL>embedded_predicate=None,<EOL>id_and_revision_extractor=None,<EOL>extract_empty_embedded=False,<EOL>keep_attrs_in_created_reference=True,<EOL>transformer=None):
generated_id_count = {}<EOL>if not ns_mapping:<EOL><INDENT>nas_mapping = {}<EOL><DEDENT>if not transformer:<EOL><INDENT>transformer = lambda x, y: (x, y)<EOL><DEDENT>_import_pending_stack = deque()<EOL>embedded_objects = deque()<EOL>def xml_import_(element, depth=<NUM_LIT:0>, type_info=None, inherited_id_and_rev_info=None):<EOL><INDENT>"""<STR_LIT>"""<EOL>if not inherited_id_and_rev_info:<EOL><INDENT>inherited_id_and_rev_info = main_id_and_rev_info.copy()<EOL><DEDENT>fresh_inherited_id_and_rev_info = inherited_id_and_rev_info.copy()<EOL>if element.name == '<STR_LIT>':<EOL><INDENT>return None<EOL><DEDENT>result = DingoObjDict()<EOL>if element.properties:<EOL><INDENT>for prop in element.properties:<EOL><INDENT>if not prop:<EOL><INDENT>break<EOL><DEDENT>if prop.type == '<STR_LIT>':<EOL><INDENT>try:<EOL><INDENT>result["<STR_LIT>" % (prop.ns().name, prop.name)] = prop.content<EOL><DEDENT>except:<EOL><INDENT>result["<STR_LIT>" % prop.name] = prop.content<EOL><DEDENT><DEDENT><DEDENT><DEDENT>try:<EOL><INDENT>ns = element.ns().name<EOL>result["<STR_LIT>"] = ns<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>element_dicts = []<EOL>non_ws_content = False<EOL>cdata_content = False<EOL>name_set = {}<EOL>previous_seen_child = None<EOL>double_occurrance = False<EOL>element_child_count = <NUM_LIT:0><EOL>child = element.children<EOL>while child is not None:<EOL><INDENT>if child.name == '<STR_LIT:text>':<EOL><INDENT>content = child.content.strip()<EOL>if content != "<STR_LIT>":<EOL><INDENT>non_ws_content = True<EOL><DEDENT><DEDENT>elif child.type == '<STR_LIT>':<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>cdata_content = True<EOL><DEDENT>else:<EOL><INDENT>element_child_count += <NUM_LIT:1><EOL>if previous_seen_child and (child.name in name_set) and (not child.name == previous_seen_child):<EOL><INDENT>double_occurrance = True<EOL><DEDENT>name_set[child.name] = None<EOL>if embedded_predicate:<EOL><INDENT>embedded_ns = embedded_predicate(element, child, ns_mapping)<EOL>logger.debug("<STR_LIT>" % embedded_ns)<EOL>if embedded_ns:<EOL><INDENT>inherited_id_and_rev_info = fresh_inherited_id_and_rev_info.copy()<EOL>if type(embedded_ns) == type({}):<EOL><INDENT>id_and_revision_info = embedded_ns.get('<STR_LIT>',<EOL>id_and_revision_extractor(child))<EOL>embedded_ns = embedded_ns.get('<STR_LIT>',None)<EOL><DEDENT>else:<EOL><INDENT>id_and_revision_info = id_and_revision_extractor(child)<EOL><DEDENT>if not '<STR_LIT:id>' in id_and_revision_info or not id_and_revision_info['<STR_LIT:id>']:<EOL><INDENT>if '<STR_LIT:id>' in inherited_id_and_rev_info:<EOL><INDENT>parent_id = inherited_id_and_rev_info['<STR_LIT:id>']<EOL>if parent_id in generated_id_count:<EOL><INDENT>gen_counter = generated_id_count[parent_id]<EOL>gen_counter +=<NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>gen_counter = <NUM_LIT:0><EOL><DEDENT>generated_id_count[parent_id] = gen_counter<EOL>(parent_namespace, parent_uid) = parent_id.split('<STR_LIT::>')<EOL>generated_id = "<STR_LIT>" % (parent_namespace,gen_counter,parent_uid)<EOL>logger.info("<STR_LIT>" % (element.name,generated_id))<EOL>id_and_revision_info['<STR_LIT:id>'] = generated_id<EOL>id_and_revision_info['<STR_LIT>'] = True<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>" % elt_name)<EOL><DEDENT><DEDENT>if not id_and_revision_info.get('<STR_LIT>', None):<EOL><INDENT>if inherited_id_and_rev_info and '<STR_LIT>' in inherited_id_and_rev_info:<EOL><INDENT>id_and_revision_info['<STR_LIT>'] = inherited_id_and_rev_info['<STR_LIT>']<EOL>id_and_revision_info['<STR_LIT>'] = True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>inherited_id_and_rev_info['<STR_LIT>'] = id_and_revision_info['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT:id>' in id_and_revision_info:<EOL><INDENT>if (not '<STR_LIT::>' in id_and_revision_info['<STR_LIT:id>']<EOL>and inherited_id_and_rev_info['<STR_LIT:id>']<EOL>and '<STR_LIT::>' in inherited_id_and_rev_info['<STR_LIT:id>']):<EOL><INDENT>id_and_revision_info['<STR_LIT:id>'] = "<STR_LIT>" % (inherited_id_and_rev_info['<STR_LIT:id>'].split('<STR_LIT::>')[<NUM_LIT:0>],<EOL>id_and_revision_info['<STR_LIT:id>'])<EOL>id_and_revision_info['<STR_LIT>'] = True<EOL><DEDENT>inherited_id_and_rev_info['<STR_LIT:id>'] = id_and_revision_info['<STR_LIT:id>']<EOL><DEDENT>if keep_attrs_in_created_reference:<EOL><INDENT>reference_dict = extract_attributes(child, prefix_key_char='<STR_LIT:@>',<EOL>dict_constructor=DingoObjDict)<EOL><DEDENT>else:<EOL><INDENT>reference_dict = DingoObjDict()<EOL><DEDENT>reference_dict['<STR_LIT>'] = id_and_revision_info['<STR_LIT:id>']<EOL>reference_dict['<STR_LIT>'] = id_and_revision_info['<STR_LIT>']<EOL>try:<EOL><INDENT>reference_dict['<STR_LIT>'] = child.ns().name<EOL><DEDENT>except:<EOL><INDENT>reference_dict['<STR_LIT>'] = None<EOL><DEDENT>if embedded_ns == True:<EOL><INDENT>embedded_ns = None<EOL><DEDENT>logger.debug("<STR_LIT>" % embedded_ns)<EOL>reference_dict['<STR_LIT>'] = embedded_ns<EOL>element_dicts.append((child.name, reference_dict))<EOL>if (child.children or child.content)or extract_empty_embeddedor '<STR_LIT>' in id_and_revision_info:<EOL><INDENT>id_and_revision_info['<STR_LIT>'] = fresh_inherited_id_and_rev_info.copy()<EOL>if '<STR_LIT>' in id_and_revision_info['<STR_LIT>']:<EOL><INDENT>for key in id_and_revision_info['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>if not key in id_and_revision_info['<STR_LIT>']:<EOL><INDENT>id_and_revision_info['<STR_LIT>'][key] = id_and_revision_info['<STR_LIT>']['<STR_LIT>'][key]<EOL><DEDENT><DEDENT>del(id_and_revision_info['<STR_LIT>']['<STR_LIT>'])<EOL><DEDENT>logger.debug(<EOL>"<STR_LIT>" % (<EOL>id_and_revision_info, embedded_ns))<EOL>_import_pending_stack.append((id_and_revision_info, embedded_ns, child))<EOL><DEDENT>else:<EOL><INDENT>logger.info(<EOL>"<STR_LIT>" % (<EOL>id_and_revision_info, embedded_ns))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>child_import = xml_import_(child, depth + <NUM_LIT:1>, inherited_id_and_rev_info=inherited_id_and_rev_info)<EOL>if child_import:<EOL><INDENT>element_dicts.append(child_import)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>child_import = xml_import_(child, depth + <NUM_LIT:1>, inherited_id_and_rev_info=inherited_id_and_rev_info)<EOL>if child_import:<EOL><INDENT>element_dicts.append(child_import)<EOL><DEDENT><DEDENT><DEDENT>child = child.next<EOL><DEDENT>distinct_child_count = len(name_set.keys())<EOL>if distinct_child_count == <NUM_LIT:0>:<EOL><INDENT>result['<STR_LIT>'] = element.content<EOL>if cdata_content:<EOL><INDENT>result['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT><DEDENT>elif non_ws_content == True:<EOL><INDENT>sub_child = element.children<EOL>serialization = '<STR_LIT>'<EOL>while sub_child:<EOL><INDENT>serialization += sub_child.serialize()<EOL>sub_child = sub_child.next<EOL><DEDENT>result['<STR_LIT>'] = serialization.strip()<EOL>result['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>elif double_occurrance: <EOL><INDENT>logger.warning("<STR_LIT>" % (<EOL>element.name, name_set.keys(), element_child_count))<EOL>sub_child = element.children<EOL>serialization = '<STR_LIT>'<EOL>while sub_child:<EOL><INDENT>serialization += sub_child.serialize()<EOL>sub_child = sub_child.next<EOL><DEDENT>result['<STR_LIT>'] = serialization.strip()<EOL>result['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>previously_written_name = None<EOL>for (name, element_dict) in element_dicts:<EOL><INDENT>if not previously_written_name or name != previously_written_name:<EOL><INDENT>result[name] = element_dict<EOL>previously_written_name = name<EOL><DEDENT>else: <EOL><INDENT>if type(result[name]) == type([]):<EOL><INDENT>result[name].append(element_dict)<EOL><DEDENT>else:<EOL><INDENT>result[name] = [result[name], element_dict]<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if type_info:<EOL><INDENT>result['<STR_LIT>'] = type_info<EOL><DEDENT>element_ns = None<EOL>try:<EOL><INDENT>element_ns = element.ns().name<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>return transformer(element.name, result)<EOL><DEDENT>if xml_content:<EOL><INDENT>if isinstance(xml_content,libxml2.xmlNode):<EOL><INDENT>root = xml_content<EOL><DEDENT>else:<EOL><INDENT>doc = libxml2.parseDoc(xml_content)<EOL>root = doc.getRootElement()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>doc = libxml2.recoverFile(xml_fname)<EOL>root = doc.getRootElement()<EOL>with open(xml_fname, '<STR_LIT:r>') as content_file:<EOL><INDENT>xml_content = content_file.read()<EOL><DEDENT><DEDENT>try:<EOL><INDENT>ns_def = root.nsDefs()<EOL>while ns_def:<EOL><INDENT>ns_mapping[ns_def.name] = ns_def.content<EOL>ns_def = ns_def.next<EOL><DEDENT><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>main_id_and_rev_info = id_and_revision_extractor(root)<EOL>(main_elt_name, main_elt_dict) = xml_import_(root, <NUM_LIT:0>)<EOL>do_not_process_list = []<EOL>while _import_pending_stack:<EOL><INDENT>(id_and_revision_info, type_info, elt) = _import_pending_stack.pop()<EOL>if '<STR_LIT>' in id_and_revision_info:<EOL><INDENT>do_not_process_list.append((id_and_revision_info,type_info,elt))<EOL><DEDENT>else:<EOL><INDENT>(elt_name, elt_dict) = xml_import_(elt, <NUM_LIT:0>,<EOL>type_info=type_info,<EOL>inherited_id_and_rev_info=id_and_revision_info.copy())<EOL>embedded_objects.append({'<STR_LIT>': id_and_revision_info,<EOL>'<STR_LIT>': elt_name,<EOL>'<STR_LIT>': elt_dict})<EOL><DEDENT><DEDENT>result= {'<STR_LIT>': main_id_and_rev_info,<EOL>'<STR_LIT>': main_elt_name,<EOL>'<STR_LIT>': main_elt_dict,<EOL>'<STR_LIT>': embedded_objects,<EOL>'<STR_LIT>' : do_not_process_list,<EOL>'<STR_LIT>': xml_content}<EOL>return result<EOL>
This is the generic XML import function for dingos. Its parameters
are as follows:

- xml_fname: Filename of the XML file to be read
- xml_content: Alternatively, the xml_content can be provided as string
  or as XMLNode (i.e., a piece of XML that has already been parsed)
- ns_mapping: A dictionary that may already contain mappings of namespaces
  to namespace URIs. Attention: this dictionary will be enriched with namespace
  information found in the XML file!!!
- embedded_predicate:
  A function that, when given an XML node and a child node, determines whether
  the child node should be treated as separate entity that has been embedded.
  Please refer to existing import modules such as for STIX or OpenIOC for
  examples of how to use this parameter.

- id_and_revision_extractor:
  A function that, when given an XML node and a child node, determines whether
  this node specifies an identifier and possibly a timestamp.
  Please refer to existing import modules such as for STIX or OpenIOC for
  examples of how to use this parameter.

- extract_empty_embedded:
  A flag (True/False) governing whether elements that are recognized as
  being embedded but contain no childs should be extracted as separate
  object or not. The default is False; the setting "True" may be necessary
  in cases where there are embedded objects that contain all its information
  in attributes rather than using child elements.

- keep_attrs_in_created_reference:
  A flag (True/False) governing the shape of the reference created for
  an embedded object: when an embedding is recognized, it is extracted
  and a reference using 'idref' inserted instead. If 'keep_attrs_in_created_reference'
  is True, then the top-level attributes contained in the found embedded object
  are also retained in the reference.


- transformer:
  A function that, when given an element name and a DingoObjDict containing
  the result of importing the contents under the element of the given name,
  may or may not change the element name and transform the DingoObjDict.
  Please refer to existing import MANTIS modules such as for OpenIOC for
  examples of how to use this parameter.


Note: a good starting point for understanding how to use the python bindings
of libxml2 is http://mikekneller.com/kb/python/libxml2python/part1.