def _parse_tokens(chunk, format=[WORD, POS, CHUNK, PNP, REL, ANCHOR, LEMMA]):
tokens = []<EOL>if not (chunk.tag == XML_CHUNK or chunk.tag == XML_CHINK):<EOL><INDENT>return []<EOL><DEDENT>type = chunk.get(XML_TYPE, "<STR_LIT:O>")<EOL>if type == "<STR_LIT>":<EOL><INDENT>for ch in chunk:<EOL><INDENT>tokens.extend(_parse_tokens(ch, format))<EOL><DEDENT>if PNP in format:<EOL><INDENT>i = format.index(PNP)<EOL>for j, token in enumerate(tokens):<EOL><INDENT>token[i] = (j==<NUM_LIT:0> and "<STR_LIT>" or "<STR_LIT>") + "<STR_LIT>"<EOL><DEDENT><DEDENT>a = chunk.get(XML_OF).split(_UID_SEPARATOR)[-<NUM_LIT:1>]<EOL>if a:<EOL><INDENT>_attachments.setdefault(a, [])<EOL>_attachments[a].append(tokens)<EOL><DEDENT>return tokens<EOL><DEDENT>relation = _parse_relation(chunk, type)<EOL>for word in filter(lambda n: n.tag == XML_WORD, chunk):<EOL><INDENT>tokens.append(_parse_token(word, chunk=type, relation=relation, format=format))<EOL><DEDENT>if CHUNK in format:<EOL><INDENT>i = format.index(CHUNK)<EOL>for j, token in enumerate(tokens):<EOL><INDENT>token[i] = token[i] != "<STR_LIT:O>" and ((j==<NUM_LIT:0> and "<STR_LIT>" or "<STR_LIT>") + token[i]) or "<STR_LIT:O>"<EOL><DEDENT><DEDENT>a = chunk.get(XML_ANCHOR, "<STR_LIT>").split(_UID_SEPARATOR)[-<NUM_LIT:1>]<EOL>if a: <EOL><INDENT>_anchors[a] = tokens<EOL><DEDENT>return tokens<EOL>
Parses tokens from <word> elements in the given XML <chunk> element.
        Returns a flat list of tokens, in which each token is [WORD, POS, CHUNK, PNP, RELATION, ANCHOR, LEMMA].
        If a <chunk type="PNP"> is encountered, traverses all of the chunks in the PNP.