def detailed_tokens(tokenizer, text):
node = tokenizer.parseToNode(text)<EOL>node = node.next  <EOL>words = []<EOL>while node.posid != <NUM_LIT:0>:<EOL><INDENT>surface = node.surface<EOL>base = surface  <EOL>parts = node.feature.split("<STR_LIT:U+002C>")<EOL>pos = "<STR_LIT:U+002C>".join(parts[<NUM_LIT:0>:<NUM_LIT:4>])<EOL>if len(parts) > <NUM_LIT:7>:<EOL><INDENT>base = parts[<NUM_LIT:7>]<EOL><DEDENT>words.append(ShortUnitWord(surface, base, pos))<EOL>node = node.next<EOL><DEDENT>return words<EOL>
Format Mecab output into a nice data structure, based on Janome.