def _get_tokens(morph_fd):
for line in morph_fd:<EOL><INDENT>line = line.rstrip()   <EOL>line = line.split('<STR_LIT:;>', <NUM_LIT:1>)[<NUM_LIT:0>]  <EOL>squash_token = []  <EOL>if '<STR_LIT>' in line:  <EOL><INDENT>assert '<STR_LIT>' in line, '<STR_LIT>'<EOL>continue<EOL><DEDENT>for token in line.replace('<STR_LIT:(>', '<STR_LIT>').replace('<STR_LIT:)>', '<STR_LIT>').split():<EOL><INDENT>if squash_token:<EOL><INDENT>squash_token.append(token)<EOL>if token.endswith('<STR_LIT:">'):<EOL><INDENT>token = '<STR_LIT:U+0020>'.join(squash_token)<EOL>squash_token = []<EOL>yield token<EOL><DEDENT><DEDENT>elif token.startswith('<STR_LIT:">') and not token.endswith('<STR_LIT:">'):<EOL><INDENT>squash_token.append(token)<EOL><DEDENT>else:<EOL><INDENT>yield token<EOL><DEDENT><DEDENT><DEDENT>
split a file-like into tokens: split on whitespace

    Note: this also strips newlines and comments