def token(self):
if self.tokens_queue:<EOL><INDENT>self.last_token = self.tokens_queue.pop(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>r = self.lex.token()<EOL>if isinstance(r, MultiToken):<EOL><INDENT>self.tokens_queue.extend(r.tokens)<EOL>self.last_token = self.tokens_queue.pop(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>if r is None and self.cur_indent > <NUM_LIT:0>:<EOL><INDENT>if (self.last_token and<EOL>self.last_token.type not in ('<STR_LIT>', '<STR_LIT>')):<EOL><INDENT>newline_token = _create_token(<EOL>'<STR_LIT>', '<STR_LIT:\n>', self.lex.lineno, self.lex.lexpos)<EOL>self.tokens_queue.append(newline_token)<EOL><DEDENT>dedent_count = self.cur_indent<EOL>dedent_token = _create_token(<EOL>'<STR_LIT>', '<STR_LIT:\t>', self.lex.lineno, self.lex.lexpos)<EOL>self.tokens_queue.extend([dedent_token] * dedent_count)<EOL>self.cur_indent = <NUM_LIT:0><EOL>self.last_token = self.tokens_queue.pop(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>self.last_token = r<EOL><DEDENT><DEDENT><DEDENT>return self.last_token<EOL>
Returns the next LexToken. Returns None when all tokens have been
exhausted.