def get_tokens_unprocessed(self, text=None, context=None):
tokendefs = self._tokens<EOL>if not context:<EOL><INDENT>ctx = LexerContext(text, <NUM_LIT:0>)<EOL>statetokens = tokendefs['<STR_LIT:root>']<EOL><DEDENT>else:<EOL><INDENT>ctx = context<EOL>statetokens = tokendefs[ctx.stack[-<NUM_LIT:1>]]<EOL>text = ctx.text<EOL><DEDENT>while <NUM_LIT:1>:<EOL><INDENT>for rexmatch, action, new_state in statetokens:<EOL><INDENT>m = rexmatch(text, ctx.pos, ctx.end)<EOL>if m:<EOL><INDENT>if type(action) is _TokenType:<EOL><INDENT>yield ctx.pos, action, m.group()<EOL>ctx.pos = m.end()<EOL><DEDENT>else:<EOL><INDENT>for item in action(self, m, ctx):<EOL><INDENT>yield item<EOL><DEDENT>if not new_state:<EOL><INDENT>statetokens = tokendefs[ctx.stack[-<NUM_LIT:1>]]<EOL><DEDENT><DEDENT>if new_state is not None:<EOL><INDENT>if isinstance(new_state, tuple):<EOL><INDENT>ctx.stack.extend(new_state)<EOL><DEDENT>elif isinstance(new_state, int):<EOL><INDENT>del ctx.stack[new_state:]<EOL><DEDENT>elif new_state == '<STR_LIT>':<EOL><INDENT>ctx.stack.append(ctx.stack[-<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>assert False, "<STR_LIT>" % new_state<EOL><DEDENT>statetokens = tokendefs[ctx.stack[-<NUM_LIT:1>]]<EOL><DEDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>if ctx.pos >= ctx.end:<EOL><INDENT>break<EOL><DEDENT>if text[ctx.pos] == '<STR_LIT:\n>':<EOL><INDENT>ctx.pos += <NUM_LIT:1><EOL>ctx.stack = ['<STR_LIT:root>']<EOL>statetokens = tokendefs['<STR_LIT:root>']<EOL>yield ctx.pos, Text, '<STR_LIT:\n>'<EOL>continue<EOL><DEDENT>yield ctx.pos, Error, text[ctx.pos]<EOL>ctx.pos += <NUM_LIT:1><EOL><DEDENT>except IndexError:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>
Split ``text`` into (tokentype, text) pairs.
If ``context`` is given, use this lexer context instead.