def get_tokens_unprocessed(self, text, stack=('<STR_LIT:root>',)):
pos = <NUM_LIT:0><EOL>tokendefs = self._tokens<EOL>statestack = list(stack)<EOL>statetokens = tokendefs[statestack[-<NUM_LIT:1>]]<EOL>while <NUM_LIT:1>:<EOL><INDENT>for rexmatch, action, new_state in statetokens:<EOL><INDENT>m = rexmatch(text, pos)<EOL>if m:<EOL><INDENT>if type(action) is _TokenType:<EOL><INDENT>yield pos, action, m.group()<EOL><DEDENT>else:<EOL><INDENT>for item in action(self, m):<EOL><INDENT>yield item<EOL><DEDENT><DEDENT>pos = m.end()<EOL>if new_state is not None:<EOL><INDENT>if isinstance(new_state, tuple):<EOL><INDENT>for state in new_state:<EOL><INDENT>if state == '<STR_LIT:#pop>':<EOL><INDENT>statestack.pop()<EOL><DEDENT>elif state == '<STR_LIT>':<EOL><INDENT>statestack.append(statestack[-<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>statestack.append(state)<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(new_state, int):<EOL><INDENT>del statestack[new_state:]<EOL><DEDENT>elif new_state == '<STR_LIT>':<EOL><INDENT>statestack.append(statestack[-<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>assert False, "<STR_LIT>" % new_state<EOL><DEDENT>statetokens = tokendefs[statestack[-<NUM_LIT:1>]]<EOL><DEDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>if text[pos] == '<STR_LIT:\n>':<EOL><INDENT>pos += <NUM_LIT:1><EOL>statestack = ['<STR_LIT:root>']<EOL>statetokens = tokendefs['<STR_LIT:root>']<EOL>yield pos, Text, '<STR_LIT:\n>'<EOL>continue<EOL><DEDENT>yield pos, Error, text[pos]<EOL>pos += <NUM_LIT:1><EOL><DEDENT>except IndexError:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>
Split ``text`` into (tokentype, text) pairs.

``stack`` is the inital stack (default: ``['root']``)