def get_tokens(self, text, unfiltered=False):
if not isinstance(text, str):<EOL><INDENT>if self.encoding == '<STR_LIT>':<EOL><INDENT>try:<EOL><INDENT>text = text.decode('<STR_LIT:utf-8>')<EOL>if text.startswith('<STR_LIT>'):<EOL><INDENT>text = text[len('<STR_LIT>'):]<EOL><DEDENT><DEDENT>except UnicodeDecodeError:<EOL><INDENT>text = text.decode('<STR_LIT>')<EOL><DEDENT><DEDENT>elif self.encoding == '<STR_LIT>':<EOL><INDENT>try:<EOL><INDENT>import chardet<EOL><DEDENT>except ImportError:<EOL><INDENT>raise ImportError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>enc = chardet.detect(text)<EOL>text = text.decode(enc['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>text = text.decode(self.encoding)<EOL><DEDENT><DEDENT>text = text.replace('<STR_LIT:\r\n>', '<STR_LIT:\n>')<EOL>text = text.replace('<STR_LIT:\r>', '<STR_LIT:\n>')<EOL>if self.stripall:<EOL><INDENT>text = text.strip()<EOL><DEDENT>elif self.stripnl:<EOL><INDENT>text = text.strip('<STR_LIT:\n>')<EOL><DEDENT>if self.tabsize > <NUM_LIT:0>:<EOL><INDENT>text = text.expandtabs(self.tabsize)<EOL><DEDENT>if self.ensurenl and not text.endswith('<STR_LIT:\n>'):<EOL><INDENT>text += '<STR_LIT:\n>'<EOL><DEDENT>def streamer():<EOL><INDENT>for i, t, v in self.get_tokens_unprocessed(text):<EOL><INDENT>yield t, v<EOL><DEDENT><DEDENT>stream = streamer()<EOL>if not unfiltered:<EOL><INDENT>stream = apply_filters(stream, self.filters, self)<EOL><DEDENT>return stream<EOL>
Return an iterable of (tokentype, value) pairs generated from
`text`. If `unfiltered` is set to `True`, the filtering mechanism
is bypassed even if filters are defined.

Also preprocess the text, i.e. expand tabs and strip it if
wanted and applies registered filters.