def _tokenize(self, text):
text = self._clean_text(text)<EOL>text = self._tokenize_chinese_chars(text)<EOL>orig_tokens = self._whitespace_tokenize(text)<EOL>split_tokens = []<EOL>for token in orig_tokens:<EOL><INDENT>if self.lower:<EOL><INDENT>token = token.lower()<EOL>token = self._run_strip_accents(token)<EOL><DEDENT>split_tokens.extend(self._run_split_on_punc(token))<EOL><DEDENT>output_tokens = self._whitespace_tokenize('<STR_LIT:U+0020>'.join(split_tokens))<EOL>return output_tokens<EOL>
Tokenizes a piece of text.