def tokenize(self, config):
tokens = []<EOL>reg_ex = re.compile(self.TOKENS[<NUM_LIT:0>], re.M | re.I)<EOL>for token in re.finditer(reg_ex, config):<EOL><INDENT>value = token.group(<NUM_LIT:0>)<EOL>if token.group("<STR_LIT>"):<EOL><INDENT>t_type = "<STR_LIT>"<EOL><DEDENT>elif token.group("<STR_LIT>"):<EOL><INDENT>t_type = "<STR_LIT>"<EOL><DEDENT>elif token.group("<STR_LIT>"):<EOL><INDENT>t_type = "<STR_LIT>"<EOL><DEDENT>elif token.group("<STR_LIT>"):<EOL><INDENT>t_type = "<STR_LIT>"<EOL><DEDENT>elif token.group("<STR_LIT>"):<EOL><INDENT>t_type = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>continue<EOL><DEDENT>tokens.append(<EOL>{"<STR_LIT:type>": t_type, "<STR_LIT:value>": value, "<STR_LIT>": token, "<STR_LIT:start>": token.start()}<EOL>)<EOL><DEDENT>self.tokens = tokens<EOL>
Break the config into a series of tokens