@staticmethod<EOL><INDENT>def clean_query(query, replace=chr(<NUM_LIT>)):<DEDENT>
accum = []<EOL>any_invalid = False<EOL>tokens = _quote_re.findall(query)<EOL>for token in tokens:<EOL><INDENT>if token.startswith('<STR_LIT:">') and token.endswith('<STR_LIT:">'):<EOL><INDENT>accum.append(token)<EOL>continue<EOL><DEDENT>token_set = set(token)<EOL>invalid_for_token = token_set & _invalid_ascii<EOL>if invalid_for_token:<EOL><INDENT>any_invalid = True<EOL>for c in invalid_for_token:<EOL><INDENT>token = token.replace(c, replace)<EOL><DEDENT><DEDENT>accum.append(token)<EOL><DEDENT>if any_invalid:<EOL><INDENT>return '<STR_LIT:U+0020>'.join(accum)<EOL><DEDENT>return query<EOL>
Clean a query of invalid tokens.