@token('<STR_LIT:string>')<EOL>def tokenize_string(text, delimiters=None):
if delimiters is None:<EOL><INDENT>delimiters = ALL_TOKENS<EOL><DEDENT>result = TokenWithPosition('<STR_LIT>', text.position)<EOL>for c in text:<EOL><INDENT>if c == '<STR_LIT:\\>' and str(text.peek()) in delimiters and str(c + text.peek()) not in delimiters:<EOL><INDENT>c += next(text)<EOL><DEDENT>elif str(c) in delimiters:  <EOL><INDENT>text.backward(<NUM_LIT:1>)<EOL>return result<EOL><DEDENT>result += c<EOL>if text.peek((<NUM_LIT:0>, <NUM_LIT:2>)) == '<STR_LIT>':<EOL><INDENT>result += text.forward(<NUM_LIT:2>)<EOL><DEDENT>if text.peek((<NUM_LIT:0>, <NUM_LIT:2>)) == '<STR_LIT>':<EOL><INDENT>result += text.forward(<NUM_LIT:2>)<EOL>return result<EOL><DEDENT><DEDENT>return result<EOL>
r"""Process a string of text

    :param Buffer text: iterator over line, with current position
    :param Union[None,iterable,str] delimiters: defines the delimiters

    >>> tokenize_string(Buffer('hello'))
    'hello'
    >>> b = Buffer(r'hello again\command')
    >>> tokenize_string(b)
    'hello again'
    >>> print(b.peek())
    \
    >>> print(tokenize_string(Buffer(r'0 & 1 \\\command')))
    0 & 1 \\