def get_tokens(source):
if not source.endswith('<STR_LIT:\n>'):<EOL><INDENT>source += '<STR_LIT:\n>'<EOL><DEDENT>valid_identifier_first_chars = VALID_IDENTIFIER_FIRST_CHARS<EOL>valid_identifier_chars = VALID_IDENTIFIER_CHARS<EOL>hex_digits = HEX_DIGITS<EOL>int_or_float_digits = INT_OR_FLOAT_DIGITS<EOL>int_or_float_digits2 = int_or_float_digits | set('<STR_LIT:.>')<EOL>count_ifs = <NUM_LIT:0><EOL>i = <NUM_LIT:0><EOL>end = len(source)<EOL>while i < end:<EOL><INDENT>while i < end and source[i].isspace():<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT>if i >= end:<EOL><INDENT>return<EOL><DEDENT>token_type = UNKNOWN<EOL>start = i<EOL>c = source[i]<EOL>if c in valid_identifier_first_chars or c == '<STR_LIT:_>':<EOL><INDENT>token_type = NAME<EOL>while source[i] in valid_identifier_chars:<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT>if source[i] == "<STR_LIT:'>" and source[start:i] in _STR_PREFIXES:<EOL><INDENT>token_type = CONSTANT<EOL>i = _get_char(source, start, i)<EOL><DEDENT>elif source[i] == '<STR_LIT:">' and source[start:i] in _STR_PREFIXES:<EOL><INDENT>token_type = CONSTANT<EOL>i = _get_string(source, i)<EOL><DEDENT><DEDENT>elif c == '<STR_LIT:/>' and source[i + <NUM_LIT:1>] == '<STR_LIT:/>':  <EOL><INDENT>i = _find(source, '<STR_LIT:\n>', i)<EOL>continue<EOL><DEDENT>elif c == '<STR_LIT:/>' and source[i + <NUM_LIT:1>] == '<STR_LIT:*>':  <EOL><INDENT>i = _find(source, '<STR_LIT>', i) + <NUM_LIT:2><EOL>continue<EOL><DEDENT>elif c in '<STR_LIT>':                          <EOL><INDENT>token_type = SYNTAX<EOL>i += <NUM_LIT:1><EOL>new_ch = source[i]<EOL>if new_ch == c and c != '<STR_LIT:>>':<EOL><INDENT>i += <NUM_LIT:1><EOL>new_ch = source[i]<EOL><DEDENT>if new_ch == '<STR_LIT:=>':<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT><DEDENT>elif c in '<STR_LIT>':                      <EOL><INDENT>token_type = SYNTAX<EOL>i += <NUM_LIT:1><EOL>new_ch = source[i]<EOL>if new_ch == c:<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT>elif c == '<STR_LIT:->' and new_ch == '<STR_LIT:>>':<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT>elif new_ch == '<STR_LIT:=>':<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT><DEDENT>elif c in '<STR_LIT>':                       <EOL><INDENT>token_type = SYNTAX<EOL>i += <NUM_LIT:1><EOL>new_ch = source[i]<EOL>if new_ch == '<STR_LIT:=>':<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT><DEDENT>elif c in '<STR_LIT>':                 <EOL><INDENT>token_type = SYNTAX<EOL>i += <NUM_LIT:1><EOL>if c == '<STR_LIT:.>' and source[i].isdigit():<EOL><INDENT>token_type = CONSTANT<EOL>i += <NUM_LIT:1><EOL>while source[i] in int_or_float_digits:<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT>for suffix in ('<STR_LIT:l>', '<STR_LIT:f>'):<EOL><INDENT>if suffix == source[i:i + <NUM_LIT:1>].lower():<EOL><INDENT>i += <NUM_LIT:1><EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif c.isdigit():                        <EOL><INDENT>token_type = CONSTANT<EOL>if c == '<STR_LIT:0>' and source[i + <NUM_LIT:1>] in '<STR_LIT>':<EOL><INDENT>i += <NUM_LIT:2><EOL>while source[i] in hex_digits:<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>while source[i] in int_or_float_digits2:<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT><DEDENT>if source[i].isalpha():<EOL><INDENT>for suffix in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:l>', '<STR_LIT:f>', '<STR_LIT:u>'):<EOL><INDENT>size = len(suffix)<EOL>if suffix == source[i:i + size].lower():<EOL><INDENT>i += size<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif c == '<STR_LIT:">':                           <EOL><INDENT>token_type = CONSTANT<EOL>i = _get_string(source, i)<EOL><DEDENT>elif c == "<STR_LIT:'>":                           <EOL><INDENT>token_type = CONSTANT<EOL>i = _get_char(source, start, i)<EOL><DEDENT>elif c == '<STR_LIT:#>':                           <EOL><INDENT>token_type = PREPROCESSOR<EOL>got_if = source[i:i + <NUM_LIT:3>] == '<STR_LIT>'<EOL>if count_ifs and source[i:i + <NUM_LIT:6>] == '<STR_LIT>':<EOL><INDENT>count_ifs -= <NUM_LIT:1><EOL>if count_ifs == <NUM_LIT:0>:<EOL><INDENT>source = source[:i].ljust(i + <NUM_LIT:6>) + source[i + <NUM_LIT:6>:]<EOL>continue<EOL><DEDENT><DEDENT>while True:<EOL><INDENT>i1 = source.find('<STR_LIT:\n>', i)<EOL>i2 = source.find('<STR_LIT>', i)<EOL>i3 = source.find('<STR_LIT>', i)<EOL>i4 = source.find('<STR_LIT:">', i)<EOL>i = min([x for x in (i1, i2, i3, i4, end) if x != -<NUM_LIT:1>])<EOL>if i == i3:<EOL><INDENT>i = _find(source, '<STR_LIT>', i) + <NUM_LIT:2><EOL>source = source[:i3].ljust(i) + source[i:]<EOL>continue<EOL><DEDENT>if source[i] == '<STR_LIT:">':<EOL><INDENT>i = _find(source, '<STR_LIT:">', i + <NUM_LIT:1>) + <NUM_LIT:1><EOL>continue<EOL><DEDENT>if i == i1 and source[i - <NUM_LIT:1>] == '<STR_LIT:\\>':<EOL><INDENT>i += <NUM_LIT:1><EOL>continue<EOL><DEDENT>if got_if:<EOL><INDENT>begin = source.find('<STR_LIT:(>', start, i)<EOL>if begin == -<NUM_LIT:1>:<EOL><INDENT>begin = source.find('<STR_LIT:U+0020>', start)<EOL><DEDENT>begin = begin + <NUM_LIT:1><EOL>s1 = source.find('<STR_LIT:U+0020>', begin)<EOL>s2 = source.find('<STR_LIT:)>', begin)<EOL>s3 = source.find('<STR_LIT:\n>', begin)<EOL>s = min([x for x in (s1, s2, s3, end) if x != -<NUM_LIT:1>])<EOL>condition = source[begin:s]<EOL>if (<EOL>count_ifs or<EOL>condition == '<STR_LIT:0>' or<EOL>condition == '<STR_LIT>'<EOL>):<EOL><INDENT>count_ifs += <NUM_LIT:1><EOL><DEDENT><DEDENT>break<EOL><DEDENT><DEDENT>elif c == '<STR_LIT:\\>':                          <EOL><INDENT>i += <NUM_LIT:1><EOL>continue<EOL><DEDENT>elif count_ifs:<EOL><INDENT>i += <NUM_LIT:1><EOL>continue<EOL><DEDENT>else:<EOL><INDENT>raise TokenError("<STR_LIT>".format(c))<EOL><DEDENT>if count_ifs:<EOL><INDENT>continue<EOL><DEDENT>assert i > <NUM_LIT:0><EOL>yield Token(token_type, source[start:i], start, i)<EOL><DEDENT>
Returns a sequence of Tokens.

    Args:
      source: string of C++ source code.

    Yields:
      Token that represents the next token in the source.