def find_tokens(string, punctuation=PUNCTUATION, abbreviations=ABBREVIATIONS, replace=replacements, linebreak=r"<STR_LIT>"):
<EOL>punctuation = tuple(punctuation.replace("<STR_LIT:.>", "<STR_LIT>"))<EOL>for a, b in replace.items():<EOL><INDENT>string = re.sub(a, b, string)<EOL><DEDENT>if isinstance(string, unicode):<EOL><INDENT>string = string.replace(u"<STR_LIT>", u"<STR_LIT>")<EOL>string = string.replace(u"<STR_LIT>", u"<STR_LIT>")<EOL>string = string.replace(u"<STR_LIT>", u"<STR_LIT>")<EOL>string = string.replace(u"<STR_LIT>", u"<STR_LIT>")<EOL><DEDENT>string = re.sub("<STR_LIT:\r\n>", "<STR_LIT:\n>", string)<EOL>string = re.sub(linebreak, "<STR_LIT>" % EOS, string)<EOL>string = re.sub(r"<STR_LIT>", "<STR_LIT:U+0020>", string)<EOL>tokens = []<EOL>for t in TOKEN.findall(string+"<STR_LIT:U+0020>"):<EOL><INDENT>if len(t) > <NUM_LIT:0>:<EOL><INDENT>tail = []<EOL>while t.startswith(punctuation) andnot t in replace:<EOL><INDENT>if t.startswith(punctuation):<EOL><INDENT>tokens.append(t[<NUM_LIT:0>]); t=t[<NUM_LIT:1>:]<EOL><DEDENT><DEDENT>while t.endswith(punctuation+("<STR_LIT:.>",)) andnot t in replace:<EOL><INDENT>if t.endswith(punctuation):<EOL><INDENT>tail.append(t[-<NUM_LIT:1>]); t=t[:-<NUM_LIT:1>]<EOL><DEDENT>if t.endswith("<STR_LIT>"):<EOL><INDENT>tail.append("<STR_LIT>"); t=t[:-<NUM_LIT:3>].rstrip("<STR_LIT:.>")<EOL><DEDENT>if t.endswith("<STR_LIT:.>"):<EOL><INDENT>if t in abbreviations orRE_ABBR1.match(t) is not None orRE_ABBR2.match(t) is not None orRE_ABBR3.match(t) is not None:<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>tail.append(t[-<NUM_LIT:1>]); t=t[:-<NUM_LIT:1>]<EOL><DEDENT><DEDENT><DEDENT>if t != "<STR_LIT>":<EOL><INDENT>tokens.append(t)<EOL><DEDENT>tokens.extend(reversed(tail))<EOL><DEDENT><DEDENT>sentences, i, j = [[]], <NUM_LIT:0>, <NUM_LIT:0><EOL>while j < len(tokens):<EOL><INDENT>if tokens[j] in ("<STR_LIT>", "<STR_LIT:.>", "<STR_LIT:!>", "<STR_LIT:?>", EOS):<EOL><INDENT>while j < len(tokens)and tokens[j] in ("<STR_LIT:'>", "<STR_LIT>", u"<STR_LIT>", u"<STR_LIT>", "<STR_LIT>", "<STR_LIT:.>", "<STR_LIT:!>", "<STR_LIT:?>", "<STR_LIT:)>", EOS):<EOL><INDENT>if tokens[j] in ("<STR_LIT:'>", "<STR_LIT>") and sentences[-<NUM_LIT:1>].count(tokens[j]) % <NUM_LIT:2> == <NUM_LIT:0>:<EOL><INDENT>break <EOL><DEDENT>j += <NUM_LIT:1><EOL><DEDENT>sentences[-<NUM_LIT:1>].extend(t for t in tokens[i:j] if t != EOS)<EOL>sentences.append([])<EOL>i = j<EOL><DEDENT>j += <NUM_LIT:1><EOL><DEDENT>sentences[-<NUM_LIT:1>].extend(tokens[i:j])<EOL>sentences = ("<STR_LIT:U+0020>".join(s) for s in sentences if len(s) > <NUM_LIT:0>)<EOL>sentences = (RE_SARCASM.sub("<STR_LIT>", s) for s in sentences)<EOL>sentences = [RE_EMOTICONS.sub(<EOL>lambda m: m.group(<NUM_LIT:1>).replace("<STR_LIT:U+0020>", "<STR_LIT>") + m.group(<NUM_LIT:2>), s) for s in sentences]<EOL>return sentences<EOL>
Returns a list of sentences. Each sentence is a space-separated string of tokens (words).
        Handles common cases of abbreviations (e.g., etc., ...).
        Punctuation marks are split from other words. Periods (or ?!) mark the end of a sentence.
        Headings without an ending period are inferred by line breaks.