def get_words(s, splitter_regex=rex.word_sep_except_external_appostrophe,<EOL>preprocessor=strip_HTML, postprocessor=strip_edge_punc, min_len=None,<EOL>max_len=None, blacklist=None, whitelist=None, lower=False,<EOL>filter_fun=None, str_type=str):
<EOL>postprocessor = postprocessor or str_type<EOL>preprocessor = preprocessor or str_type<EOL>if min_len is None:<EOL><INDENT>min_len = get_words.min_len<EOL><DEDENT>if max_len is None:<EOL><INDENT>max_len = get_words.max_len<EOL><DEDENT>blacklist = blacklist or get_words.blacklist<EOL>whitelist = whitelist or get_words.whitelist<EOL>filter_fun = filter_fun or get_words.filter_fun<EOL>lower = lower or get_words.lower<EOL>try:<EOL><INDENT>s = open(s, '<STR_LIT:r>')<EOL><DEDENT>except (IOError, FileNotFoundError):<EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>s = s.read()<EOL><DEDENT>except (IOError, AttributeError, TypeError):<EOL><INDENT>pass<EOL><DEDENT>if not isinstance(s, basestring):<EOL><INDENT>try:<EOL><INDENT>return [word for obj in s for word in get_words(obj)]<EOL><DEDENT>except (IOError, IndexError, ValueError, AttributeError, TypeError):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>try:<EOL><INDENT>s = preprocessor(s)<EOL><DEDENT>except (IndexError, ValueError, AttributeError, TypeError):<EOL><INDENT>pass<EOL><DEDENT>if isinstance(splitter_regex, basestring):<EOL><INDENT>splitter_regex = re.compile(splitter_regex)<EOL><DEDENT>s = list(map(postprocessor, splitter_regex.split(s)))<EOL>s = list(map(str_type, s))<EOL>if not filter_fun:<EOL><INDENT>return s<EOL><DEDENT>return [word for word in s if<EOL>filter_fun(word, min_len=min_len, max_len=max_len,<EOL>blacklist=blacklist, whitelist=whitelist, lower=lower)]<EOL>
r"""Segment words (tokens), returning a list of all tokens

    Does not return any separating whitespace or punctuation marks.
    Attempts to return external apostrophes at the end of words.
    Comparable to `nltk.word_toeknize`.

    Arguments:
      splitter_regex (str or re): compiled or uncompiled regular expression
        Applied to the input string using `re.split()`
      preprocessor (function): defaults to a function that strips out all HTML tags
      postprocessor (function): a function to apply to each token before return it as an element in the word list
        Applied using the `map()` builtin
      min_len (int): delete all words shorter than this number of characters
      max_len (int): delete all words longer than this number of characters
      blacklist and whitelist (list of str): words to delete or preserve
      lower (bool): whether to convert all words to lowercase
      str_type (type): typically `str` or `unicode`, any type constructor that should can be applied
        to all words before returning the list

    Returns:
      list of str: list of tokens

    >>> get_words('He said, "She called me \'Hoss\'!". I didn\'t hear.')
    ['He', 'said', 'She', 'called', 'me', 'Hoss', 'I', "didn't", 'hear']
    >>> get_words('The foxes\' oh-so-tiny den was 2empty!')
    ['The', 'foxes', 'oh-so-tiny', 'den', 'was', '2empty']