def __iter__(self, ngrams=None):
ngrams = ngrams or self.ngrams<EOL>if ngrams > <NUM_LIT:1>:<EOL><INDENT>original_tokens = list(self.__iter__(ngrams=<NUM_LIT:1>))<EOL>for tok in original_tokens:<EOL><INDENT>yield tok<EOL><DEDENT>for i in range(<NUM_LIT:2>, ngrams + <NUM_LIT:1>):<EOL><INDENT>for tok in list_ngrams(original_tokens, n=i, join='<STR_LIT:U+0020>'):<EOL><INDENT>yield tok<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for w in self.regex.finditer(self.doc):<EOL><INDENT>if w:<EOL><INDENT>w = w.group()<EOL>w = w if not self.strip_chars else str.strip(w, self.strip_chars)<EOL>w = w if not self.strip else self.strip(w)<EOL>w = w if not self.stem else self.stem(w)<EOL>w = w if not self.lemmatize else self.lemmatize(w)<EOL>w = w if not self.lower else self.lower(w)<EOL>if w and (not self.nonwords or not re.match(r'<STR_LIT>' + RE_NONWORD + '<STR_LIT:$>', w)) and (<EOL>not self.nonwords_regex or not self.nonwords_regex.match(w)) and (<EOL>w not in self.nonwords_set):<EOL><INDENT>yield w<EOL><DEDENT><DEDENT><DEDENT><DEDENT>
r"""Generate a sequence of words or tokens, using a re.match iteratively through the str

        TODO:
          - need two different self.lower and lemmatize transforms, 1 before and 1 after nonword detection
          - each of 3 nonword filters on a separate line, setting w=None when nonword "hits"
          - refactor `nonwords` arg/attr to `ignore_stopwords` to be more explicit

        >>> doc = ("John D. Rock\n\nObjective: \n\tSeeking a position as Software --Architect-- / " +
        ...        "_Project Lead_ that can utilize my expertise and")
        >>> doc += " experiences in business application development and proven records in delivering 90's software. "
        >>> doc += "\n\nSummary: \n\tSoftware Architect"
        >>> doc += (" who has gone through several full product-delivery life cycles from requirements " +
        ...         "gathering to deployment / production, and")
        >>> doc += (" skilled in all areas of software development from client-side JavaScript to " +
        ...         "database modeling. With strong experiences in:")
        >>> doc += " \n\tRequirements gathering and analysis."

        The python splitter will produce 2 tokens that are only punctuation ("/")
        >>> len([s for s in doc.split() if s])
        72

        The built-in nonword REGEX ignores all-punctuation words, so there are 2 less here:
        >>> len(list(Tokenizer(doc, strip=False, nonwords=False)))
        70

        In addition, punctuation at the end of tokens is stripped so "D. Rock" doesn't tokenize to "D." but rather "D"
        >>> run_together_tokens = ''.join(list(Tokenizer(doc, strip=False, nonwords=False)))
        >>> '/' in run_together_tokens or ':' in ''.join(run_together_tokens)
        False

        But you can turn off stripping when instantiating the object.
        >>> all(t in Tokenizer(doc, strip=False, nonwords=True) for t in
        ...     ('D', '_Project', 'Lead_', "90's", "product-delivery"))
        True