r"""Return a sequence of words or tokens, using a re.match iteratively through the str

    >>> doc = "John D. Rock\n\nObjective: \n\tSeeking a position as Software --Architect-- / _Project Lead_ that can utilize my expertise and"
    >>> doc += " experiences in business application development and proven records in delivering 90's software. \n\nSummary: \n\tSoftware Architect"
    >>> doc += " who has gone through several full product-delivery life cycles from requirements gathering to deployment / production, and"
    >>> doc += " skilled in all areas of software development from client-side JavaScript to database modeling. With strong experiences in:"
    >>> doc += " \n\tRequirements gathering and analysis."
    >>> len(list(generate_tokens(doc, strip=False, nonwords=True)))
    82
    >>> seq = list(generate_tokens(doc, strip=False, nonwords=False))
    >>> len(seq)
    70
    >>> '.' in seq or ':' in seq
    False
    >>> s = set(generate_tokens(doc, strip=False, nonwords=True))
    >>> all(t in s for t in ('D', '.', ':', '_Project', 'Lead_', "90's", "Architect", "product-delivery"))
    True

def generate_tokens(doc, regex=CRE_TOKEN, strip=True, nonwords=False):
    r"""Return a sequence of words or tokens, using a re.match iteratively through the str

    >>> doc = "John D. Rock\n\nObjective: \n\tSeeking a position as Software --Architect-- / _Project Lead_ that can utilize my expertise and"
    >>> doc += " experiences in business application development and proven records in delivering 90's software. \n\nSummary: \n\tSoftware Architect"
    >>> doc += " who has gone through several full product-delivery life cycles from requirements gathering to deployment / production, and"
    >>> doc += " skilled in all areas of software development from client-side JavaScript to database modeling. With strong experiences in:"
    >>> doc += " \n\tRequirements gathering and analysis."
    >>> len(list(generate_tokens(doc, strip=False, nonwords=True)))
    82
    >>> seq = list(generate_tokens(doc, strip=False, nonwords=False))
    >>> len(seq)
    70
    >>> '.' in seq or ':' in seq
    False
    >>> s = set(generate_tokens(doc, strip=False, nonwords=True))
    >>> all(t in s for t in ('D', '.', ':', '_Project', 'Lead_', "90's", "Architect", "product-delivery"))
    True
    """
    if isinstance(regex, basestring):
        regex = re.compile(regex)
    for w in regex.finditer(doc):
        if w:
            w = w.group()
            if strip:
                w = w.strip(r'-_*`()}{' + r"'")
            if w and (nonwords or not re.match(r'^' + RE_NONWORD + '$', w)):
                yield w