Get the set of words used anywhere in a sequence of documents and count occurrences

    >>> gen = ('label: ' + chr(ord('A') + i % 3)*3 for i in range(11))
    >>> vocab_freq(gen, verbose=0)
    Counter({'label': 11, 'AAA': 4, 'BBB': 4, 'CCC': 3})

def vocab_freq(docs, limit=1e6, verbose=1, tokenizer=generate_tokens):
    """Get the set of words used anywhere in a sequence of documents and count occurrences

    >>> gen = ('label: ' + chr(ord('A') + i % 3)*3 for i in range(11))
    >>> vocab_freq(gen, verbose=0)
    Counter({'label': 11, 'AAA': 4, 'BBB': 4, 'CCC': 3})
    """
    total = Counter()
    try:
        limit = min(limit, docs.count())
        docs = docs.iterator()
    except:
        pass
    for i, doc in enumerate(docs):
        try:
            doc = doc.values()
        except AttributeError:
            if not isinstance(doc, basestring):
                doc = ' '.join([stringify(v) for v in doc])
            else:
                doc = stringify(doc)
        if i >= limit:
            break
        c = Counter(tokenizer(doc, strip=True, nonwords=False))
        if verbose and (verbose < 1e-3 or not i % int(limit * verbose)):
            print('{}: {} ... {}'.format(i, c.keys()[:3], c.keys()[-3:] if len(c.keys()) > 6 else ''))
        total += c
    return total