def vocab_freq(docs, limit=1e6, verbose=1, tokenizer=generate_tokens):
    """Get the set of words used anywhere in a sequence of documents and count occurrences

    >>> gen = ('label: ' + chr(ord('A') + i % 3)*3 for i in range(11))
    >>> vocab_freq(gen, verbose=0)
    Counter({'label': 11, 'AAA': 4, 'BBB': 4, 'CCC': 3})
    """
    total = Counter()
    try:
        limit = min(limit, docs.count())
        docs = docs.iterator()
    except:
        pass
    for i, doc in enumerate(docs):
        try:
            doc = doc.values()
        except AttributeError:
            if not isinstance(doc, basestring):
                doc = ' '.join([stringify(v) for v in doc])
            else:
                doc = stringify(doc)
        if i >= limit:
            break
        c = Counter(tokenizer(doc, strip=True, nonwords=False))
        if verbose and (verbose < 1e-3 or not i % int(limit * verbose)):
            print('{}: {} ... {}'.format(i, c.keys()[:3], c.keys()[-3:] if len(c.keys()) > 6 else ''))
        total += c
    return total
def vocab_freq ( docs , limit = 1e6 , verbose = 1 , tokenizer = generate_tokens ) : total = Counter ( ) try : limit = min ( limit , docs . count ( ) ) docs = docs . iterator ( ) except : pass for i , doc in enumerate ( docs ) : try : doc = doc . values ( ) except AttributeError : if not isinstance ( doc , basestring ) : doc = ' ' . join ( [ stringify ( v ) for v in doc ] ) else : doc = stringify ( doc ) if i >= limit : break c = Counter ( tokenizer ( doc , strip = True , nonwords = False ) ) if verbose and ( verbose < 1e-3 or not i % int ( limit * verbose ) ) : print ( '{}: {} ... {}' . format ( i , c . keys ( ) [ : 3 ] , c . keys ( ) [ - 3 : ] if len ( c . keys ( ) ) > 6 else '' ) ) total += c return total
Get the set of words used anywhere in a sequence of documents and count occurrences

    >>> gen = ('label: ' + chr(ord('A') + i % 3)*3 for i in range(11))
    >>> vocab_freq(gen, verbose=0)
    Counter({'label': 11, 'AAA': 4, 'BBB': 4, 'CCC': 3})