def make_global_state(self, shreds_tags):
    doc_counts = collections.defaultdict(int)

    for doc, tags in shreds_tags.items():
        for tag in tags:
            doc_counts[tag] += 1

    num_docs = float(len(shreds_tags))

    idf = {}
    for tag, count in doc_counts.items():
        idf[tag] = math.log(num_docs / count)
    return {
        'idf_map': idf,
        'all_terms': sorted(idf.keys()),
    }
Precomputes TF-DF global state.
dataset/ETHPy150Open dchaplinsky/unshred-tag/metrics/tfidf.py/make_global_state