<filename>pyspan/procedural_words.py
#!/usr/bin/python2.7

"""Helper functions for scripts/get_materials_for_procedural_words_coding.py.
"""

from __future__ import division
from collections import defaultdict
from itertools import chain
import numpy as np
import os
import pickle
from pyspan.config import *
input_dir = paths["input_dir"]
metrics_dir = paths["metrics_dir"]
proc_txt_dir = paths["proc_txt_dir"]
from pyspan.utils import *

class ProceduralWords(object):
    def __init__(self, mode = "unigrams"):
        assert mode in ("unigrams", "bigrams")
        self.mode = mode
        if self.mode == "unigrams":
            # Generated by scripts/clean_leipzig_corpus.py
            self.gen_corpus_fn = "leipzig_corpus/en_word_freqs"
        else:
            # Script to generate this is clean_coca_corpus.py
            self.gen_corpus_fn = "coca/2_grams"

    def load_df(self, fn = "relative_metrics", reload_ = False):
        fn = "{}{}-{}".format(input_dir, fn, self.mode)
        if os.path.exists(fn) and not reload_:
            self.df = pickle.load(open(fn, "rb"))
        else:
            self.gen_df = pickle.load(open("{}{}".format(input_dir,
                                                         self.gen_corpus_fn),
                                           "rb"))
            self.party_df = pickle.load(open("{}frequencies-{}".format(metrics_dir,
                                                                       self.mode),
                                             "rb"))
            self.party_df["party_frequency"] = self.party_df["dmetric"] + self.party_df["rmetric"]
            self.gen_df.rename(columns = { "frequency": "gen_frequency" },
                               inplace = True)
            self.df = self.gen_df[["gen_frequency"]].join(self.party_df[["party_frequency"]],
                                                          how = "right")
            self.df.fillna(0, inplace = True)

            # Compute probabilities
            denom = sum(self.df["gen_frequency"])
            self.df["gen_probability"] = self.df["gen_frequency"]/denom
            assert approx_eq(sum(self.df["gen_probability"]), 1)

            denom = sum(self.df["party_frequency"])
            self.df["party_probability"] = self.df["party_frequency"]/denom
            assert approx_eq(sum(self.df["party_probability"]), 1)

            n_speeches = []
            for word in self.df.index:
                n_speeches.append(int(os.popen("grep -P \'(?<![[:alnum:]]){}(?![[:alnum:]])\' {}/*/*/*speech | wc -l".format(word, proc_txt_dir)).read()))
            self.df["n_speeches"] = n_speeches

            total_n_speeches = int(os.popen("cat {}/*/*/*speech | wc -l".format(proc_txt_dir)).read())
            assert total_n_speeches == 144009
            self.df["frac_speeches"] = np.array(n_speeches)/total_n_speeches

            # Save to file
            with open(fn, "wb") as wfh:
                pickle.dump(self.df, wfh)

    # Return a list of words that are said in at least 2% of speeches and that
    # are said at least 100x more frequently than in the general corpus
    def filter_(self):
        procedural_words = [ w for w in self.df.index if
                             self.df.loc[w]["frac_speeches"] >= .01 and
                             self.df.loc[w]["party_probability"] >=
                             10*self.df.loc[w]["gen_probability"] ]
        return procedural_words

if __name__ == "__main__":
    for mode in ("unigrams", "bigrams"):
        pw = ProceduralWords(mode = mode)
        pw.load_df()

<filename>pyspan/procedural_words.py
#!/usr/bin/python2.7

"""Helper functions for scripts/get_materials_for_procedural_words_coding.py.
"""

from __future__ import division
from collections import defaultdict
from itertools import chain
import numpy as np
import os
import pickle
from pyspan.config import *
input_dir = paths["input_dir"]
metrics_dir = paths["metrics_dir"]
proc_txt_dir = paths["proc_txt_dir"]
from pyspan.utils import *

class ProceduralWords(object):
    def __init__(self, mode = "unigrams"):
        assert mode in ("unigrams", "bigrams")
        self.mode = mode
        if self.mode == "unigrams":
            # Generated by scripts/clean_leipzig_corpus.py
            self.gen_corpus_fn = "leipzig_corpus/en_word_freqs"
        else:
            # Script to generate this is clean_coca_corpus.py
            self.gen_corpus_fn = "coca/2_grams"

    def load_df(self, fn = "relative_metrics", reload_ = False):
        fn = "{}{}-{}".format(input_dir, fn, self.mode)
        if os.path.exists(fn) and not reload_:
            self.df = pickle.load(open(fn, "rb"))
        else:
            self.gen_df = pickle.load(open("{}{}".format(input_dir,
                                                         self.gen_corpus_fn),
                                           "rb"))
            self.party_df = pickle.load(open("{}frequencies-{}".format(metrics_dir,
                                                                       self.mode),
                                             "rb"))
            self.party_df["party_frequency"] = self.party_df["dmetric"] + self.party_df["rmetric"]
            self.gen_df.rename(columns = { "frequency": "gen_frequency" },
                               inplace = True)
            self.df = self.gen_df[["gen_frequency"]].join(self.party_df[["party_frequency"]],
                                                          how = "right")
            self.df.fillna(0, inplace = True)

            # Compute probabilities
            denom = sum(self.df["gen_frequency"])
            self.df["gen_probability"] = self.df["gen_frequency"]/denom
            assert approx_eq(sum(self.df["gen_probability"]), 1)

            denom = sum(self.df["party_frequency"])
            self.df["party_probability"] = self.df["party_frequency"]/denom
            assert approx_eq(sum(self.df["party_probability"]), 1)

            n_speeches = []
            for word in self.df.index:
                n_speeches.append(int(os.popen("grep -P \'(?<![[:alnum:]]){}(?![[:alnum:]])\' {}/*/*/*speech | wc -l".format(word, proc_txt_dir)).read()))
            self.df["n_speeches"] = n_speeches

            total_n_speeches = int(os.popen("cat {}/*/*/*speech | wc -l".format(proc_txt_dir)).read())
            assert total_n_speeches == 144009
            self.df["frac_speeches"] = np.array(n_speeches)/total_n_speeches

            # Save to file
            with open(fn, "wb") as wfh:
                pickle.dump(self.df, wfh)

    # Return a list of words that are said in at least 2% of speeches and that
    # are said at least 100x more frequently than in the general corpus
    def filter_(self):
        procedural_words = [ w for w in self.df.index if
                             self.df.loc[w]["frac_speeches"] >= .01 and
                             self.df.loc[w]["party_probability"] >=
                             10*self.df.loc[w]["gen_probability"] ]
        return procedural_words

if __name__ == "__main__":
    for mode in ("unigrams", "bigrams"):
        pw = ProceduralWords(mode = mode)
        pw.load_df()

#!/usr/bin/python2.7 Helper functions for scripts/get_materials_for_procedural_words_coding.py. # Generated by scripts/clean_leipzig_corpus.py # Script to generate this is clean_coca_corpus.py # Compute probabilities # Save to file # Return a list of words that are said in at least 2% of speeches and that # are said at least 100x more frequently than in the general corpus