def columns_dataset(self):
        """
        Generate the columns and the whole dataset.
        """
        data = {}
        words_total = {}

        for instance, words in self.raw_dataset.items():
            words_item_total = {}
            for word in words:
                words_total.setdefault(word, 0)
                words_item_total.setdefault(word, 0)
                words_total[word] += 1
                words_item_total[word] += 1
            data[instance] = words_item_total

        columns = sorted(words_total.keys(),
                         key=lambda w: words_total[w],
                         reverse=True)[:250]
        columns = sorted(columns)
        dataset = {}
        for instance in data.keys():
            dataset[instance] = [data[instance].get(word, 0)
                                 for word in columns]
        return columns, dataset
def columns_dataset ( self ) : data = { } words_total = { } for instance , words in self . raw_dataset . items ( ) : words_item_total = { } for word in words : words_total . setdefault ( word , 0 ) words_item_total . setdefault ( word , 0 ) words_total [ word ] += 1 words_item_total [ word ] += 1 data [ instance ] = words_item_total columns = sorted ( words_total . keys ( ) , key = lambda w : words_total [ w ] , reverse = True ) [ : 250 ] columns = sorted ( columns ) dataset = { } for instance in data . keys ( ) : dataset [ instance ] = [ data [ instance ] . get ( word , 0 ) for word in columns ] return columns , dataset
Generate the columns and the whole dataset.