<s> '''<STR_LIT>''' <EOL> from __future__ import absolute_import , division <EOL> import json <EOL> import logging <EOL> import os <EOL> import sys <EOL> import time <EOL> import datetime <EOL> import argparse <EOL> import dblogger <EOL> import json <EOL> import yakonfig <EOL> import streamcorpus <EOL> from streamcorpus import Chunk , make_stream_item <EOL> import streamcorpus_pipeline <EOL> from streamcorpus_pipeline . stages import PipelineStages <EOL> from streamcorpus_pipeline . _pipeline import PipelineFactory <EOL> from dossier . fc import FeatureCollectionChunk as FCChunk <EOL> from dossier . fc import FeatureCollection , StringCounter <EOL> import operator <EOL> try : <EOL> from collections import Counter , defaultdict <EOL> except ImportError : <EOL> from backport_collections import Counter , defaultdict <EOL> import numpy as np <EOL> from sklearn . feature_extraction import DictVectorizer <EOL> from sklearn . naive_bayes import BernoulliNB <EOL> logger = logging . getLogger ( __name__ ) <EOL> def extract ( positive_fcs , negative_fcs , features = None ) : <EOL> '''<STR_LIT>''' <EOL> labels = np . array ( [ <NUM_LIT:1> ] * len ( positive_fcs ) + [ <NUM_LIT:0> ] * len ( negative_fcs ) ) <EOL> v = DictVectorizer ( sparse = False ) <EOL> D = list ( ) <EOL> for fc in ( positive_fcs + negative_fcs ) : <EOL> feat = StringCounter ( ) <EOL> if not fc : <EOL> logger . warn ( '<STR_LIT>' , fc ) <EOL> else : <EOL> for f in features : <EOL> feat += fc [ f ] <EOL> D . append ( feat ) <EOL> X = v . fit_transform ( D ) <EOL> clf = BernoulliNB ( ) <EOL> clf . fit ( X , labels ) <EOL> positive_keywords = v . inverse_transform ( clf . feature_log_prob_ [ <NUM_LIT:1> ] ) [ <NUM_LIT:0> ] <EOL> negative_keywords = v . inverse_transform ( clf . feature_log_prob_ [ <NUM_LIT:0> ] ) [ <NUM_LIT:0> ] <EOL> pos_words = Counter ( positive_keywords ) <EOL> neg_words = Counter ( negative_keywords ) <EOL> pos_ordered = sorted ( pos_words . items ( ) , <EOL> key = operator . itemgetter ( <NUM_LIT:1> ) , reverse = True ) <EOL> neg_ordered = sorted ( neg_words . items ( ) , <EOL> key = operator . itemgetter ( <NUM_LIT:1> ) , reverse = True ) <EOL> return pos_ordered , neg_ordered <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> '''<STR_LIT>''' <EOL> parser = argparse . ArgumentParser ( <EOL> usage = '<STR_LIT>' , <EOL> description = __doc__ ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> args = parser . parse_args ( ) <EOL> positive_fcs = list ( ) <EOL> negative_fcs = list ( ) <EOL> for i , fc in enumerate ( FCChunk ( args . corpus ) ) : <EOL> if i % <NUM_LIT:2> : <EOL> positive_fcs . append ( fc ) <EOL> else : <EOL> negative_fcs . append ( fc ) <EOL> keywords = extract ( <EOL> positive_fcs , <EOL> negative_fcs , <EOL> features = [ '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ] <EOL> ) <EOL> logger . info ( '<STR_LIT>' , keywords [ <NUM_LIT:0> ] ) <EOL> logger . info ( '<STR_LIT>' , keywords [ <NUM_LIT:1> ] ) </s>
