<s> """<STR_LIT>""" <EOL> from __future__ import division <EOL> from __future__ import absolute_import <EOL> from __future__ import print_function <EOL> from __future__ import unicode_literals <EOL> import argparse <EOL> import os <EOL> import numpy as np <EOL> from sklearn . metrics import precision_score <EOL> from sklearn . feature_extraction . text import CountVectorizer <EOL> from sklearn . feature_extraction . text import TfidfVectorizer <EOL> from sklearn import linear_model <EOL> from sklearn import cross_validation <EOL> from nltk . corpus import stopwords <EOL> import unicodecsv <EOL> def reader ( class_name ) : <EOL> class_reader = unicodecsv . reader ( open ( class_name ) , encoding = '<STR_LIT:utf-8>' ) <EOL> row0 = next ( class_reader ) <EOL> assert row0 == [ "<STR_LIT>" , "<STR_LIT>" ] <EOL> lines = [ ] <EOL> for tweet_id , tweet_text in class_reader : <EOL> txt = tweet_text . strip ( ) <EOL> if len ( txt ) > <NUM_LIT:0> : <EOL> lines . append ( txt ) <EOL> return lines <EOL> def cross_entropy_error ( Y , probas_ ) : <EOL> probas_class1 = probas_ [ : , <NUM_LIT:1> ] <EOL> cross_entropy_errors = ( ( - Y ) * ( np . log ( probas_class1 ) ) ) - ( ( <NUM_LIT:1> - Y ) * ( np . log ( <NUM_LIT:1> - probas_class1 ) ) ) <EOL> return cross_entropy_errors <EOL> def show_errors ( cross_entropy_errors_by_fold , method = "<STR_LIT>" , lower_is_better = True ) : <EOL> print ( "<STR_LIT>" % ( method ) + str ( cross_entropy_errors_by_fold ) ) <EOL> if lower_is_better : <EOL> note = "<STR_LIT>" <EOL> else : <EOL> note = "<STR_LIT>" <EOL> print ( "<STR_LIT>" % ( method , note , cross_entropy_errors_by_fold . mean ( ) , cross_entropy_errors_by_fold . std ( ) / <NUM_LIT:2> ) ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> parser = argparse . ArgumentParser ( description = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , default = False , action = "<STR_LIT:store_true>" , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , default = False , action = "<STR_LIT:store_true>" , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , default = False , action = "<STR_LIT:store_true>" , help = '<STR_LIT>' ) <EOL> args = parser . parse_args ( ) <EOL> data_dir = "<STR_LIT:data>" <EOL> in_class_name = os . path . join ( data_dir , args . table + '<STR_LIT>' ) <EOL> out_class_name = os . path . join ( data_dir , args . table + '<STR_LIT>' ) <EOL> in_class_lines = reader ( in_class_name ) <EOL> out_class_lines = reader ( out_class_name ) <EOL> train_set = np . array ( out_class_lines + in_class_lines ) <EOL> target = np . array ( [ <NUM_LIT:0> ] * len ( out_class_lines ) + [ <NUM_LIT:1> ] * len ( in_class_lines ) ) <EOL> stopWords = stopwords . words ( '<STR_LIT>' ) <EOL> MIN_DF = <NUM_LIT:2> <EOL> vectorizer_binary = CountVectorizer ( stop_words = stopWords , min_df = MIN_DF , binary = True , ngram_range = ( <NUM_LIT:1> , <NUM_LIT:3> ) ) <EOL> vectorizer_tfidf = TfidfVectorizer ( stop_words = stopWords , min_df = MIN_DF , ngram_range = ( <NUM_LIT:1> , <NUM_LIT:3> ) ) <EOL> vectorizer = vectorizer_tfidf <EOL> print ( vectorizer ) <EOL> clf = linear_model . LogisticRegression ( ) <EOL> kf = cross_validation . KFold ( n = len ( target ) , n_folds = <NUM_LIT:5> , shuffle = True ) <EOL> cross_entropy_errors_test_by_fold = np . zeros ( len ( kf ) ) <EOL> cross_entropy_errors_train_by_fold = np . zeros ( len ( kf ) ) <EOL> precisions_by_fold = np . zeros ( len ( kf ) ) <EOL> for i , ( train_rows , test_rows ) in enumerate ( kf ) : <EOL> tweets_train_rows = train_set [ train_rows ] <EOL> tweets_test_rows = train_set [ test_rows ] <EOL> Y_train = target [ train_rows ] <EOL> Y_test = target [ test_rows ] <EOL> X_train = vectorizer . fit_transform ( tweets_train_rows ) . toarray ( ) <EOL> X_test = vectorizer . transform ( tweets_test_rows ) . todense ( ) <EOL> clf . fit ( X_train , Y_train ) <EOL> probas_test_ = clf . predict_proba ( X_test ) <EOL> probas_train_ = clf . predict_proba ( X_train ) <EOL> cross_entropy_errors_test = cross_entropy_error ( Y_test , probas_test_ ) <EOL> cross_entropy_errors_train = cross_entropy_error ( Y_train , probas_train_ ) <EOL> cross_entropy_errors_test_by_fold [ i ] = np . average ( cross_entropy_errors_test ) <EOL> cross_entropy_errors_train_by_fold [ i ] = np . average ( cross_entropy_errors_train ) <EOL> precisions_by_fold [ i ] = precision_score ( Y_test , clf . predict ( X_test ) ) <EOL> print ( "<STR_LIT>" ) <EOL> show_errors ( cross_entropy_errors_train_by_fold ) <EOL> print ( "<STR_LIT>" ) <EOL> show_errors ( cross_entropy_errors_test_by_fold ) <EOL> print ( "<STR_LIT>" ) <EOL> show_errors ( precisions_by_fold , method = "<STR_LIT>" , lower_is_better = False ) </s>
