<s> import time <EOL> start_time = time . time ( ) <EOL> import numpy as np <EOL> from sklearn . metrics import classification_report <EOL> from sklearn . metrics import precision_recall_curve , roc_curve , auc <EOL> from sklearn . cross_validation import KFold <EOL> from sklearn import neighbors <EOL> from data import chosen , chosen_meta <EOL> from utils import plot_pr <EOL> from utils import plot_feat_importance <EOL> from utils import load_meta <EOL> from utils import fetch_posts <EOL> from utils import plot_feat_hist <EOL> from utils import plot_bias_variance <EOL> from utils import plot_k_complexity <EOL> meta , id_to_idx , idx_to_id = load_meta ( chosen_meta ) <EOL> import nltk <EOL> all_questions = sorted ( [ q for q , v in meta . items ( ) if v [ '<STR_LIT>' ] == - <NUM_LIT:1> ] ) <EOL> all_answers = sorted ( [ q for q , v in meta . items ( ) if v [ '<STR_LIT>' ] != - <NUM_LIT:1> ] ) <EOL> feature_names = np . array ( ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' <EOL> ) ) <EOL> def prepare_sent_features ( ) : <EOL> for pid , text in fetch_posts ( chosen , with_index = True ) : <EOL> if not text : <EOL> meta [ pid ] [ '<STR_LIT>' ] = meta [ pid ] [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> else : <EOL> from platform import python_version <EOL> if python_version ( ) . startswith ( '<STR_LIT:2>' ) : <EOL> text = text . decode ( '<STR_LIT:utf-8>' ) <EOL> sent_lens = [ len ( nltk . word_tokenize ( <EOL> sent ) ) for sent in nltk . sent_tokenize ( text ) ] <EOL> meta [ pid ] [ '<STR_LIT>' ] = np . mean ( sent_lens ) <EOL> meta [ pid ] [ '<STR_LIT>' ] = np . mean ( <EOL> [ len ( w ) for w in nltk . word_tokenize ( text ) ] ) <EOL> meta [ pid ] [ '<STR_LIT>' ] = np . sum ( <EOL> [ word . isupper ( ) for word in nltk . word_tokenize ( text ) ] ) <EOL> meta [ pid ] [ '<STR_LIT>' ] = text . count ( '<STR_LIT:!>' ) <EOL> prepare_sent_features ( ) <EOL> def get_features ( aid ) : <EOL> return tuple ( meta [ aid ] [ fn ] for fn in feature_names ) <EOL> qa_X = np . asarray ( [ get_features ( aid ) for aid in all_answers ] ) <EOL> classifying_answer = "<STR_LIT>" <EOL> if classifying_answer == "<STR_LIT>" : <EOL> qa_Y = np . asarray ( [ meta [ aid ] [ '<STR_LIT>' ] > <NUM_LIT:0> for aid in all_answers ] ) <EOL> elif classifying_answer == "<STR_LIT>" : <EOL> qa_Y = np . asarray ( [ meta [ aid ] [ '<STR_LIT>' ] <= <NUM_LIT:0> for aid in all_answers ] ) <EOL> else : <EOL> raise Exception ( "<STR_LIT>" % <EOL> classifying_answer ) <EOL> for idx , feat in enumerate ( feature_names ) : <EOL> plot_feat_hist ( [ ( qa_X [ : , idx ] , feat ) ] ) <EOL> avg_scores_summary = [ ] <EOL> def measure ( clf_class , parameters , name , data_size = None , plot = False ) : <EOL> start_time_clf = time . time ( ) <EOL> if data_size is None : <EOL> X = qa_X <EOL> Y = qa_Y <EOL> else : <EOL> X = qa_X [ : data_size ] <EOL> Y = qa_Y [ : data_size ] <EOL> cv = KFold ( n = len ( X ) , n_folds = <NUM_LIT:10> , indices = True ) <EOL> train_errors = [ ] <EOL> test_errors = [ ] <EOL> scores = [ ] <EOL> roc_scores = [ ] <EOL> fprs , tprs = [ ] , [ ] <EOL> pr_scores = [ ] <EOL> precisions , recalls , thresholds = [ ] , [ ] , [ ] <EOL> for fold_idx , ( train , test ) in enumerate ( cv ) : <EOL> X_train , y_train = X [ train ] , Y [ train ] <EOL> X_test , y_test = X [ test ] , Y [ test ] <EOL> only_one_class_in_train = len ( set ( y_train ) ) == <NUM_LIT:1> <EOL> only_one_class_in_test = len ( set ( y_test ) ) == <NUM_LIT:1> <EOL> if only_one_class_in_train or only_one_class_in_test : <EOL> continue <EOL> clf = clf_class ( ** parameters ) <EOL> clf . fit ( X_train , y_train ) <EOL> train_score = clf . score ( X_train , y_train ) <EOL> test_score = clf . score ( X_test , y_test ) <EOL> train_errors . append ( <NUM_LIT:1> - train_score ) <EOL> test_errors . append ( <NUM_LIT:1> - test_score ) <EOL> scores . append ( test_score ) <EOL> proba = clf . predict_proba ( X_test ) <EOL> label_idx = <NUM_LIT:1> <EOL> fpr , tpr , roc_thresholds = roc_curve ( y_test , proba [ : , label_idx ] ) <EOL> precision , recall , pr_thresholds = precision_recall_curve ( <EOL> y_test , proba [ : , label_idx ] ) <EOL> roc_scores . append ( auc ( fpr , tpr ) ) <EOL> fprs . append ( fpr ) <EOL> tprs . append ( tpr ) <EOL> pr_scores . append ( auc ( recall , precision ) ) <EOL> precisions . append ( precision ) <EOL> recalls . append ( recall ) <EOL> thresholds . append ( pr_thresholds ) <EOL> threshold_for_detecting_good_answers = <NUM_LIT> <EOL> print ( "<STR_LIT>" % fold_idx ) <EOL> print ( classification_report ( y_test , proba [ : , label_idx ] > <EOL> threshold_for_detecting_good_answers , target_names = [ '<STR_LIT>' , '<STR_LIT>' ] ) ) <EOL> scores_to_sort = pr_scores <EOL> medium = np . argsort ( scores_to_sort ) [ len ( scores_to_sort ) / <NUM_LIT:2> ] <EOL> print ( "<STR_LIT>" % medium ) <EOL> if plot : <EOL> plot_pr ( pr_scores [ medium ] , name , precisions [ medium ] , <EOL> recalls [ medium ] , classifying_answer + "<STR_LIT>" ) <EOL> if hasattr ( clf , '<STR_LIT>' ) : <EOL> plot_feat_importance ( feature_names , clf , name ) <EOL> summary = ( name , <EOL> np . mean ( scores ) , np . std ( scores ) , <EOL> np . mean ( roc_scores ) , np . std ( roc_scores ) , <EOL> np . mean ( pr_scores ) , np . std ( pr_scores ) , <EOL> time . time ( ) - start_time_clf ) <EOL> print ( summary ) <EOL> avg_scores_summary . append ( summary ) <EOL> precisions = precisions [ medium ] <EOL> recalls = recalls [ medium ] <EOL> thresholds = np . hstack ( ( [ <NUM_LIT:0> ] , thresholds [ medium ] ) ) <EOL> idx80 = precisions >= <NUM_LIT> <EOL> print ( "<STR_LIT>" % ( precisions [ idx80 ] [ <NUM_LIT:0> ] , recalls [ <EOL> idx80 ] [ <NUM_LIT:0> ] , thresholds [ idx80 ] [ <NUM_LIT:0> ] ) ) <EOL> return np . mean ( train_errors ) , np . mean ( test_errors ) <EOL> def bias_variance_analysis ( clf_class , parameters , name ) : <EOL> data_sizes = np . arange ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT:4> ) <EOL> train_errors = [ ] <EOL> test_errors = [ ] <EOL> for data_size in data_sizes : <EOL> train_error , test_error = measure ( <EOL> clf_class , parameters , name , data_size = data_size ) <EOL> train_errors . append ( train_error ) <EOL> test_errors . append ( test_error ) <EOL> plot_bias_variance ( data_sizes , train_errors , <EOL> test_errors , name , "<STR_LIT>" % name ) <EOL> def k_complexity_analysis ( clf_class , parameters ) : <EOL> ks = np . hstack ( ( np . arange ( <NUM_LIT:1> , <NUM_LIT:20> ) , np . arange ( <NUM_LIT> , <NUM_LIT:100> , <NUM_LIT:5> ) ) ) <EOL> train_errors = [ ] <EOL> test_errors = [ ] <EOL> for k in ks : <EOL> parameters [ '<STR_LIT>' ] = k <EOL> train_error , test_error = measure ( <EOL> clf_class , parameters , "<STR_LIT>" % k , data_size = <NUM_LIT> ) <EOL> train_errors . append ( train_error ) <EOL> test_errors . append ( test_error ) <EOL> plot_k_complexity ( ks , train_errors , test_errors ) <EOL> for k in [ <NUM_LIT:5> ] : <EOL> bias_variance_analysis ( neighbors . KNeighborsClassifier , { <EOL> '<STR_LIT>' : k } , "<STR_LIT>" % k ) <EOL> k_complexity_analysis ( neighbors . KNeighborsClassifier , { '<STR_LIT>' : k } ) <EOL> from sklearn . linear_model import LogisticRegression <EOL> for C in [ <NUM_LIT:0.1> ] : <EOL> name = "<STR_LIT>" % C <EOL> bias_variance_analysis ( LogisticRegression , { '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT:C>' : C } , name ) <EOL> measure ( LogisticRegression , { '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT:C>' : C } , name , plot = True ) <EOL> print ( "<STR_LIT:=>" * <NUM_LIT:50> ) <EOL> from operator import itemgetter <EOL> for s in reversed ( sorted ( avg_scores_summary , key = itemgetter ( <NUM_LIT:1> ) ) ) : <EOL> print ( "<STR_LIT>" % s ) <EOL> print ( "<STR_LIT>" , time . time ( ) - start_time ) </s>
