<s> from __future__ import print_function <EOL> import numpy as np <EOL> import pandas as pd <EOL> import os <EOL> import time <EOL> import pprint <EOL> import cPickle <EOL> from keras . callbacks import RemoteMonitor <EOL> from keras . preprocessing import sequence <EOL> from keras . models import Sequential <EOL> from keras . optimizers import SGD , Adam , RMSprop <EOL> from keras . preprocessing . text import Tokenizer <EOL> from keras . layers . embeddings import Embedding <EOL> from keras . utils import np_utils <EOL> from mboxConvert import parseEmails , getEmailStats , mboxToBinaryCSV <EOL> from kerasPlotter import Plotter <EOL> from keras . layers . embeddings import Embedding <EOL> from keras . layers . convolutional import Convolution1D , MaxPooling1D <EOL> from keras . layers . recurrent import LSTM , GRU <EOL> from keras . layers . core import Dense , Dropout , Activation , Flatten <EOL> from sklearn . feature_extraction . text import CountVectorizer , TfidfVectorizer <EOL> def get_word_features ( emails , verbose = True , nb_words = <NUM_LIT> , skip_top = <NUM_LIT:0> , maxlen = None , as_matrix = True , matrix_type = '<STR_LIT:count>' , label_cutoff = <NUM_LIT> , max_n = <NUM_LIT:1> ) : <EOL> ( totalWordsCount , fromCount , domainCount , labels ) = getEmailStats ( emails ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' % str ( labels ) ) <EOL> print ( '<STR_LIT>' ) <EOL> total = <NUM_LIT:0> <EOL> for label in labels : <EOL> count = sum ( totalWordsCount [ label ] . values ( ) ) <EOL> total += count <EOL> print ( '<STR_LIT>' % ( label , count ) ) <EOL> print ( '<STR_LIT>' % total ) <EOL> labelCounts = { label : <NUM_LIT:0> for label in labels } <EOL> for email in emails : <EOL> labelCounts [ email . label ] += <NUM_LIT:1> <EOL> cutoff = int ( len ( emails ) * label_cutoff ) <EOL> removed = <NUM_LIT:0> <EOL> for label in labels [ : ] : <EOL> if labelCounts [ label ] < cutoff or label == '<STR_LIT>' or label == '<STR_LIT>' or label == '<STR_LIT>' : <EOL> removed += <NUM_LIT:1> <EOL> labels . remove ( label ) <EOL> labelNums = { labels [ i ] : i for i in range ( len ( labels ) ) } <EOL> if verbose : <EOL> print ( '<STR_LIT>' % ( removed , cutoff ) ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' % str ( labels ) ) <EOL> print ( '<STR_LIT>' ) <EOL> total = <NUM_LIT:0> <EOL> for label in labels : <EOL> print ( '<STR_LIT>' % ( label , labelCounts [ label ] ) ) <EOL> print ( '<STR_LIT>' % sum ( [ labelCounts [ label ] for label in labels ] ) ) <EOL> texts = [ ] <EOL> emailLabels = [ ] <EOL> for email in emails : <EOL> if email . label not in labels : <EOL> continue <EOL> text = email . sender + "<STR_LIT:U+0020>" + str ( email . subject ) <EOL> text += email . fromDomain <EOL> text += email . content <EOL> texts . append ( text . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) . replace ( '<STR_LIT:\r>' , '<STR_LIT>' ) ) <EOL> emailLabels . append ( labelNums [ email . label ] ) <EOL> emailLabels = np . array ( emailLabels ) <EOL> if max_n == <NUM_LIT:1> or not as_matrix : <EOL> tokenizer = Tokenizer ( nb_words ) <EOL> tokenizer . fit_on_texts ( texts ) <EOL> reverse_word_index = { tokenizer . word_index [ word ] : word for word in tokenizer . word_index } <EOL> word_list = [ reverse_word_index [ i + <NUM_LIT:1> ] for i in range ( nb_words ) ] <EOL> if as_matrix : <EOL> feature_matrix = tokenizer . texts_to_matrix ( texts , mode = matrix_type ) <EOL> return feature_matrix , emailLabels , word_list , labels <EOL> else : <EOL> sequences = tokenizer . texts_to_sequences ( texts ) <EOL> return sequences , emailLabels , word_list , labels <EOL> else : <EOL> if matrix_type == '<STR_LIT>' : <EOL> vectorizer = TfidfVectorizer ( ngram_range = ( <NUM_LIT:1> , max_n ) , max_features = nb_words ) <EOL> else : <EOL> vectorizer = CounterVectorizer ( ngram_range = ( <NUM_LIT:1> , max_n ) , max_features = nb_words , binary = matrix_type == '<STR_LIT>' ) <EOL> feature_matrix = vectorizer . fit_transform ( texts ) <EOL> word_list = vectorizer . get_feature_names ( ) <EOL> return feature_matrix , emailLabels , word_list , labels <EOL> def write_csv ( csvfile , feature_matrix , labels , feature_names = None , verbose = True ) : <EOL> dataframe = pd . DataFrame ( data = feature_matrix , columns = feature_names ) <EOL> dataframe [ '<STR_LIT:label>' ] = labels <EOL> dataframe . to_csv ( csvfile ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' % ( str ( dataframe . columns ) , csvfile ) ) <EOL> def read_csv ( csvfile , verbose = True ) : <EOL> dataframe = pd . read_csv ( csvfile , header = <NUM_LIT:0> ) <EOL> labels = dataframe [ u'<STR_LIT:label>' ] . tolist ( ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' % str ( dataframe . columns ) ) <EOL> dataframe . drop ( u'<STR_LIT:label>' , inplace = True , axis = <NUM_LIT:1> ) <EOL> if u'<STR_LIT>' in dataframe . columns : <EOL> dataframe . drop ( u'<STR_LIT>' , inplace = True , axis = <NUM_LIT:1> ) <EOL> feature_matrix = dataframe . as_matrix ( ) <EOL> feature_names = dataframe . columns <EOL> return feature_matrix , labels , feature_names <EOL> def write_info ( txtfile , label_names , verbose = True ) : <EOL> with open ( txtfile , '<STR_LIT:w>' ) as writeto : <EOL> writeto . write ( '<STR_LIT:U+002C>' . join ( label_names ) ) <EOL> def read_info ( txtfile , verbose = True ) : <EOL> with open ( txtfile , '<STR_LIT:r>' ) as readfrom : <EOL> label_names = readfrom . readline ( ) . split ( '<STR_LIT:U+002C>' ) <EOL> return label_names <EOL> def write_sequences ( txtfile , sequences , labels , verbose = True ) : <EOL> with open ( txtfile , '<STR_LIT:w>' ) as writeto : <EOL> for sequence , label in zip ( sequences , labels ) : <EOL> writeto . write ( '<STR_LIT:U+002C>' . join ( [ str ( x ) for x in sequence ] ) + '<STR_LIT>' + str ( label ) + '<STR_LIT:\n>' ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' % len ( sequences ) ) <EOL> def read_sequences ( txtfile , verbose = True ) : <EOL> sequences = [ ] <EOL> labels = [ ] <EOL> linesnum = <NUM_LIT:0> <EOL> with open ( txtfile , '<STR_LIT:r>' ) as readfrom : <EOL> for line in readfrom : <EOL> linesnum += <NUM_LIT:1> <EOL> parts = line . split ( '<STR_LIT>' ) <EOL> split = parts [ <NUM_LIT:0> ] . split ( '<STR_LIT:U+002C>' ) <EOL> if len ( split ) <= <NUM_LIT:1> : <EOL> continue <EOL> sequences . append ( np . asarray ( split ) ) <EOL> labels . append ( ( int ) ( parts [ <NUM_LIT:1> ] ) ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' % linesnum ) <EOL> return sequences , labels <EOL> dataframe = pd . read_csv ( csvfile , header = <NUM_LIT:0> ) <EOL> labels = dataframe [ u'<STR_LIT:label>' ] . tolist ( ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' % str ( dataframe . columns ) ) <EOL> dataframe . drop ( '<STR_LIT:label>' , inplace = True , axis = <NUM_LIT:1> ) <EOL> feature_matrix = dataframe . as_matrix ( ) <EOL> return feature_matrix , labels <EOL> def make_dataset ( features , labels , num_labels , test_split = <NUM_LIT:0.1> , nb_words = <NUM_LIT:1000> ) : <EOL> if type ( features ) == list : <EOL> num_examples = len ( features ) <EOL> random_order = np . random . permutation ( num_examples ) <EOL> index_split = ( int ) ( test_split * num_examples ) <EOL> train_indices = random_order [ index_split : ] <EOL> test_indices = random_order [ : index_split ] <EOL> X_train = [ features [ i ] for i in train_indices ] <EOL> X_test = [ features [ i ] for i in test_indices ] <EOL> Y_train = [ labels [ i ] for i in train_indices ] <EOL> Y_test = [ labels [ i ] for i in test_indices ] <EOL> else : <EOL> num_examples = features . shape [ <NUM_LIT:0> ] <EOL> random_order = np . random . permutation ( num_examples ) <EOL> index_split = ( int ) ( test_split * num_examples ) <EOL> train_indices = random_order [ index_split : ] <EOL> test_indices = random_order [ : index_split ] <EOL> X_train = features [ train_indices ] <EOL> X_test = features [ test_indices ] <EOL> Y_train = [ labels [ i ] for i in train_indices ] <EOL> Y_test = [ labels [ i ] for i in test_indices ] <EOL> Y_train_c = np_utils . to_categorical ( Y_train , num_labels ) <EOL> Y_test_c = np_utils . to_categorical ( Y_test , num_labels ) <EOL> return ( ( X_train , Y_train_c ) , ( X_test , Y_test_c ) ) , Y_train , Y_test <EOL> def get_emails ( verbose = True ) : <EOL> picklefile = '<STR_LIT>' <EOL> if os . path . isfile ( picklefile ) : <EOL> with open ( picklefile , '<STR_LIT:r>' ) as load_from : <EOL> emails = cPickle . load ( load_from ) <EOL> else : <EOL> emails = parseEmails ( '<STR_LIT:.>' , printInfo = verbose ) <EOL> with open ( picklefile , '<STR_LIT:w>' ) as store_to : <EOL> cPickle . dump ( emails , store_to ) <EOL> return emails <EOL> def get_ngram_data ( num_words = <NUM_LIT:1000> , matrix_type = '<STR_LIT>' , verbose = True , max_n = <NUM_LIT:1> ) : <EOL> csvfile = '<STR_LIT>' % ( num_words , str ( matrix_type ) ) <EOL> infofile = '<STR_LIT>' <EOL> if os . path . isfile ( csvfile ) : <EOL> features , labels , feature_names = read_csv ( csvfile , verbose = verbose ) <EOL> label_names = read_info ( infofile ) <EOL> else : <EOL> emails = get_emails ( verbose = verbose ) <EOL> features , labels , feature_names , label_names = get_word_features ( emails , nb_words = num_words , matrix_type = matrix_type , verbose = verbose , max_n = max_n ) <EOL> if max_n == <NUM_LIT:1> : <EOL> write_csv ( csvfile , features , labels , feature_names , verbose = verbose ) <EOL> write_info ( infofile , label_names ) <EOL> return features , labels , feature_names , label_names <EOL> def get_my_data ( per_label = False ) : <EOL> csvfile = '<STR_LIT>' % str ( per_label ) <EOL> infofile = '<STR_LIT>' <EOL> if os . path . isfile ( csvfile ) : <EOL> features , labels , feature_names = read_csv ( csvfile ) <EOL> label_names = read_info ( infofile ) <EOL> else : <EOL> mboxToBinaryCSV ( '<STR_LIT:.>' , csvfile , perLabel = per_label ) <EOL> features , labels , feature_names = read_csv ( csvfile ) <EOL> label_names = list ( set ( labels ) ) <EOL> write_info ( infofile , label_names ) <EOL> num_labels = max ( labels ) + <NUM_LIT:1> <EOL> return features , labels , feature_names , label_names <EOL> def get_sequence_data ( ) : <EOL> txtfile = '<STR_LIT>' <EOL> infofile = '<STR_LIT>' <EOL> if os . path . isfile ( txtfile ) : <EOL> features , labels = read_sequences ( txtfile ) <EOL> label_names = read_info ( infofile ) <EOL> else : <EOL> emails = parseEmails ( '<STR_LIT:.>' ) <EOL> features , labels , words , labelVals = get_keras_features ( emails , as_matrix = False ) <EOL> write_sequences ( txtfile , features , labels ) <EOL> write_info ( infofile , labelVals ) <EOL> num_labels = max ( labels ) + <NUM_LIT:1> <EOL> return features , labels , label_names <EOL> def evaluate_mlp_model ( dataset , num_classes , extra_layers = <NUM_LIT:0> , num_hidden = <NUM_LIT> , dropout = <NUM_LIT:0.5> , graph_to = None , verbose = True ) : <EOL> ( X_train , Y_train ) , ( X_test , Y_test ) = dataset <EOL> batch_size = <NUM_LIT:32> <EOL> nb_epoch = <NUM_LIT:5> <EOL> max_features = <NUM_LIT> <EOL> maxlen = <NUM_LIT> <EOL> if verbose : <EOL> print ( len ( X_train ) , '<STR_LIT>' ) <EOL> print ( len ( X_test ) , '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' , X_train . shape ) <EOL> print ( '<STR_LIT>' , X_test . shape ) <EOL> print ( '<STR_LIT>' , Y_train . shape ) <EOL> print ( '<STR_LIT>' , Y_test . shape ) <EOL> print ( '<STR_LIT>' ) <EOL> model = Sequential ( ) <EOL> model . add ( Dense ( num_hidden ) ) <EOL> model . add ( Activation ( '<STR_LIT:relu>' ) ) <EOL> model . add ( Dropout ( dropout ) ) <EOL> for i in range ( extra_layers ) : <EOL> model . add ( Dense ( num_hidden ) ) <EOL> model . add ( Activation ( '<STR_LIT:relu>' ) ) <EOL> model . add ( Dropout ( dropout ) ) <EOL> model . add ( Dense ( num_classes ) ) <EOL> model . add ( Activation ( '<STR_LIT>' ) ) <EOL> model . compile ( loss = '<STR_LIT>' , optimizer = '<STR_LIT>' ) <EOL> plotter = Plotter ( save_to_filepath = graph_to , show_plot_window = True ) <EOL> callbacks = [ plotter ] if graph_to else [ ] <EOL> history = model . fit ( X_train , Y_train , nb_epoch = nb_epoch , batch_size = batch_size , verbose = <NUM_LIT:1> if verbose else <NUM_LIT:0> , show_accuracy = True , validation_split = <NUM_LIT:0.1> , callbacks = callbacks ) <EOL> score = model . evaluate ( X_test , Y_test , batch_size = batch_size , verbose = <NUM_LIT:1> if verbose else <NUM_LIT:0> , show_accuracy = True ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' , score [ <NUM_LIT:0> ] ) <EOL> print ( '<STR_LIT>' , score [ <NUM_LIT:1> ] ) <EOL> predictions = model . predict_classes ( X_test , verbose = <NUM_LIT:1> if verbose else <NUM_LIT:0> ) <EOL> return predictions , score [ <NUM_LIT:1> ] <EOL> def evaluate_recurrent_model ( dataset , num_classes ) : <EOL> ( X_train , Y_train ) , ( X_test , Y_test ) = dataset <EOL> max_features = <NUM_LIT> <EOL> maxlen = <NUM_LIT> <EOL> batch_size = <NUM_LIT:32> <EOL> print ( len ( X_train ) , '<STR_LIT>' ) <EOL> print ( len ( X_test ) , '<STR_LIT>' ) <EOL> print ( "<STR_LIT>" % maxlen ) <EOL> X_train = sequence . pad_sequences ( X_train , maxlen = maxlen ) <EOL> X_test = sequence . pad_sequences ( X_test , maxlen = maxlen ) <EOL> print ( '<STR_LIT>' , X_train . shape ) <EOL> print ( '<STR_LIT>' , X_test . shape ) <EOL> print ( '<STR_LIT>' ) <EOL> model = Sequential ( ) <EOL> model . add ( Embedding ( max_features , <NUM_LIT> , input_length = maxlen ) ) <EOL> model . add ( GRU ( <NUM_LIT> ) ) <EOL> model . add ( Dropout ( <NUM_LIT:0.5> ) ) <EOL> model . add ( Dense ( num_classes ) ) <EOL> model . add ( Activation ( '<STR_LIT>' ) ) <EOL> model . compile ( loss = '<STR_LIT>' , optimizer = '<STR_LIT>' ) <EOL> print ( "<STR_LIT>" ) <EOL> model . fit ( X_train , Y_train , batch_size = batch_size , nb_epoch = <NUM_LIT:15> , <EOL> validation_data = ( X_test , Y_test ) , show_accuracy = True ) <EOL> score , acc = model . evaluate ( X_test , Y_test , <EOL> batch_size = batch_size , <EOL> show_accuracy = True ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' , score ) <EOL> print ( '<STR_LIT>' , acc ) <EOL> return score [ <NUM_LIT:1> ] <EOL> def evaluate_conv_model ( dataset , num_classes , maxlen = <NUM_LIT> , embedding_dims = <NUM_LIT> , max_features = <NUM_LIT> , nb_filter = <NUM_LIT> , filter_length = <NUM_LIT:3> , num_hidden = <NUM_LIT> , dropout = <NUM_LIT> , verbose = True , pool_length = <NUM_LIT:2> , with_lstm = False ) : <EOL> ( X_train , Y_train ) , ( X_test , Y_test ) = dataset <EOL> batch_size = <NUM_LIT:32> <EOL> nb_epoch = <NUM_LIT:5> <EOL> if verbose : <EOL> print ( '<STR_LIT>' ) <EOL> print ( len ( X_train ) , '<STR_LIT>' ) <EOL> print ( len ( X_test ) , '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> X_train = sequence . pad_sequences ( X_train , maxlen = maxlen ) <EOL> X_test = sequence . pad_sequences ( X_test , maxlen = maxlen ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' , X_train . shape ) <EOL> print ( '<STR_LIT>' , X_test . shape ) <EOL> print ( '<STR_LIT>' ) <EOL> model = Sequential ( ) <EOL> model . add ( Embedding ( max_features , embedding_dims , input_length = maxlen ) ) <EOL> model . add ( Dropout ( dropout ) ) <EOL> model . add ( Convolution1D ( nb_filter = nb_filter , <EOL> filter_length = filter_length , <EOL> border_mode = '<STR_LIT>' , <EOL> activation = '<STR_LIT:relu>' , <EOL> subsample_length = <NUM_LIT:1> ) ) <EOL> if pool_length : <EOL> model . add ( MaxPooling1D ( pool_length = <NUM_LIT:2> ) ) <EOL> if with_lstm : <EOL> model . add ( LSTM ( <NUM_LIT> ) ) <EOL> else : <EOL> model . add ( Flatten ( ) ) <EOL> model . add ( Dense ( num_hidden ) ) <EOL> model . add ( Activation ( '<STR_LIT:relu>' ) ) <EOL> model . add ( Dropout ( dropout ) ) <EOL> model . add ( Dense ( num_classes ) ) <EOL> model . add ( Activation ( '<STR_LIT>' ) ) <EOL> model . compile ( loss = '<STR_LIT>' , optimizer = '<STR_LIT>' ) <EOL> model . fit ( X_train , Y_train , batch_size = batch_size , nb_epoch = nb_epoch , show_accuracy = True , validation_split = <NUM_LIT:0.1> ) <EOL> score = model . evaluate ( X_test , Y_test , batch_size = batch_size , verbose = <NUM_LIT:1> if verbose else <NUM_LIT:0> , show_accuracy = True ) <EOL> if verbose : <EOL> print ( '<STR_LIT>' , score [ <NUM_LIT:0> ] ) <EOL> print ( '<STR_LIT>' , score [ <NUM_LIT:1> ] ) <EOL> predictions = model . predict_classes ( X_test , verbose = <NUM_LIT:1> if verbose else <NUM_LIT:0> ) <EOL> return predictions , score [ <NUM_LIT:1> ] </s>
