<s> from collections import defaultdict , Counter <EOL> from math import log <EOL> from csv import DictWriter <EOL> import argparse <EOL> import operator <EOL> from numpy import percentile <EOL> kMAX_VAL = <NUM_LIT:1000> <EOL> def lg ( x ) : <EOL> return log ( x ) / log ( <NUM_LIT> ) <EOL> def entropy ( x ) : <EOL> if x == <NUM_LIT:0.0> or x == <NUM_LIT:1.0> : <EOL> return float ( "<STR_LIT>" ) <EOL> else : <EOL> return - ( x * lg ( x ) + ( <NUM_LIT:1.> - x ) * lg ( <NUM_LIT:1.> - x ) ) <EOL> class PurgingCounter ( Counter ) : <EOL> def __init__ ( self , max_size = <NUM_LIT> ) : <EOL> self . _max = max_size <EOL> def __setitem__ ( self , item , val ) : <EOL> if len ( self ) > self . _max * <NUM_LIT:2> : <EOL> print ( "<STR_LIT>" ) <EOL> to_delete = self . most_common ( ) [ self . _max : ] <EOL> for ii in to_delete : <EOL> del self [ ii ] <EOL> Counter . __setitem__ ( self , item , val ) <EOL> class VwDiscreteReader : <EOL> def __init__ ( self , input_file , name , label , num_vals = <NUM_LIT:100> , max_sent = <NUM_LIT:5> , <EOL> min_freq = <NUM_LIT:3> ) : <EOL> self . _observations = defaultdict ( PurgingCounter ) <EOL> self . _input_file = input_file <EOL> self . _label_file = label <EOL> self . _name = name <EOL> self . _max_sent = max_sent <EOL> self . _num_vals = num_vals <EOL> self . _min_freq = min_freq <EOL> self . _fields = [ "<STR_LIT>" , "<STR_LIT:name>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> def read_file ( self ) : <EOL> self . _num_true = defaultdict ( int ) <EOL> self . _num_false = defaultdict ( int ) <EOL> for feat , label in zip ( open ( self . _input_file ) , <EOL> open ( self . _label_file ) ) : <EOL> fields = label . split ( ) <EOL> label = True if int ( fields [ <NUM_LIT:0> ] ) > <NUM_LIT:0> else False <EOL> sent = int ( float ( [ x . split ( "<STR_LIT::>" ) [ <NUM_LIT:1> ] for x in fields <EOL> if x . startswith ( "<STR_LIT>" ) ] [ <NUM_LIT:0> ] ) ) <EOL> if label : <EOL> self . _num_true [ sent ] += <NUM_LIT:1> <EOL> else : <EOL> self . _num_false [ sent ] += <NUM_LIT:1> <EOL> for jj in [ x for x in feat . split ( ) <EOL> if not "<STR_LIT::>" in x and not "<STR_LIT:|>" in x ] : <EOL> self . _observations [ ( label , sent ) ] [ jj ] += <NUM_LIT:1> <EOL> def __iter__ ( self ) : <EOL> scores = defaultdict ( dict ) <EOL> base_entropy = { } <EOL> total = { } <EOL> for ss in self . _num_true : <EOL> base_entropy [ ss ] = entropy ( self . _num_true [ ss ] / <EOL> float ( self . _num_false [ ss ] <EOL> + self . _num_true [ ss ] ) ) <EOL> total [ ss ] = self . _num_true [ ss ] + self . _num_false [ ss ] <EOL> keys = self . _observations . keys ( ) <EOL> for label , sent in keys : <EOL> for feat in self . _observations [ ( label , sent ) ] : <EOL> feature_total = self . _observations [ ( True , sent ) ] [ feat ] + self . _observations [ ( False , sent ) ] [ feat ] <EOL> if feature_total == <NUM_LIT:0> or sent > self . _max_sent : <EOL> continue <EOL> present_prob = self . _observations [ ( True , sent ) ] [ feat ] / float ( feature_total ) <EOL> present_entropy = entropy ( present_prob ) <EOL> absent_prob = ( self . _num_true [ sent ] - <EOL> self . _observations [ ( True , sent ) ] [ feat ] ) / float ( total [ sent ] - feature_total ) <EOL> absent_entropy = entropy ( absent_prob ) <EOL> if present_entropy < <NUM_LIT:0.0> and absent_entropy < <NUM_LIT:0.0> : <EOL> continue <EOL> ig = base_entropy [ sent ] - feature_total / float ( total [ sent ] ) * present_entropy - ( total [ sent ] - feature_total ) / float ( total [ sent ] ) * absent_entropy <EOL> if feat in scores [ sent ] : <EOL> continue <EOL> else : <EOL> scores [ sent ] [ feat ] = min ( kMAX_VAL , ig ) <EOL> for sent in scores : <EOL> count = <NUM_LIT:0> <EOL> for feat , val in sorted ( scores [ sent ] . items ( ) , <EOL> key = operator . itemgetter ( <NUM_LIT:1> ) ) : <EOL> freq = self . _observations [ ( True , sent ) ] [ feat ] + self . _observations [ ( False , sent ) ] [ feat ] <EOL> if freq < self . _min_freq : <EOL> continue <EOL> d = { } <EOL> d [ "<STR_LIT>" ] = freq <EOL> d [ "<STR_LIT>" ] = self . _name <EOL> d [ "<STR_LIT:name>" ] = feat <EOL> d [ "<STR_LIT>" ] = val <EOL> d [ "<STR_LIT>" ] = sent <EOL> count += <NUM_LIT:1> <EOL> yield d <EOL> if count > self . _num_vals : <EOL> break <EOL> class VwContReader : <EOL> def __init__ ( self , input_file , name , label , sent_max = <NUM_LIT:5> , stat_buffer = <NUM_LIT> , <EOL> low_percentile = <NUM_LIT> , high_percentile = <NUM_LIT> ) : <EOL> self . _discrete = defaultdict ( int ) <EOL> self . _continuous = defaultdict ( set ) <EOL> self . _name = name <EOL> self . _input_file = input_file <EOL> self . _label_file = label <EOL> self . _sent_limit = sent_max <EOL> self . _stat_buffer = stat_buffer <EOL> self . _low_percentile = low_percentile <EOL> self . _high_percentile = high_percentile <EOL> self . _fields = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:value>" , "<STR_LIT>" , "<STR_LIT:name>" ] <EOL> def __iter__ ( self ) : <EOL> examples_read = <NUM_LIT:0> <EOL> stats_buffer = defaultdict ( list ) <EOL> for feat , label in zip ( open ( self . _input_file ) , <EOL> open ( self . _label_file ) ) : <EOL> examples_read += <NUM_LIT:1> <EOL> for ii in [ x for x in feat . split ( ) if "<STR_LIT::>" in x ] : <EOL> feature , val = ii . split ( "<STR_LIT::>" ) <EOL> val = float ( val ) <EOL> stats_buffer [ feature ] . append ( val ) <EOL> if examples_read > self . _stat_buffer and all ( len ( stats_buffer [ x ] ) > self . _stat_buffer <EOL> for x in stats_buffer ) : <EOL> break <EOL> print ( list ( list ( set ( stats_buffer [ x ] ) ) [ : <NUM_LIT:10> ] for x in stats_buffer ) ) <EOL> highs = { } <EOL> lows = { } <EOL> for ii in stats_buffer : <EOL> lows [ ii ] = percentile ( stats_buffer [ ii ] , self . _low_percentile ) <EOL> highs [ ii ] = percentile ( stats_buffer [ ii ] , self . _high_percentile ) <EOL> print ( lows ) <EOL> print ( highs ) <EOL> for feat , label in zip ( open ( self . _input_file ) , <EOL> open ( self . _label_file ) ) : <EOL> fields = label . split ( ) <EOL> d = { } <EOL> d [ "<STR_LIT>" ] = "<STR_LIT>" if int ( fields [ <NUM_LIT:0> ] ) > <NUM_LIT:0> else "<STR_LIT>" <EOL> d [ "<STR_LIT>" ] = float ( [ x . split ( "<STR_LIT::>" ) [ <NUM_LIT:1> ] for x in fields <EOL> if x . startswith ( "<STR_LIT>" ) ] [ <NUM_LIT:0> ] ) <EOL> if d [ "<STR_LIT>" ] > self . _sent_limit : <EOL> continue <EOL> guess_index = fields . index ( "<STR_LIT>" ) + <NUM_LIT:1> <EOL> d [ "<STR_LIT>" ] = fields [ guess_index ] <EOL> d [ "<STR_LIT>" ] = self . _name <EOL> for ii in [ x for x in feat . split ( ) if "<STR_LIT::>" in x ] : <EOL> feature , val = ii . split ( "<STR_LIT::>" ) <EOL> val = float ( val ) <EOL> d [ "<STR_LIT:name>" ] = feature <EOL> d [ "<STR_LIT:value>" ] = val <EOL> if val > lows [ feature ] and val < highs [ feature ] : <EOL> yield d <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> parser = argparse . ArgumentParser ( description = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , nargs = '<STR_LIT:*>' , default = [ ] , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , type = str , default = '<STR_LIT>' ) <EOL> parser . add_argument ( "<STR_LIT>" , type = str , <EOL> default = "<STR_LIT>" , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_argument ( "<STR_LIT>" , type = str , <EOL> default = "<STR_LIT>" , <EOL> help = "<STR_LIT>" ) <EOL> flags = parser . parse_args ( ) <EOL> discrete = { } <EOL> continuous = { } <EOL> for feat in flags . feats : <EOL> file = feat <EOL> feat = feat . replace ( "<STR_LIT>" , "<STR_LIT>" ) . rsplit ( "<STR_LIT:.>" , <NUM_LIT:1> ) [ - <NUM_LIT:1> ] <EOL> print ( file , feat ) <EOL> discrete [ feat ] = VwDiscreteReader ( file , feat , flags . label ) <EOL> discrete [ feat ] . read_file ( ) <EOL> continuous [ feat ] = VwContReader ( file , feat , flags . label ) <EOL> o = DictWriter ( open ( flags . output_disc , '<STR_LIT:w>' ) , <EOL> fieldnames = discrete . values ( ) [ <NUM_LIT:0> ] . _fields ) <EOL> o . writeheader ( ) <EOL> for ii in discrete : <EOL> for jj in discrete [ ii ] : <EOL> o . writerow ( jj ) <EOL> o = DictWriter ( open ( flags . output_cont , '<STR_LIT:w>' ) , <EOL> fieldnames = continuous . values ( ) [ <NUM_LIT:0> ] . _fields ) <EOL> o . writeheader ( ) <EOL> for ii in continuous : <EOL> for jj in continuous [ ii ] : <EOL> o . writerow ( jj ) </s>
