<s> class ExtendedFeatures : <EOL> def __init__ ( self , dataset ) : <EOL> self . feature_dic = { } <EOL> self . feature_names = [ ] <EOL> self . nr_feats = <NUM_LIT:0> <EOL> self . feature_list = [ ] <EOL> self . add_features = False <EOL> self . dataset = dataset <EOL> self . node_feature_cache = { } <EOL> self . initial_state_feature_cache = { } <EOL> self . edge_feature_cache = { } <EOL> self . final_edge_feature_cache = { } <EOL> def build_features ( self ) : <EOL> self . add_features = True <EOL> for seq in self . dataset . train . seq_list : <EOL> seq_node_features , seq_edge_features = self . get_seq_features ( seq ) <EOL> self . feature_list . append ( [ seq_node_features , seq_edge_features ] ) <EOL> self . nr_feats = len ( self . feature_names ) <EOL> self . add_features = False <EOL> def get_seq_features ( self , seq ) : <EOL> seq_node_features = [ ] <EOL> seq_edge_features = [ ] <EOL> idx = [ ] <EOL> idx = self . add_node_features ( seq , <NUM_LIT:0> , seq . y [ <NUM_LIT:0> ] , idx ) <EOL> idx = self . add_init_state_features ( seq , <NUM_LIT:0> , seq . y [ <NUM_LIT:0> ] , idx ) <EOL> seq_node_features . append ( idx ) <EOL> seq_edge_features . append ( [ ] ) <EOL> for i , tag in enumerate ( seq . y [ <NUM_LIT:1> : ] ) : <EOL> idx = [ ] <EOL> edge_idx = [ ] <EOL> j = i + <NUM_LIT:1> <EOL> prev_tag = seq . y [ j - <NUM_LIT:1> ] <EOL> edge_idx = self . add_edge_features ( seq , j , tag , prev_tag , edge_idx ) <EOL> idx = self . add_node_features ( seq , j , tag , idx ) <EOL> seq_node_features . append ( idx ) <EOL> seq_edge_features . append ( edge_idx ) <EOL> return seq_node_features , seq_edge_features <EOL> def add_node_features ( self , seq , pos , y , idx ) : <EOL> x = seq . x [ pos ] <EOL> y_name = self . dataset . int_to_pos [ y ] <EOL> word = self . dataset . int_to_word [ x ] <EOL> feat = "<STR_LIT>" % ( word , y_name ) <EOL> nr_feat = self . add_feature ( feat ) <EOL> if ( nr_feat != - <NUM_LIT:1> ) : <EOL> idx . append ( nr_feat ) <EOL> if unicode . istitle ( word ) : <EOL> feat = "<STR_LIT>" % y_name <EOL> nr_feat = self . add_feature ( feat ) <EOL> if ( nr_feat != - <NUM_LIT:1> ) : <EOL> idx . append ( nr_feat ) <EOL> if unicode . isdigit ( word ) : <EOL> feat = "<STR_LIT>" % y_name <EOL> nr_feat = self . add_feature ( feat ) <EOL> if ( nr_feat != - <NUM_LIT:1> ) : <EOL> idx . append ( nr_feat ) <EOL> if unicode . find ( word , "<STR_LIT:->" ) != - <NUM_LIT:1> : <EOL> feat = "<STR_LIT>" % y_name <EOL> nr_feat = self . add_feature ( feat ) <EOL> if ( nr_feat != - <NUM_LIT:1> ) : <EOL> idx . append ( nr_feat ) <EOL> max_suffix = <NUM_LIT:3> <EOL> for i in xrange ( max_suffix ) : <EOL> if ( len ( word ) > i + <NUM_LIT:1> ) : <EOL> suffix = word [ - ( i + <NUM_LIT:1> ) : ] <EOL> feat = "<STR_LIT>" % ( suffix , y_name ) <EOL> nr_feat = self . add_feature ( feat ) <EOL> if ( nr_feat != - <NUM_LIT:1> ) : <EOL> idx . append ( nr_feat ) <EOL> max_prefix = <NUM_LIT:3> <EOL> for i in xrange ( max_prefix ) : <EOL> if ( len ( word ) > i + <NUM_LIT:1> ) : <EOL> prefix = word [ : i + <NUM_LIT:1> ] <EOL> feat = "<STR_LIT>" % ( prefix , y_name ) <EOL> nr_feat = self . add_feature ( feat ) <EOL> if ( nr_feat != - <NUM_LIT:1> ) : <EOL> idx . append ( nr_feat ) <EOL> if ( self . dataset . word_counts [ x ] <= <NUM_LIT:5> ) : <EOL> feat = "<STR_LIT>" % ( y_name ) <EOL> nr_feat = self . add_feature ( feat ) <EOL> if ( nr_feat != - <NUM_LIT:1> ) : <EOL> idx . append ( nr_feat ) <EOL> return idx <EOL> def get_node_features ( self , seq , pos , y ) : <EOL> all_feat = [ ] <EOL> x = seq . x [ pos ] <EOL> if ( x not in self . node_feature_cache ) : <EOL> self . node_feature_cache [ x ] = { } <EOL> if ( y not in self . node_feature_cache [ x ] ) : <EOL> node_idx = [ ] <EOL> node_idx = self . add_node_features ( seq , pos , y , node_idx ) <EOL> self . node_feature_cache [ x ] [ y ] = node_idx <EOL> idx = self . node_feature_cache [ x ] [ y ] <EOL> all_feat = idx [ : ] <EOL> if ( pos == <NUM_LIT:0> ) : <EOL> if ( y not in self . initial_state_feature_cache ) : <EOL> init_idx = [ ] <EOL> init_idx = self . add_init_state_features ( seq , pos , y , init_idx ) <EOL> self . initial_state_feature_cache [ y ] = init_idx <EOL> all_feat . extend ( self . initial_state_feature_cache [ y ] ) <EOL> return all_feat <EOL> def add_init_state_features ( self , seq , pos , y , init_idx ) : <EOL> y_name = self . dataset . int_to_pos [ y ] <EOL> feat = "<STR_LIT>" % ( y_name ) <EOL> nr_feat = self . add_feature ( feat ) <EOL> if ( nr_feat != - <NUM_LIT:1> ) : <EOL> init_idx . append ( nr_feat ) <EOL> return init_idx <EOL> def add_edge_features ( self , seq , pos , y , y_prev , edge_idx ) : <EOL> y_name = self . dataset . int_to_pos [ y ] <EOL> y_prev_name = self . dataset . int_to_pos [ y_prev ] <EOL> if ( pos == len ( seq . x ) - <NUM_LIT:1> ) : <EOL> feat = "<STR_LIT>" % ( y_prev_name , y_name ) <EOL> else : <EOL> feat = "<STR_LIT>" % ( y_prev_name , y_name ) <EOL> nr_feat = self . add_feature ( feat ) <EOL> if ( nr_feat != - <NUM_LIT:1> ) : <EOL> edge_idx . append ( nr_feat ) <EOL> return edge_idx <EOL> def get_edge_features ( self , seq , pos , y , y_prev ) : <EOL> if ( pos == len ( seq . x ) - <NUM_LIT:1> ) : <EOL> if ( y not in self . final_edge_feature_cache ) : <EOL> self . final_edge_feature_cache [ y ] = { } <EOL> if ( y_prev not in self . final_edge_feature_cache [ y ] ) : <EOL> edge_idx = [ ] <EOL> edge = self . add_edge_features ( seq , pos , y , y_prev , edge_idx ) <EOL> self . final_edge_feature_cache [ y ] [ y_prev ] = edge_idx <EOL> return self . final_edge_feature_cache [ y ] [ y_prev ] <EOL> else : <EOL> if ( y not in self . edge_feature_cache ) : <EOL> self . edge_feature_cache [ y ] = { } <EOL> if ( y_prev not in self . edge_feature_cache [ y ] ) : <EOL> edge_idx = [ ] <EOL> edge = self . add_edge_features ( seq , pos , y , y_prev , edge_idx ) <EOL> self . edge_feature_cache [ y ] [ y_prev ] = edge_idx <EOL> return self . edge_feature_cache [ y ] [ y_prev ] <EOL> def add_feature ( self , feat ) : <EOL> if ( feat in self . feature_dic ) : <EOL> return self . feature_dic [ feat ] <EOL> if ( self . add_features == False ) : <EOL> return - <NUM_LIT:1> <EOL> nr_feat = len ( self . feature_dic . keys ( ) ) <EOL> self . feature_dic [ feat ] = nr_feat <EOL> self . feature_names . append ( feat ) <EOL> return nr_feat <EOL> def get_sequence_feat_str ( self , seq ) : <EOL> seq_nr = seq . nr <EOL> node_f_list = self . feature_list [ seq_nr ] [ <NUM_LIT:0> ] <EOL> edge_f_list = self . feature_list [ seq_nr ] [ <NUM_LIT:1> ] <EOL> word = seq . x [ <NUM_LIT:0> ] <EOL> word_n = self . dataset . int_to_word [ word ] <EOL> tag = seq . y [ <NUM_LIT:0> ] <EOL> tag_n = self . dataset . int_to_pos [ tag ] <EOL> txt = "<STR_LIT>" <EOL> for i , tag in enumerate ( seq . y ) : <EOL> word = seq . x [ i ] <EOL> word_n = self . dataset . int_to_word [ word ] <EOL> tag_n = self . dataset . int_to_pos [ tag ] <EOL> txt += "<STR_LIT>" % ( i , word_n , tag_n ) <EOL> for nf in node_f_list [ i ] : <EOL> txt += "<STR_LIT>" % self . feature_names [ nf ] <EOL> if ( edge_f_list [ i ] != [ ] ) : <EOL> txt += "<STR_LIT>" <EOL> for nf in edge_f_list [ i ] : <EOL> txt += "<STR_LIT>" % self . feature_names [ nf ] <EOL> txt += "<STR_LIT:\n>" <EOL> return txt <EOL> def print_sequence_features ( self , seq ) : <EOL> txt = "<STR_LIT>" <EOL> for i , tag in enumerate ( seq . y ) : <EOL> word = seq . x [ i ] <EOL> word_n = self . dataset . int_to_word [ word ] <EOL> tag_n = self . dataset . int_to_pos [ tag ] <EOL> txt += "<STR_LIT>" % ( i , word_n , tag_n ) <EOL> prev_tag = seq . y [ i - <NUM_LIT:1> ] <EOL> if ( i > <NUM_LIT:0> ) : <EOL> edge_f_list = self . get_edge_features ( seq , i , tag , prev_tag ) <EOL> else : <EOL> edge_f_list = [ ] <EOL> node_f_list = self . get_node_features ( seq , i , tag ) <EOL> for nf in node_f_list : <EOL> txt += "<STR_LIT>" % self . feature_names [ nf ] <EOL> if ( edge_f_list != [ ] ) : <EOL> txt += "<STR_LIT>" <EOL> for nf in edge_f_list : <EOL> txt += "<STR_LIT>" % self . feature_names [ nf ] <EOL> txt += "<STR_LIT:\n>" <EOL> return txt </s>
