<s> """<STR_LIT>""" <EOL> import numpy as np <EOL> from . Representation import Representation <EOL> from itertools import combinations <EOL> from rlpy . Tools import addNewElementForAllActions , PriorityQueueWithNovelty <EOL> import matplotlib . pyplot as plt <EOL> __copyright__ = "<STR_LIT>" <EOL> __credits__ = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" ] <EOL> __license__ = "<STR_LIT>" <EOL> __author__ = "<STR_LIT>" <EOL> class KernelizedFeature ( object ) : <EOL> index = - <NUM_LIT:1> <EOL> relevance = <NUM_LIT:0.> <EOL> dim = [ ] <EOL> center = None <EOL> def __init__ ( self , center , dim , kernel , index = - <NUM_LIT:1> , <EOL> base_ids = None , kernel_args = [ ] ) : <EOL> self . index = index <EOL> self . kernel_args = kernel_args <EOL> self . center = center <EOL> self . dim = dim <EOL> self . kernel = kernel <EOL> if base_ids is None : <EOL> self . base_ids = frozenset ( [ self . index ] ) <EOL> else : <EOL> self . base_ids = base_ids <EOL> def __str__ ( self ) : <EOL> res = "<STR_LIT:{>" + "<STR_LIT:U+002CU+0020>" . join ( sorted ( [ str ( i ) for i in self . base_ids ] ) ) + "<STR_LIT>" <EOL> res += "<STR_LIT:U+002CU+0020>" . join ( [ "<STR_LIT>" . format ( d + <NUM_LIT:1> , self . center [ d ] ) <EOL> for d in self . dim ] ) <EOL> return res <EOL> def output ( self , s ) : <EOL> return self . kernel ( s , self . center , self . dim , * self . kernel_args ) <EOL> class Candidate ( object ) : <EOL> """<STR_LIT>""" <EOL> activation_count = <NUM_LIT:0.> <EOL> td_error_sum = <NUM_LIT:0.> <EOL> relevance = <NUM_LIT:0.> <EOL> idx1 = - <NUM_LIT:1> <EOL> idx2 = - <NUM_LIT:1> <EOL> def __init__ ( self , idx1 , idx2 ) : <EOL> self . idx1 = idx1 <EOL> self . idx2 = idx2 <EOL> class KernelizediFDD ( Representation ) : <EOL> """<STR_LIT>""" <EOL> features = [ ] <EOL> candidates = { } <EOL> base_id_sets = set ( ) <EOL> base_feature_ids = [ ] <EOL> max_relevance = <NUM_LIT:0.> <EOL> def __init__ ( self , domain , kernel , active_threshold , discover_threshold , <EOL> kernel_args = [ ] , normalization = True , sparsify = True , <EOL> max_active_base_feat = <NUM_LIT:2> , max_base_feat_sim = <NUM_LIT> ) : <EOL> super ( KernelizediFDD , self ) . __init__ ( domain ) <EOL> self . kernel = kernel <EOL> self . kernel_args = kernel_args <EOL> self . active_threshold = active_threshold <EOL> self . discover_threshold = discover_threshold <EOL> self . normalization = normalization <EOL> self . sparsify = sparsify <EOL> self . sorted_ids = PriorityQueueWithNovelty ( ) <EOL> self . max_active_base_feat = max_active_base_feat <EOL> self . max_base_feat_sim = max_base_feat_sim <EOL> self . candidates = { } <EOL> self . features = [ ] <EOL> self . base_features_ids = [ ] <EOL> self . max_relevance = <NUM_LIT:0.> <EOL> def show_features ( self ) : <EOL> l = self . sorted_ids . toList ( ) [ : ] <EOL> key = lambda x : ( <EOL> len ( self . features [ x ] . base_ids ) , <EOL> tuple ( self . features [ x ] . dim ) , <EOL> tuple ( self . features [ x ] . center [ self . features [ x ] . dim ] ) ) <EOL> l . sort ( key = key ) <EOL> for i in l : <EOL> f = self . features [ i ] <EOL> print "<STR_LIT>" . format ( i , f ) <EOL> def plot_1d_features ( self , dimension_idx = None ) : <EOL> """<STR_LIT>""" <EOL> idx = dimension_idx <EOL> if isinstance ( idx , int ) : <EOL> idx = [ idx ] <EOL> elif idx is None : <EOL> idx = self . domain . continuous_dims <EOL> feat_list = range ( self . features_num ) <EOL> key = lambda x : ( <EOL> len ( self . features [ x ] . base_ids ) , <EOL> tuple ( self . features [ x ] . dim ) , <EOL> tuple ( self . features [ x ] . center [ self . features [ x ] . dim ] ) ) <EOL> feat_list . sort ( key = key ) <EOL> last_i = - <NUM_LIT:1> <EOL> for k in feat_list : <EOL> if len ( self . features [ k ] . dim ) > <NUM_LIT:1> : <EOL> break <EOL> cur_i = self . features [ k ] . dim [ <NUM_LIT:0> ] <EOL> if cur_i != last_i : <EOL> if last_i in idx : <EOL> plt . draw ( ) <EOL> if cur_i in idx : <EOL> xi = np . linspace ( <EOL> self . domain . statespace_limits [ <EOL> cur_i , <EOL> <NUM_LIT:0> ] , <EOL> self . domain . statespace_limits [ <EOL> cur_i , <EOL> <NUM_LIT:1> ] , <EOL> <NUM_LIT:200> ) <EOL> x = np . zeros ( ( <NUM_LIT:200> , self . domain . statespace_limits . shape [ <NUM_LIT:0> ] ) ) <EOL> x [ : , cur_i ] = xi <EOL> plt . figure ( "<STR_LIT>" . format ( cur_i ) ) <EOL> if cur_i in idx : <EOL> y = [ self . features [ k ] . output ( xk ) for xk in x ] <EOL> plt . plot ( x , y , label = "<STR_LIT>" . format ( k ) ) <EOL> last_i = cur_i <EOL> plt . draw ( ) <EOL> def plot_2d_features ( self , d1 = None , d2 = None , n_lines = <NUM_LIT:3> ) : <EOL> """<STR_LIT>""" <EOL> if d1 is None and d2 is None : <EOL> idx = self . domain . continuous_dims [ : <NUM_LIT:2> ] <EOL> else : <EOL> idx = [ d1 , d2 ] <EOL> idx . sort ( ) <EOL> feat_list = range ( self . features_num ) <EOL> key = lambda x : ( <EOL> len ( self . features [ x ] . base_ids ) , <EOL> tuple ( self . features [ x ] . dim ) , <EOL> tuple ( self . features [ x ] . center [ self . features [ x ] . dim ] ) ) <EOL> feat_list . sort ( key = key ) <EOL> last_i = - <NUM_LIT:1> <EOL> last_j = - <NUM_LIT:1> <EOL> for k in feat_list : <EOL> if len ( self . features [ k ] . dim ) < <NUM_LIT:2> : <EOL> continue <EOL> elif len ( self . features [ k ] . dim ) > <NUM_LIT:2> : <EOL> break <EOL> cur_i = self . features [ k ] . dim [ <NUM_LIT:0> ] <EOL> cur_j = self . features [ k ] . dim [ <NUM_LIT:1> ] <EOL> if cur_i != last_i or cur_j != last_j : <EOL> if last_i in idx and last_j in idx : <EOL> plt . draw ( ) <EOL> if cur_i in idx and cur_j in idx : <EOL> xi = np . linspace ( <EOL> self . domain . statespace_limits [ <EOL> cur_i , <EOL> <NUM_LIT:0> ] , <EOL> self . domain . statespace_limits [ <EOL> cur_i , <EOL> <NUM_LIT:1> ] , <EOL> <NUM_LIT:100> ) <EOL> xj = np . linspace ( <EOL> self . domain . statespace_limits [ <EOL> cur_j , <EOL> <NUM_LIT:0> ] , <EOL> self . domain . statespace_limits [ <EOL> cur_j , <EOL> <NUM_LIT:1> ] , <EOL> <NUM_LIT:100> ) <EOL> X , Y = np . meshgrid ( xi , xj ) <EOL> plt . figure ( <EOL> "<STR_LIT>" . format ( cur_i , cur_j ) ) <EOL> if cur_i in idx and cur_j in idx : <EOL> Z = np . zeros_like ( X ) <EOL> for m in xrange ( <NUM_LIT:100> ) : <EOL> for n in xrange ( <NUM_LIT:100> ) : <EOL> x = np . zeros ( self . domain . statespace_limits . shape [ <NUM_LIT:0> ] ) <EOL> x [ cur_i ] = X [ m , n ] <EOL> x [ cur_j ] = Y [ m , n ] <EOL> Z [ m , n ] = self . features [ k ] . output ( x ) <EOL> plt . contour ( X , Y , Z , n_lines ) <EOL> last_i = cur_i <EOL> last_j = cur_j <EOL> plt . draw ( ) <EOL> def plot_2d_feature_centers ( self , d1 = None , d2 = None ) : <EOL> """<STR_LIT>""" <EOL> if d1 is None and d2 is None : <EOL> idx = self . domain . continuous_dims [ : <NUM_LIT:2> ] <EOL> else : <EOL> idx = [ d1 , d2 ] <EOL> idx . sort ( ) <EOL> feat_list = range ( self . features_num ) <EOL> key = lambda x : ( <EOL> len ( self . features [ x ] . base_ids ) , <EOL> tuple ( self . features [ x ] . dim ) , <EOL> tuple ( self . features [ x ] . center [ self . features [ x ] . dim ] ) ) <EOL> feat_list . sort ( key = key ) <EOL> last_i = - <NUM_LIT:1> <EOL> last_j = - <NUM_LIT:1> <EOL> for k in feat_list : <EOL> if len ( self . features [ k ] . dim ) < <NUM_LIT:2> : <EOL> continue <EOL> elif len ( self . features [ k ] . dim ) > <NUM_LIT:2> : <EOL> break <EOL> cur_i = self . features [ k ] . dim [ <NUM_LIT:0> ] <EOL> cur_j = self . features [ k ] . dim [ <NUM_LIT:1> ] <EOL> if cur_i != last_i or cur_j != last_j : <EOL> if last_i in idx and last_j in idx : <EOL> plt . draw ( ) <EOL> if cur_i in idx and cur_j in idx : <EOL> plt . figure ( <EOL> "<STR_LIT>" . format ( cur_i , cur_j ) ) <EOL> if cur_i in idx and cur_j in idx : <EOL> plt . plot ( <EOL> [ self . features [ k ] . center [ cur_i ] ] , <EOL> [ self . features [ k ] . center [ cur_j ] ] , <EOL> "<STR_LIT:r>" , <EOL> marker = "<STR_LIT:x>" ) <EOL> last_i = cur_i <EOL> last_j = cur_j <EOL> plt . draw ( ) <EOL> def phi_nonTerminal ( self , s ) : <EOL> out = np . zeros ( self . features_num ) <EOL> if not self . sparsify : <EOL> for i in xrange ( self . features_num ) : <EOL> out [ i ] = self . features [ i ] . output ( s ) <EOL> else : <EOL> active_bases = set ( [ ] ) <EOL> for i in self . sorted_ids . toList ( ) [ : : - <NUM_LIT:1> ] : <EOL> if len ( self . features [ i ] . base_ids ) > <NUM_LIT:1> : <EOL> break <EOL> if self . features [ i ] . output ( s ) >= self . active_threshold : <EOL> active_bases . add ( i ) <EOL> base_vals = { k : <NUM_LIT:1.> for k in active_bases } <EOL> for i in self . sorted_ids . toList ( ) : <EOL> if active_bases . issuperset ( self . features [ i ] . base_ids ) : <EOL> if self . sparsify > <NUM_LIT:1> : <EOL> out [ i ] = self . features [ i ] . output ( s ) <EOL> if self . sparsify > <NUM_LIT:2> or out [ i ] >= self . active_threshold : <EOL> active_bases -= self . features [ i ] . base_ids <EOL> else : <EOL> u = <NUM_LIT:0> <EOL> for k in self . features [ i ] . base_ids : <EOL> u = max ( u , base_vals [ k ] ) <EOL> out [ i ] = self . features [ i ] . output ( s ) * u <EOL> for k in self . features [ i ] . base_ids : <EOL> base_vals [ k ] -= out [ i ] <EOL> if base_vals [ k ] < <NUM_LIT:0> : <EOL> active_bases . remove ( k ) <EOL> if self . normalization : <EOL> summ = out . sum ( ) <EOL> if summ != <NUM_LIT:0> : <EOL> out /= out . sum ( ) <EOL> return out <EOL> def phi_raw ( self , s , terminal ) : <EOL> assert ( terminal is False ) <EOL> out = np . zeros ( self . features_num ) <EOL> for i in xrange ( self . features_num ) : <EOL> out [ i ] = self . features [ i ] . output ( s ) <EOL> return out <EOL> def post_discover ( self , s , terminal , a , td_error , phi_s = None ) : <EOL> if phi_s is None : <EOL> phi_s = self . phi ( s , terminal ) <EOL> phi_s_unnorm = self . phi_raw ( s , terminal ) <EOL> discovered = <NUM_LIT:0> <EOL> Q = self . Qs ( s , terminal , phi_s = phi_s ) . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> active_indices = list ( <EOL> np . where ( phi_s_unnorm > self . active_threshold ) [ <NUM_LIT:0> ] ) <EOL> active_dimensions = np . zeros ( ( len ( s ) ) , dtype = "<STR_LIT:int>" ) <EOL> closest_neighbor = np . zeros ( ( len ( s ) ) ) <EOL> for i in active_indices : <EOL> for j in self . features [ i ] . dim : <EOL> active_dimensions [ j ] += <NUM_LIT:1> <EOL> closest_neighbor [ j ] = max ( closest_neighbor [ j ] , phi_s_unnorm [ i ] ) <EOL> for j in xrange ( len ( s ) ) : <EOL> if active_dimensions [ j ] < self . max_active_base_feat and ( closest_neighbor [ j ] < self . max_base_feat_sim or active_dimensions [ j ] < <NUM_LIT:1> ) : <EOL> active_indices . append ( self . add_base_feature ( s , j , Q = Q ) ) <EOL> discovered += <NUM_LIT:1> <EOL> if discovered : <EOL> phi_s = self . phi ( s , terminal ) <EOL> la = len ( active_indices ) <EOL> if la * ( la - <NUM_LIT:1> ) < len ( self . candidates ) : <EOL> for ind , cand in self . candidates . items ( ) : <EOL> g , h = ind <EOL> rel = self . update_relevance_stat ( <EOL> cand , <EOL> g , <EOL> h , <EOL> td_error , <EOL> s , <EOL> a , <EOL> phi_s ) <EOL> self . max_relevance = max ( rel , self . max_relevance ) <EOL> if rel > self . discover_threshold : <EOL> self . add_refined_feature ( g , h , Q = Q ) <EOL> discovered += <NUM_LIT:1> <EOL> else : <EOL> for g , h in combinations ( active_indices , <NUM_LIT:2> ) : <EOL> cand = self . candidates . get ( ( g , h ) ) <EOL> if cand is None : <EOL> continue <EOL> rel = self . update_relevance_stat ( <EOL> cand , <EOL> g , <EOL> h , <EOL> td_error , <EOL> s , <EOL> a , <EOL> phi_s ) <EOL> self . max_relevance = max ( rel , self . max_relevance ) <EOL> if rel > self . discover_threshold : <EOL> self . add_refined_feature ( g , h , Q = Q ) <EOL> discovered += <NUM_LIT:1> <EOL> if discovered : <EOL> self . max_relevance = <NUM_LIT:0.> <EOL> return discovered <EOL> def update_relevance_stat ( <EOL> self , candidate , index1 , index2 , td_error , s , a , phi_s ) : <EOL> """<STR_LIT>""" <EOL> candidate . td_error_sum += phi_s [ index1 ] * phi_s [ index2 ] * td_error <EOL> candidate . activation_count += phi_s [ index1 ] ** <NUM_LIT:2> * phi_s [ index2 ] ** <NUM_LIT:2> <EOL> if candidate . activation_count == <NUM_LIT:0.> : <EOL> return <NUM_LIT:0.> <EOL> rel = np . abs ( candidate . td_error_sum ) / np . sqrt ( candidate . activation_count ) <EOL> return rel <EOL> def add_base_feature ( self , center , dim , Q ) : <EOL> """<STR_LIT>""" <EOL> new_f = KernelizedFeature ( <EOL> center = center , dim = [ dim ] , kernel_args = self . kernel_args , <EOL> kernel = self . kernel , index = self . features_num ) <EOL> self . features . append ( new_f ) <EOL> self . base_id_sets . add ( new_f . base_ids ) <EOL> self . sorted_ids . push ( - <NUM_LIT:1> , self . features_num ) <EOL> self . logger . debug ( <EOL> "<STR_LIT>" . format ( <EOL> self . features_num , <EOL> new_f ) ) <EOL> new_cand = { ( f , self . features_num ) : Candidate ( f , self . features_num ) <EOL> for f in xrange ( self . features_num ) if dim not in self . features [ f ] . dim } <EOL> self . candidates . update ( new_cand ) <EOL> for f , _ in new_cand . keys ( ) : <EOL> self . base_id_sets . add ( new_f . base_ids | self . features [ f ] . base_ids ) <EOL> self . features_num += <NUM_LIT:1> <EOL> if self . normalization : <EOL> self . weight_vec = addNewElementForAllActions ( <EOL> self . weight_vec , <EOL> self . domain . actions_num , <EOL> Q ) <EOL> else : <EOL> self . weight_vec = addNewElementForAllActions ( <EOL> self . weight_vec , <EOL> self . domain . actions_num ) <EOL> return self . features_num - <NUM_LIT:1> <EOL> def add_refined_feature ( self , index1 , index2 , Q ) : <EOL> """<STR_LIT>""" <EOL> f1 = self . features [ index1 ] <EOL> f2 = self . features [ index2 ] <EOL> new_center = np . zeros_like ( f1 . center ) <EOL> cnt = np . zeros_like ( f1 . center ) <EOL> cnt [ f1 . dim ] += <NUM_LIT:1> <EOL> cnt [ f2 . dim ] += <NUM_LIT:1> <EOL> cnt [ cnt == <NUM_LIT:0> ] = <NUM_LIT:1.> <EOL> new_center [ f1 . dim ] += f1 . center [ f1 . dim ] <EOL> new_center [ f2 . dim ] += f2 . center [ f2 . dim ] <EOL> new_center /= cnt <EOL> new_dim = list ( frozenset ( f1 . dim ) | frozenset ( f2 . dim ) ) <EOL> new_base_ids = f1 . base_ids | f2 . base_ids <EOL> new_dim . sort ( ) <EOL> new_f = KernelizedFeature ( center = new_center , dim = new_dim , <EOL> kernel_args = self . kernel_args , <EOL> kernel = self . kernel , index = self . features_num , <EOL> base_ids = new_base_ids ) <EOL> self . features . append ( new_f ) <EOL> self . sorted_ids . push ( - len ( new_f . base_ids ) , self . features_num ) <EOL> self . base_id_sets . add ( new_f . base_ids ) <EOL> del self . candidates [ ( index1 , index2 ) ] <EOL> new_cand = { ( f , self . features_num ) : Candidate ( f , self . features_num ) for f in xrange ( self . features_num ) <EOL> if ( self . features [ f ] . base_ids | new_base_ids ) not in self . base_id_sets <EOL> and len ( frozenset ( self . features [ f ] . dim ) & frozenset ( new_dim ) ) == <NUM_LIT:0> } <EOL> for c , _ in new_cand . keys ( ) : <EOL> self . base_id_sets . add ( new_base_ids | self . features [ c ] . base_ids ) <EOL> self . candidates . update ( new_cand ) <EOL> self . logger . debug ( <EOL> "<STR_LIT>" . format ( <EOL> self . features_num , <EOL> new_f ) ) <EOL> self . logger . debug ( "<STR_LIT>" . format ( len ( self . candidates ) ) ) <EOL> self . features_num += <NUM_LIT:1> <EOL> if self . normalization : <EOL> self . weight_vec = addNewElementForAllActions ( <EOL> self . weight_vec , <EOL> self . domain . actions_num , <EOL> Q ) <EOL> else : <EOL> self . weight_vec = addNewElementForAllActions ( <EOL> self . weight_vec , <EOL> self . domain . actions_num ) <EOL> return self . features_num - <NUM_LIT:1> <EOL> try : <EOL> from kernels import * <EOL> except ImportError : <EOL> print "<STR_LIT>" <EOL> from slow_kernels import * </s>
