<s> import theano . tensor as TT <EOL> import theano <EOL> import scipy . optimize <EOL> from rllab . misc import logger <EOL> from rllab . misc . overrides import overrides <EOL> from rllab . misc import ext <EOL> from rllab . algos . batch_polopt import BatchPolopt <EOL> from rllab . core . serializable import Serializable <EOL> import numpy as np <EOL> from rllab . misc import tensor_utils <EOL> class REPS ( BatchPolopt , Serializable ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( <EOL> self , <EOL> epsilon = <NUM_LIT:0.5> , <EOL> L2_reg_dual = <NUM_LIT:0.> , <EOL> L2_reg_loss = <NUM_LIT:0.> , <EOL> max_opt_itr = <NUM_LIT:50> , <EOL> optimizer = scipy . optimize . fmin_l_bfgs_b , <EOL> ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> Serializable . quick_init ( self , locals ( ) ) <EOL> super ( REPS , self ) . __init__ ( ** kwargs ) <EOL> self . epsilon = epsilon <EOL> self . L2_reg_dual = L2_reg_dual <EOL> self . L2_reg_loss = L2_reg_loss <EOL> self . max_opt_itr = max_opt_itr <EOL> self . optimizer = optimizer <EOL> self . opt_info = None <EOL> @ overrides <EOL> def init_opt ( self ) : <EOL> is_recurrent = int ( self . policy . recurrent ) <EOL> self . param_eta = <NUM_LIT> <EOL> self . param_v = np . random . rand ( self . env . observation_space . flat_dim * <NUM_LIT:2> + <NUM_LIT:4> ) <EOL> obs_var = self . env . observation_space . new_tensor_variable ( <EOL> '<STR_LIT>' , <EOL> extra_dims = <NUM_LIT:1> + is_recurrent , <EOL> ) <EOL> action_var = self . env . action_space . new_tensor_variable ( <EOL> '<STR_LIT:action>' , <EOL> extra_dims = <NUM_LIT:1> + is_recurrent , <EOL> ) <EOL> rewards = ext . new_tensor ( <EOL> '<STR_LIT>' , <EOL> ndim = <NUM_LIT:1> + is_recurrent , <EOL> dtype = theano . config . floatX , <EOL> ) <EOL> feat_diff = ext . new_tensor ( <EOL> '<STR_LIT>' , <EOL> ndim = <NUM_LIT:2> + is_recurrent , <EOL> dtype = theano . config . floatX <EOL> ) <EOL> param_v = TT . vector ( '<STR_LIT>' ) <EOL> param_eta = TT . scalar ( '<STR_LIT>' ) <EOL> valid_var = TT . matrix ( '<STR_LIT>' ) <EOL> state_info_vars = { <EOL> k : ext . new_tensor ( <EOL> k , <EOL> ndim = <NUM_LIT:2> + is_recurrent , <EOL> dtype = theano . config . floatX <EOL> ) for k in self . policy . state_info_keys <EOL> } <EOL> state_info_vars_list = [ state_info_vars [ k ] for k in self . policy . state_info_keys ] <EOL> dist_info_vars = self . policy . dist_info_sym ( obs_var , state_info_vars ) <EOL> dist = self . policy . distribution <EOL> logli = dist . log_likelihood_sym ( action_var , dist_info_vars ) <EOL> delta_v = rewards + TT . dot ( feat_diff , param_v ) <EOL> if is_recurrent : <EOL> loss = - TT . sum ( logli * TT . exp ( <EOL> delta_v / param_eta - TT . max ( delta_v / param_eta ) <EOL> ) * valid_var ) / TT . sum ( valid_var ) <EOL> else : <EOL> loss = - TT . mean ( logli * TT . exp ( <EOL> delta_v / param_eta - TT . max ( delta_v / param_eta ) <EOL> ) ) <EOL> reg_params = self . policy . get_params ( regularizable = True ) <EOL> loss += self . L2_reg_loss * TT . sum ( <EOL> [ TT . mean ( TT . square ( param ) ) for param in reg_params ] <EOL> ) / len ( reg_params ) <EOL> loss_grad = TT . grad ( <EOL> loss , self . policy . get_params ( trainable = True ) ) <EOL> if is_recurrent : <EOL> recurrent_vars = [ valid_var ] <EOL> else : <EOL> recurrent_vars = [ ] <EOL> input = [ rewards , obs_var , feat_diff , <EOL> action_var ] + state_info_vars_list + recurrent_vars + [ param_eta , param_v ] <EOL> f_loss = ext . compile_function ( <EOL> inputs = input , <EOL> outputs = loss , <EOL> ) <EOL> f_loss_grad = ext . compile_function ( <EOL> inputs = input , <EOL> outputs = loss_grad , <EOL> ) <EOL> old_dist_info_vars = { <EOL> k : ext . new_tensor ( <EOL> '<STR_LIT>' % k , <EOL> ndim = <NUM_LIT:2> + is_recurrent , <EOL> dtype = theano . config . floatX <EOL> ) for k in dist . dist_info_keys <EOL> } <EOL> old_dist_info_vars_list = [ old_dist_info_vars [ k ] for k in dist . dist_info_keys ] <EOL> if is_recurrent : <EOL> mean_kl = TT . sum ( dist . kl_sym ( old_dist_info_vars , dist_info_vars ) * valid_var ) / TT . sum ( valid_var ) <EOL> else : <EOL> mean_kl = TT . mean ( dist . kl_sym ( old_dist_info_vars , dist_info_vars ) ) <EOL> f_kl = ext . compile_function ( <EOL> inputs = [ obs_var , action_var ] + state_info_vars_list + old_dist_info_vars_list + recurrent_vars , <EOL> outputs = mean_kl , <EOL> ) <EOL> if is_recurrent : <EOL> dual = param_eta * self . epsilon + param_eta * TT . log ( <EOL> TT . sum ( <EOL> TT . exp ( <EOL> delta_v / param_eta - TT . max ( delta_v / param_eta ) <EOL> ) * valid_var <EOL> ) / TT . sum ( valid_var ) <EOL> ) + param_eta * TT . max ( delta_v / param_eta ) <EOL> else : <EOL> dual = param_eta * self . epsilon + param_eta * TT . log ( <EOL> TT . mean ( <EOL> TT . exp ( <EOL> delta_v / param_eta - TT . max ( delta_v / param_eta ) <EOL> ) <EOL> ) <EOL> ) + param_eta * TT . max ( delta_v / param_eta ) <EOL> dual += self . L2_reg_dual * ( TT . square ( param_eta ) + TT . square ( <NUM_LIT:1> / param_eta ) ) <EOL> dual_grad = TT . grad ( cost = dual , wrt = [ param_eta , param_v ] ) <EOL> f_dual = ext . compile_function ( <EOL> inputs = [ rewards , feat_diff ] + state_info_vars_list + recurrent_vars + [ param_eta , param_v ] , <EOL> outputs = dual <EOL> ) <EOL> f_dual_grad = ext . compile_function ( <EOL> inputs = [ rewards , feat_diff ] + state_info_vars_list + recurrent_vars + [ param_eta , param_v ] , <EOL> outputs = dual_grad <EOL> ) <EOL> self . opt_info = dict ( <EOL> f_loss_grad = f_loss_grad , <EOL> f_loss = f_loss , <EOL> f_dual = f_dual , <EOL> f_dual_grad = f_dual_grad , <EOL> f_kl = f_kl <EOL> ) <EOL> def _features ( self , path ) : <EOL> o = np . clip ( path [ "<STR_LIT>" ] , - <NUM_LIT:10> , <NUM_LIT:10> ) <EOL> l = len ( path [ "<STR_LIT>" ] ) <EOL> al = np . arange ( l ) . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> ) / <NUM_LIT> <EOL> return np . concatenate ( [ o , o ** <NUM_LIT:2> , al , al ** <NUM_LIT:2> , al ** <NUM_LIT:3> , np . ones ( ( l , <NUM_LIT:1> ) ) ] , axis = <NUM_LIT:1> ) <EOL> @ overrides <EOL> def optimize_policy ( self , itr , samples_data ) : <EOL> rewards = samples_data [ '<STR_LIT>' ] <EOL> actions = samples_data [ '<STR_LIT>' ] <EOL> observations = samples_data [ '<STR_LIT>' ] <EOL> agent_infos = samples_data [ "<STR_LIT>" ] <EOL> state_info_list = [ agent_infos [ k ] for k in self . policy . state_info_keys ] <EOL> dist_info_list = [ agent_infos [ k ] for k in self . policy . distribution . dist_info_keys ] <EOL> if self . policy . recurrent : <EOL> recurrent_vals = [ samples_data [ "<STR_LIT>" ] ] <EOL> else : <EOL> recurrent_vals = [ ] <EOL> feat_diff = [ ] <EOL> for path in samples_data [ '<STR_LIT>' ] : <EOL> feats = self . _features ( path ) <EOL> feats = np . vstack ( [ feats , np . zeros ( feats . shape [ <NUM_LIT:1> ] ) ] ) <EOL> feat_diff . append ( feats [ <NUM_LIT:1> : ] - feats [ : - <NUM_LIT:1> ] ) <EOL> if self . policy . recurrent : <EOL> max_path_length = max ( [ len ( path [ "<STR_LIT>" ] ) for path in samples_data [ "<STR_LIT>" ] ] ) <EOL> feat_diff = np . array ( [ tensor_utils . pad_tensor ( fd , max_path_length ) for fd in feat_diff ] ) <EOL> else : <EOL> feat_diff = np . vstack ( feat_diff ) <EOL> f_dual = self . opt_info [ '<STR_LIT>' ] <EOL> f_dual_grad = self . opt_info [ '<STR_LIT>' ] <EOL> def eval_dual ( input ) : <EOL> param_eta = input [ <NUM_LIT:0> ] <EOL> param_v = input [ <NUM_LIT:1> : ] <EOL> val = f_dual ( * ( [ rewards , feat_diff ] + state_info_list + recurrent_vals + [ param_eta , param_v ] ) ) <EOL> return val . astype ( np . float64 ) <EOL> def eval_dual_grad ( input ) : <EOL> param_eta = input [ <NUM_LIT:0> ] <EOL> param_v = input [ <NUM_LIT:1> : ] <EOL> grad = f_dual_grad ( * ( [ rewards , feat_diff ] + state_info_list + recurrent_vals + [ param_eta , param_v ] ) ) <EOL> eta_grad = np . float ( grad [ <NUM_LIT:0> ] ) <EOL> v_grad = grad [ <NUM_LIT:1> ] <EOL> return np . hstack ( [ eta_grad , v_grad ] ) <EOL> x0 = np . hstack ( [ self . param_eta , self . param_v ] ) <EOL> bounds = [ ( - np . inf , np . inf ) for _ in x0 ] <EOL> bounds [ <NUM_LIT:0> ] = ( <NUM_LIT:0.> , np . inf ) <EOL> logger . log ( '<STR_LIT>' ) <EOL> eta_before = x0 [ <NUM_LIT:0> ] <EOL> dual_before = eval_dual ( x0 ) <EOL> params_ast , _ , _ = self . optimizer ( <EOL> func = eval_dual , x0 = x0 , <EOL> fprime = eval_dual_grad , <EOL> bounds = bounds , <EOL> maxiter = self . max_opt_itr , <EOL> disp = <NUM_LIT:0> <EOL> ) <EOL> dual_after = eval_dual ( params_ast ) <EOL> self . param_eta = params_ast [ <NUM_LIT:0> ] <EOL> self . param_v = params_ast [ <NUM_LIT:1> : ] <EOL> cur_params = self . policy . get_param_values ( trainable = True ) <EOL> f_loss = self . opt_info [ "<STR_LIT>" ] <EOL> f_loss_grad = self . opt_info [ '<STR_LIT>' ] <EOL> input = [ rewards , observations , feat_diff , <EOL> actions ] + state_info_list + recurrent_vals + [ self . param_eta , self . param_v ] <EOL> def eval_loss ( params ) : <EOL> self . policy . set_param_values ( params , trainable = True ) <EOL> val = f_loss ( * input ) <EOL> return val . astype ( np . float64 ) <EOL> def eval_loss_grad ( params ) : <EOL> self . policy . set_param_values ( params , trainable = True ) <EOL> grad = f_loss_grad ( * input ) <EOL> flattened_grad = tensor_utils . flatten_tensors ( map ( np . asarray , grad ) ) <EOL> return flattened_grad . astype ( np . float64 ) <EOL> loss_before = eval_loss ( cur_params ) <EOL> logger . log ( '<STR_LIT>' ) <EOL> params_ast , _ , _ = self . optimizer ( <EOL> func = eval_loss , x0 = cur_params , <EOL> fprime = eval_loss_grad , <EOL> disp = <NUM_LIT:0> , <EOL> maxiter = self . max_opt_itr <EOL> ) <EOL> loss_after = eval_loss ( params_ast ) <EOL> f_kl = self . opt_info [ '<STR_LIT>' ] <EOL> mean_kl = f_kl ( * ( [ observations , actions ] + state_info_list + dist_info_list + recurrent_vals ) ) . astype ( <EOL> np . float64 ) <EOL> logger . log ( '<STR_LIT>' % ( eta_before , self . param_eta ) ) <EOL> logger . record_tabular ( "<STR_LIT>" , loss_before ) <EOL> logger . record_tabular ( "<STR_LIT>" , loss_after ) <EOL> logger . record_tabular ( '<STR_LIT>' , dual_before ) <EOL> logger . record_tabular ( '<STR_LIT>' , dual_after ) <EOL> logger . record_tabular ( '<STR_LIT>' , mean_kl ) <EOL> @ overrides <EOL> def get_itr_snapshot ( self , itr , samples_data ) : <EOL> return dict ( <EOL> itr = itr , <EOL> policy = self . policy , <EOL> baseline = self . baseline , <EOL> env = self . env , <EOL> ) </s>
