<s> """<STR_LIT>""" <EOL> import numpy as np <EOL> import theano <EOL> import theano . tensor as T <EOL> from . updates import deepmind_rmsprop <EOL> from . . base_classes import QNetwork <EOL> from . theano_layers import ConvolutionalLayer , HiddenLayer <EOL> class MyQNetwork ( QNetwork ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , environment , rho , rms_epsilon , momentum , clip_delta , freeze_interval , batchSize , network_type , <EOL> update_rule , batch_accumulator , randomState , DoubleQ = False ) : <EOL> """<STR_LIT>""" <EOL> QNetwork . __init__ ( self , environment , batchSize ) <EOL> self . rho = rho <EOL> self . rms_epsilon = rms_epsilon <EOL> self . momentum = momentum <EOL> self . clip_delta = clip_delta <EOL> self . freeze_interval = freeze_interval <EOL> self . _randomState = randomState <EOL> self . _DoubleQ = DoubleQ <EOL> self . update_counter = <NUM_LIT:0> <EOL> states = [ ] <EOL> next_states = [ ] <EOL> self . states_shared = [ ] <EOL> self . next_states_shared = [ ] <EOL> for i , dim in enumerate ( self . _inputDimensions ) : <EOL> if len ( dim ) == <NUM_LIT:3> : <EOL> states . append ( T . tensor4 ( "<STR_LIT>" % ( "<STR_LIT:state>" , i ) ) ) <EOL> next_states . append ( T . tensor4 ( "<STR_LIT>" % ( "<STR_LIT>" , i ) ) ) <EOL> elif len ( dim ) == <NUM_LIT:2> : <EOL> states . append ( T . tensor3 ( "<STR_LIT>" % ( "<STR_LIT:state>" , i ) ) ) <EOL> next_states . append ( T . tensor3 ( "<STR_LIT>" % ( "<STR_LIT>" , i ) ) ) <EOL> elif len ( dim ) == <NUM_LIT:1> : <EOL> states . append ( T . matrix ( "<STR_LIT>" % ( "<STR_LIT:state>" , i ) ) ) <EOL> next_states . append ( T . matrix ( "<STR_LIT>" % ( "<STR_LIT>" , i ) ) ) <EOL> self . states_shared . append ( theano . shared ( np . zeros ( ( batchSize , ) + dim , dtype = theano . config . floatX ) , borrow = False ) ) <EOL> self . next_states_shared . append ( theano . shared ( np . zeros ( ( batchSize , ) + dim , dtype = theano . config . floatX ) , borrow = False ) ) <EOL> print ( "<STR_LIT>" . format ( len ( self . states_shared ) ) ) <EOL> print ( "<STR_LIT>" . format ( self . _inputDimensions ) ) <EOL> rewards = T . col ( '<STR_LIT>' ) <EOL> actions = T . icol ( '<STR_LIT>' ) <EOL> terminals = T . icol ( '<STR_LIT>' ) <EOL> thediscount = T . scalar ( name = '<STR_LIT>' , dtype = theano . config . floatX ) <EOL> thelr = T . scalar ( name = '<STR_LIT>' , dtype = theano . config . floatX ) <EOL> q_vals , self . params , shape_after_conv = self . _build ( network_type , states ) <EOL> print ( "<STR_LIT>" . format ( shape_after_conv ) ) <EOL> next_q_vals , self . next_params , shape_after_conv = self . _build ( network_type , next_states ) <EOL> self . _resetQHat ( ) <EOL> self . rewards_shared = theano . shared ( <EOL> np . zeros ( ( batchSize , <NUM_LIT:1> ) , dtype = theano . config . floatX ) , <EOL> broadcastable = ( False , True ) ) <EOL> self . actions_shared = theano . shared ( <EOL> np . zeros ( ( batchSize , <NUM_LIT:1> ) , dtype = '<STR_LIT>' ) , <EOL> broadcastable = ( False , True ) ) <EOL> self . terminals_shared = theano . shared ( <EOL> np . zeros ( ( batchSize , <NUM_LIT:1> ) , dtype = '<STR_LIT>' ) , <EOL> broadcastable = ( False , True ) ) <EOL> if ( self . _DoubleQ == True ) : <EOL> givens_next = { } <EOL> for i , x in enumerate ( self . next_states_shared ) : <EOL> givens_next [ states [ i ] ] = x <EOL> self . next_q_vals_current_qnet = theano . function ( [ ] , q_vals , <EOL> givens = givens_next ) <EOL> next_q_curr_qnet = theano . clone ( next_q_vals ) <EOL> argmax_next_q_vals = T . argmax ( next_q_curr_qnet , axis = <NUM_LIT:1> , keepdims = True ) <EOL> max_next_q_vals = next_q_vals [ T . arange ( batchSize ) , argmax_next_q_vals . reshape ( ( - <NUM_LIT:1> , ) ) ] . reshape ( ( - <NUM_LIT:1> , <NUM_LIT:1> ) ) <EOL> else : <EOL> max_next_q_vals = T . max ( next_q_vals , axis = <NUM_LIT:1> , keepdims = True ) <EOL> T_ones_like = T . ones_like ( T . ones_like ( terminals ) - terminals ) <EOL> target = rewards + T_ones_like * thediscount * max_next_q_vals <EOL> q_val = q_vals [ T . arange ( batchSize ) , actions . reshape ( ( - <NUM_LIT:1> , ) ) ] . reshape ( ( - <NUM_LIT:1> , <NUM_LIT:1> ) ) <EOL> diff = - q_val + target <EOL> if self . clip_delta > <NUM_LIT:0> : <EOL> quadratic_part = T . minimum ( abs ( diff ) , self . clip_delta ) <EOL> linear_part = abs ( diff ) - quadratic_part <EOL> loss_ind = <NUM_LIT:0.5> * quadratic_part ** <NUM_LIT:2> + self . clip_delta * linear_part <EOL> else : <EOL> loss_ind = <NUM_LIT:0.5> * diff ** <NUM_LIT:2> <EOL> if batch_accumulator == '<STR_LIT>' : <EOL> loss = T . sum ( loss_ind ) <EOL> elif batch_accumulator == '<STR_LIT>' : <EOL> loss = T . mean ( loss_ind ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" . format ( batch_accumulator ) ) <EOL> givens = { <EOL> rewards : self . rewards_shared , <EOL> actions : self . actions_shared , <EOL> terminals : self . terminals_shared <EOL> } <EOL> for i , x in enumerate ( self . states_shared ) : <EOL> givens [ states [ i ] ] = x <EOL> for i , x in enumerate ( self . next_states_shared ) : <EOL> givens [ next_states [ i ] ] = x <EOL> gparams = [ ] <EOL> for p in self . params : <EOL> gparam = T . grad ( loss , p ) <EOL> gparams . append ( gparam ) <EOL> updates = [ ] <EOL> if update_rule == '<STR_LIT>' : <EOL> updates = deepmind_rmsprop ( loss , self . params , gparams , thelr , self . rho , <EOL> self . rms_epsilon ) <EOL> elif update_rule == '<STR_LIT>' : <EOL> for i , ( p , g ) in enumerate ( zip ( self . params , gparams ) ) : <EOL> acc = theano . shared ( p . get_value ( ) * <NUM_LIT:0.> ) <EOL> acc_new = rho * acc + ( <NUM_LIT:1> - self . rho ) * g ** <NUM_LIT:2> <EOL> gradient_scaling = T . sqrt ( acc_new + self . rms_epsilon ) <EOL> g = g / gradient_scaling <EOL> updates . append ( ( acc , acc_new ) ) <EOL> updates . append ( ( p , p - thelr * g ) ) <EOL> elif update_rule == '<STR_LIT>' : <EOL> for i , ( param , gparam ) in enumerate ( zip ( self . params , gparams ) ) : <EOL> updates . append ( ( param , param - thelr * gparam ) ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" . format ( update_rule ) ) <EOL> if ( self . _DoubleQ == True ) : <EOL> self . _train = theano . function ( [ thediscount , thelr , next_q_curr_qnet ] , [ loss , loss_ind , q_vals ] , updates = updates , <EOL> givens = givens , <EOL> on_unused_input = '<STR_LIT>' ) <EOL> else : <EOL> self . _train = theano . function ( [ thediscount , thelr ] , [ loss , loss_ind , q_vals ] , updates = updates , <EOL> givens = givens , <EOL> on_unused_input = '<STR_LIT>' ) <EOL> givens2 = { } <EOL> for i , x in enumerate ( self . states_shared ) : <EOL> givens2 [ states [ i ] ] = x <EOL> self . _q_vals = theano . function ( [ ] , q_vals , <EOL> givens = givens2 , <EOL> on_unused_input = '<STR_LIT>' ) <EOL> def toDump ( self ) : <EOL> return None , None <EOL> def train ( self , states_val , actions_val , rewards_val , next_states_val , terminals_val ) : <EOL> """<STR_LIT>""" <EOL> for i in range ( len ( self . states_shared ) ) : <EOL> self . states_shared [ i ] . set_value ( states_val [ i ] ) <EOL> for i in range ( len ( self . states_shared ) ) : <EOL> self . next_states_shared [ i ] . set_value ( next_states_val [ i ] ) <EOL> self . actions_shared . set_value ( actions_val . reshape ( len ( actions_val ) , <NUM_LIT:1> ) ) <EOL> self . rewards_shared . set_value ( rewards_val . reshape ( len ( rewards_val ) , <NUM_LIT:1> ) ) <EOL> self . terminals_shared . set_value ( terminals_val . reshape ( len ( terminals_val ) , <NUM_LIT:1> ) ) <EOL> if self . update_counter % self . freeze_interval == <NUM_LIT:0> : <EOL> self . _resetQHat ( ) <EOL> if ( self . _DoubleQ == True ) : <EOL> self . _next_q_curr_qnet = self . next_q_vals_current_qnet ( ) <EOL> loss , diff , _ = self . _train ( self . _df , self . _lr , self . _next_q_curr_qnet ) <EOL> else : <EOL> loss , diff , _ = self . _train ( self . _df , self . _lr ) <EOL> self . update_counter += <NUM_LIT:1> <EOL> return np . sqrt ( loss ) <EOL> def qValues ( self , state_val ) : <EOL> """<STR_LIT>""" <EOL> for i in range ( len ( self . states_shared ) ) : <EOL> aa = self . states_shared [ i ] . get_value ( ) <EOL> aa [ <NUM_LIT:0> ] = state_val [ i ] <EOL> self . states_shared [ i ] . set_value ( aa ) <EOL> return self . _q_vals ( ) [ <NUM_LIT:0> ] <EOL> def chooseBestAction ( self , state ) : <EOL> """<STR_LIT>""" <EOL> q_vals = self . qValues ( state ) <EOL> return np . argmax ( q_vals ) <EOL> def _build ( self , network_type , inputs ) : <EOL> if network_type == "<STR_LIT>" : <EOL> return self . _buildG_DQN_0 ( inputs ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" . format ( network_type ) ) <EOL> def _resetQHat ( self ) : <EOL> for i , ( param , next_param ) in enumerate ( zip ( self . params , self . next_params ) ) : <EOL> next_param . set_value ( param . get_value ( ) ) <EOL> def _buildG_DQN_0 ( self , inputs ) : <EOL> """<STR_LIT>""" <EOL> layers = [ ] <EOL> outs_conv = [ ] <EOL> outs_conv_shapes = [ ] <EOL> for i , dim in enumerate ( self . _inputDimensions ) : <EOL> nfilter = [ ] <EOL> if len ( dim ) == <NUM_LIT:3> : <EOL> newR = dim [ <NUM_LIT:1> ] <EOL> newC = dim [ <NUM_LIT:2> ] <EOL> fR = <NUM_LIT:8> <EOL> fC = <NUM_LIT:8> <EOL> pR = <NUM_LIT:1> <EOL> pC = <NUM_LIT:1> <EOL> nfilter . append ( <NUM_LIT:32> ) <EOL> stride_size = <NUM_LIT:4> <EOL> l_conv1 = ConvolutionalLayer ( <EOL> rng = self . _randomState , <EOL> input = inputs [ i ] . reshape ( ( self . _batchSize , dim [ <NUM_LIT:0> ] , newR , newC ) ) , <EOL> filter_shape = ( nfilter [ <NUM_LIT:0> ] , dim [ <NUM_LIT:0> ] , fR , fC ) , <EOL> image_shape = ( self . _batchSize , dim [ <NUM_LIT:0> ] , newR , newC ) , <EOL> poolsize = ( pR , pC ) , <EOL> stride = ( stride_size , stride_size ) <EOL> ) <EOL> layers . append ( l_conv1 ) <EOL> newR = ( newR - fR + <NUM_LIT:1> - pR ) // stride_size + <NUM_LIT:1> <EOL> newC = ( newC - fC + <NUM_LIT:1> - pC ) // stride_size + <NUM_LIT:1> <EOL> fR = <NUM_LIT:4> <EOL> fC = <NUM_LIT:4> <EOL> pR = <NUM_LIT:1> <EOL> pC = <NUM_LIT:1> <EOL> nfilter . append ( <NUM_LIT:64> ) <EOL> stride_size = <NUM_LIT:2> <EOL> l_conv2 = ConvolutionalLayer ( <EOL> rng = self . _randomState , <EOL> input = l_conv1 . output . reshape ( ( self . _batchSize , nfilter [ <NUM_LIT:0> ] , newR , newC ) ) , <EOL> filter_shape = ( nfilter [ <NUM_LIT:1> ] , nfilter [ <NUM_LIT:0> ] , fR , fC ) , <EOL> image_shape = ( self . _batchSize , nfilter [ <NUM_LIT:0> ] , newR , newC ) , <EOL> poolsize = ( pR , pC ) , <EOL> stride = ( stride_size , stride_size ) <EOL> ) <EOL> layers . append ( l_conv2 ) <EOL> newR = ( newR - fR + <NUM_LIT:1> - pR ) // stride_size + <NUM_LIT:1> <EOL> newC = ( newC - fC + <NUM_LIT:1> - pC ) // stride_size + <NUM_LIT:1> <EOL> fR = <NUM_LIT:3> <EOL> fC = <NUM_LIT:3> <EOL> pR = <NUM_LIT:1> <EOL> pC = <NUM_LIT:1> <EOL> nfilter . append ( <NUM_LIT:64> ) <EOL> stride_size = <NUM_LIT:1> <EOL> l_conv3 = ConvolutionalLayer ( <EOL> rng = self . _randomState , <EOL> input = l_conv2 . output . reshape ( ( self . _batchSize , nfilter [ <NUM_LIT:1> ] , newR , newC ) ) , <EOL> filter_shape = ( nfilter [ <NUM_LIT:2> ] , nfilter [ <NUM_LIT:1> ] , fR , fC ) , <EOL> image_shape = ( self . _batchSize , nfilter [ <NUM_LIT:1> ] , newR , newC ) , <EOL> poolsize = ( pR , pC ) , <EOL> stride = ( stride_size , stride_size ) <EOL> ) <EOL> layers . append ( l_conv3 ) <EOL> newR = ( newR - fR + <NUM_LIT:1> - pR ) // stride_size + <NUM_LIT:1> <EOL> newC = ( newC - fC + <NUM_LIT:1> - pC ) // stride_size + <NUM_LIT:1> <EOL> outs_conv . append ( l_conv3 . output ) <EOL> outs_conv_shapes . append ( ( nfilter [ <NUM_LIT:2> ] , newR , newC ) ) <EOL> elif len ( dim ) == <NUM_LIT:2> and dim [ <NUM_LIT:0> ] > <NUM_LIT:3> : <EOL> newR = dim [ <NUM_LIT:0> ] <EOL> newC = dim [ <NUM_LIT:1> ] <EOL> fR = <NUM_LIT:2> <EOL> fC = <NUM_LIT:1> <EOL> pR = <NUM_LIT:1> <EOL> pC = <NUM_LIT:1> <EOL> nfilter . append ( <NUM_LIT:16> ) <EOL> stride_size = <NUM_LIT:1> <EOL> l_conv1 = ConvolutionalLayer ( <EOL> rng = self . _randomState , <EOL> input = inputs [ i ] . reshape ( ( self . _batchSize , <NUM_LIT:1> , newR , newC ) ) , <EOL> filter_shape = ( nfilter [ <NUM_LIT:0> ] , <NUM_LIT:1> , fR , fC ) , <EOL> image_shape = ( self . _batchSize , <NUM_LIT:1> , newR , newC ) , <EOL> poolsize = ( pR , pC ) , <EOL> stride = ( stride_size , stride_size ) <EOL> ) <EOL> layers . append ( l_conv1 ) <EOL> newR = ( newR - fR + <NUM_LIT:1> - pR ) // stride_size + <NUM_LIT:1> <EOL> newC = ( newC - fC + <NUM_LIT:1> - pC ) // stride_size + <NUM_LIT:1> <EOL> fR = <NUM_LIT:2> <EOL> fC = <NUM_LIT:2> <EOL> pR = <NUM_LIT:1> <EOL> pC = <NUM_LIT:1> <EOL> nfilter . append ( <NUM_LIT:16> ) <EOL> stride_size = <NUM_LIT:1> <EOL> l_conv2 = ConvolutionalLayer ( <EOL> rng = self . _randomState , <EOL> input = l_conv1 . output . reshape ( ( self . _batchSize , nfilter [ <NUM_LIT:0> ] , newR , newC ) ) , <EOL> filter_shape = ( nfilter [ <NUM_LIT:1> ] , nfilter [ <NUM_LIT:0> ] , fR , fC ) , <EOL> image_shape = ( self . _batchSize , nfilter [ <NUM_LIT:0> ] , newR , newC ) , <EOL> poolsize = ( pR , pC ) , <EOL> stride = ( stride_size , stride_size ) <EOL> ) <EOL> layers . append ( l_conv2 ) <EOL> newR = ( newR - fR + <NUM_LIT:1> - pR ) // stride_size + <NUM_LIT:1> <EOL> newC = ( newC - fC + <NUM_LIT:1> - pC ) // stride_size + <NUM_LIT:1> <EOL> outs_conv . append ( l_conv2 . output ) <EOL> outs_conv_shapes . append ( ( nfilter [ <NUM_LIT:1> ] , newR , newC ) ) <EOL> else : <EOL> if dim [ <NUM_LIT:0> ] > <NUM_LIT:3> : <EOL> newR = <NUM_LIT:1> <EOL> newC = dim [ <NUM_LIT:0> ] <EOL> fR = <NUM_LIT:1> <EOL> fC = <NUM_LIT:2> <EOL> pR = <NUM_LIT:1> <EOL> pC = <NUM_LIT:1> <EOL> nfilter . append ( <NUM_LIT:8> ) <EOL> stride_size = <NUM_LIT:1> <EOL> l_conv1 = ConvolutionalLayer ( <EOL> rng = self . _randomState , <EOL> input = inputs [ i ] . reshape ( ( self . _batchSize , <NUM_LIT:1> , newR , newC ) ) , <EOL> filter_shape = ( nfilter [ <NUM_LIT:0> ] , <NUM_LIT:1> , fR , fC ) , <EOL> image_shape = ( self . _batchSize , <NUM_LIT:1> , newR , newC ) , <EOL> poolsize = ( pR , pC ) , <EOL> stride = ( stride_size , stride_size ) <EOL> ) <EOL> layers . append ( l_conv1 ) <EOL> newC = ( newC - fC + <NUM_LIT:1> - pC ) // stride_size + <NUM_LIT:1> <EOL> fR = <NUM_LIT:1> <EOL> fC = <NUM_LIT:2> <EOL> pR = <NUM_LIT:1> <EOL> pC = <NUM_LIT:1> <EOL> nfilter . append ( <NUM_LIT:8> ) <EOL> stride_size = <NUM_LIT:1> <EOL> l_conv2 = ConvolutionalLayer ( <EOL> rng = self . _randomState , <EOL> input = l_conv1 . output . reshape ( ( self . _batchSize , nfilter [ <NUM_LIT:0> ] , newR , newC ) ) , <EOL> filter_shape = ( nfilter [ <NUM_LIT:1> ] , nfilter [ <NUM_LIT:0> ] , fR , fC ) , <EOL> image_shape = ( self . _batchSize , nfilter [ <NUM_LIT:0> ] , newR , newC ) , <EOL> poolsize = ( pR , pC ) , <EOL> stride = ( stride_size , stride_size ) <EOL> ) <EOL> layers . append ( l_conv2 ) <EOL> newC = ( newC - fC + <NUM_LIT:1> - pC ) // stride_size + <NUM_LIT:1> <EOL> outs_conv . append ( l_conv2 . output ) <EOL> outs_conv_shapes . append ( ( nfilter [ <NUM_LIT:1> ] , newC ) ) <EOL> else : <EOL> if ( len ( dim ) == <NUM_LIT:2> ) : <EOL> outs_conv_shapes . append ( ( dim [ <NUM_LIT:0> ] , dim [ <NUM_LIT:1> ] ) ) <EOL> elif ( len ( dim ) == <NUM_LIT:1> ) : <EOL> outs_conv_shapes . append ( ( <NUM_LIT:1> , dim [ <NUM_LIT:0> ] ) ) <EOL> outs_conv . append ( inputs [ i ] ) <EOL> output_conv = outs_conv [ <NUM_LIT:0> ] . flatten ( ) . reshape ( ( self . _batchSize , np . prod ( outs_conv_shapes [ <NUM_LIT:0> ] ) ) ) <EOL> shapes = np . prod ( outs_conv_shapes [ <NUM_LIT:0> ] ) <EOL> if ( len ( outs_conv ) > <NUM_LIT:1> ) : <EOL> for out_conv , out_conv_shape in zip ( outs_conv [ <NUM_LIT:1> : ] , outs_conv_shapes [ <NUM_LIT:1> : ] ) : <EOL> output_conv = T . concatenate ( ( output_conv , out_conv . flatten ( ) . reshape ( ( self . _batchSize , np . prod ( out_conv_shape ) ) ) ) , axis = <NUM_LIT:1> ) <EOL> shapes += np . prod ( out_conv_shape ) <EOL> shapes <EOL> self . hiddenLayer1 = HiddenLayer ( rng = self . _randomState , input = output_conv , <EOL> n_in = shapes , n_out = <NUM_LIT:50> , <EOL> activation = T . tanh ) <EOL> layers . append ( self . hiddenLayer1 ) <EOL> self . hiddenLayer2 = HiddenLayer ( rng = self . _randomState , input = self . hiddenLayer1 . output , <EOL> n_in = <NUM_LIT:50> , n_out = <NUM_LIT:20> , <EOL> activation = T . tanh ) <EOL> layers . append ( self . hiddenLayer2 ) <EOL> self . outLayer = HiddenLayer ( rng = self . _randomState , input = self . hiddenLayer2 . output , <EOL> n_in = <NUM_LIT:20> , n_out = self . _nActions , <EOL> activation = None ) <EOL> layers . append ( self . outLayer ) <EOL> params = [ param <EOL> for layer in layers <EOL> for param in layer . params ] <EOL> return self . outLayer . output , params , outs_conv_shapes <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> pass </s>
