<s> import os <EOL> import numpy as np <EOL> from numpy import linalg as LA <EOL> from theano import tensor as T <EOL> import theano <EOL> from theano . tensor . shared_randomstreams import RandomStreams <EOL> from deepy import NeuralClassifier <EOL> from deepy . utils import build_activation , disconnected_grad <EOL> from deepy . utils . functions import FLOATX <EOL> from deepy . layers import NeuralLayer <EOL> from experiments . attention_models . gaussian_sampler import SampleMultivariateGaussian <EOL> class AttentionLayer ( NeuralLayer ) : <EOL> def __init__ ( self , activation = '<STR_LIT:relu>' , std = <NUM_LIT:0.1> , disable_reinforce = False , random_glimpse = False ) : <EOL> self . disable_reinforce = disable_reinforce <EOL> self . random_glimpse = random_glimpse <EOL> self . gaussian_std = std <EOL> super ( AttentionLayer , self ) . __init__ ( <NUM_LIT:10> , activation ) <EOL> def initialize ( self , config , vars , x , input_n , id = "<STR_LIT>" ) : <EOL> self . _config = config <EOL> self . _vars = vars <EOL> self . input_n = input_n <EOL> self . id = id <EOL> self . x = x <EOL> self . _setup_params ( ) <EOL> self . _setup_functions ( ) <EOL> self . connected = True <EOL> def _glimpse_sensor ( self , x_t , l_p ) : <EOL> """<STR_LIT>""" <EOL> l_p = l_p * <NUM_LIT> + <NUM_LIT> - <NUM_LIT:2> <EOL> l_p = T . cast ( T . round ( l_p ) , "<STR_LIT>" ) <EOL> l_p = l_p * ( l_p >= <NUM_LIT:0> ) <EOL> l_p = l_p * ( l_p < <NUM_LIT> ) + ( l_p >= <NUM_LIT> ) * <NUM_LIT> <EOL> l_p2 = l_p - <NUM_LIT:2> <EOL> l_p2 = l_p2 * ( l_p2 >= <NUM_LIT:0> ) <EOL> l_p2 = l_p2 * ( l_p2 < <NUM_LIT:20> ) + ( l_p2 >= <NUM_LIT:20> ) * <NUM_LIT> <EOL> l_p3 = l_p - <NUM_LIT:6> <EOL> l_p3 = l_p3 * ( l_p3 >= <NUM_LIT:0> ) <EOL> l_p3 = l_p3 * ( l_p3 < <NUM_LIT:16> ) + ( l_p3 >= <NUM_LIT:16> ) * <NUM_LIT:15> <EOL> glimpse_1 = x_t [ l_p [ <NUM_LIT:0> ] : l_p [ <NUM_LIT:0> ] + <NUM_LIT:4> ] [ : , l_p [ <NUM_LIT:1> ] : l_p [ <NUM_LIT:1> ] + <NUM_LIT:4> ] <EOL> glimpse_2 = x_t [ l_p2 [ <NUM_LIT:0> ] : l_p2 [ <NUM_LIT:0> ] + <NUM_LIT:8> ] [ : , l_p2 [ <NUM_LIT:1> ] : l_p2 [ <NUM_LIT:1> ] + <NUM_LIT:8> ] <EOL> glimpse_2 = theano . tensor . signal . downsample . max_pool_2d ( glimpse_2 , ( <NUM_LIT:2> , <NUM_LIT:2> ) ) <EOL> glimpse_3 = x_t [ l_p3 [ <NUM_LIT:0> ] : l_p3 [ <NUM_LIT:0> ] + <NUM_LIT:16> ] [ : , l_p3 [ <NUM_LIT:1> ] : l_p3 [ <NUM_LIT:1> ] + <NUM_LIT:16> ] <EOL> glimpse_3 = theano . tensor . signal . downsample . max_pool_2d ( glimpse_3 , ( <NUM_LIT:4> , <NUM_LIT:4> ) ) <EOL> return T . concatenate ( [ glimpse_1 , glimpse_2 , glimpse_3 ] ) <EOL> def _refined_glimpse_sensor ( self , x_t , l_p ) : <EOL> """<STR_LIT>""" <EOL> l_p = l_p * <NUM_LIT> + <NUM_LIT> - <NUM_LIT:4> <EOL> l_p = T . cast ( T . round ( l_p ) , "<STR_LIT>" ) <EOL> l_p = l_p * ( l_p >= <NUM_LIT:0> ) <EOL> l_p = l_p * ( l_p < <NUM_LIT> ) + ( l_p >= <NUM_LIT> ) * <NUM_LIT:20> <EOL> glimpse_1 = x_t [ l_p [ <NUM_LIT:0> ] : l_p [ <NUM_LIT:0> ] + <NUM_LIT:7> ] [ : , l_p [ <NUM_LIT:1> ] : l_p [ <NUM_LIT:1> ] + <NUM_LIT:7> ] <EOL> return glimpse_1 <EOL> def _multi_gaussian_pdf ( self , vec , mean ) : <EOL> norm2d_var = ( ( <NUM_LIT:1.0> / T . sqrt ( ( <NUM_LIT:2> * np . pi ) ** <NUM_LIT:2> * self . cov_det_var ) ) * <EOL> T . exp ( - <NUM_LIT:0.5> * ( ( vec - mean ) . T . dot ( self . cov_inv_var ) . dot ( vec - mean ) ) ) ) <EOL> return norm2d_var <EOL> def _glimpse_network ( self , x_t , l_p ) : <EOL> """<STR_LIT>""" <EOL> sensor_output = self . _refined_glimpse_sensor ( x_t , l_p ) <EOL> sensor_output = T . flatten ( sensor_output ) <EOL> h_g = self . _relu ( T . dot ( sensor_output , self . W_g0 ) ) <EOL> h_l = self . _relu ( T . dot ( l_p , self . W_g1 ) ) <EOL> g = self . _relu ( T . dot ( h_g , self . W_g2_hg ) + T . dot ( h_l , self . W_g2_hl ) ) <EOL> return g <EOL> def _location_network ( self , h_t ) : <EOL> """<STR_LIT>""" <EOL> return T . dot ( h_t , self . W_l ) <EOL> def _action_network ( self , h_t ) : <EOL> """<STR_LIT>""" <EOL> z = self . _relu ( T . dot ( h_t , self . W_a ) + self . B_a ) <EOL> return self . _softmax ( z ) <EOL> def _core_network ( self , l_p , h_p , x_t ) : <EOL> """<STR_LIT>""" <EOL> g_t = self . _glimpse_network ( x_t , l_p ) <EOL> h_t = self . _tanh ( T . dot ( g_t , self . W_h_g ) + T . dot ( h_p , self . W_h ) + self . B_h ) <EOL> l_t = self . _location_network ( h_t ) <EOL> if not self . disable_reinforce : <EOL> sampled_l_t = self . _sample_gaussian ( l_t , self . cov ) <EOL> sampled_pdf = self . _multi_gaussian_pdf ( disconnected_grad ( sampled_l_t ) , l_t ) <EOL> wl_grad = T . grad ( T . log ( sampled_pdf ) , self . W_l ) <EOL> else : <EOL> sampled_l_t = l_t <EOL> wl_grad = self . W_l <EOL> if self . random_glimpse and self . disable_reinforce : <EOL> sampled_l_t = self . srng . uniform ( ( <NUM_LIT:2> , ) ) * <NUM_LIT> <EOL> a_t = self . _action_network ( h_t ) <EOL> return sampled_l_t , h_t , a_t , wl_grad <EOL> def _output_func ( self ) : <EOL> self . x = self . x . reshape ( ( <NUM_LIT> , <NUM_LIT> ) ) <EOL> [ l_ts , h_ts , a_ts , wl_grads ] , _ = theano . scan ( fn = self . _core_network , <EOL> outputs_info = [ self . l0 , self . h0 , None , None ] , <EOL> non_sequences = [ self . x ] , <EOL> n_steps = <NUM_LIT:5> ) <EOL> self . positions = l_ts <EOL> self . last_decision = T . argmax ( a_ts [ - <NUM_LIT:1> ] ) <EOL> wl_grad = T . sum ( wl_grads , axis = <NUM_LIT:0> ) / wl_grads . shape [ <NUM_LIT:0> ] <EOL> self . wl_grad = wl_grad <EOL> return a_ts [ - <NUM_LIT:1> ] . reshape ( ( <NUM_LIT:1> , <NUM_LIT:10> ) ) <EOL> def _setup_functions ( self ) : <EOL> self . _assistive_params = [ ] <EOL> self . _relu = build_activation ( "<STR_LIT>" ) <EOL> self . _tanh = build_activation ( "<STR_LIT>" ) <EOL> self . _softmax = build_activation ( "<STR_LIT>" ) <EOL> self . output_func = self . _output_func ( ) <EOL> def _setup_params ( self ) : <EOL> self . srng = RandomStreams ( seed = <NUM_LIT> ) <EOL> self . large_cov = np . array ( [ [ <NUM_LIT> , <NUM_LIT:0> ] , [ <NUM_LIT:0> , <NUM_LIT> ] ] , dtype = FLOATX ) <EOL> self . small_cov = np . array ( [ [ self . gaussian_std , <NUM_LIT:0> ] , [ <NUM_LIT:0> , self . gaussian_std ] ] , dtype = FLOATX ) <EOL> self . cov = theano . shared ( np . array ( self . small_cov , dtype = FLOATX ) ) <EOL> self . cov_inv_var = theano . shared ( np . array ( LA . inv ( self . small_cov ) , dtype = FLOATX ) ) <EOL> self . cov_det_var = theano . shared ( np . array ( LA . det ( self . small_cov ) , dtype = FLOATX ) ) <EOL> self . _sample_gaussian = SampleMultivariateGaussian ( ) <EOL> self . W_g0 = self . create_weight ( <NUM_LIT:7> * <NUM_LIT:7> , <NUM_LIT> , suffix = "<STR_LIT>" ) <EOL> self . W_g1 = self . create_weight ( <NUM_LIT:2> , <NUM_LIT> , suffix = "<STR_LIT>" ) <EOL> self . W_g2_hg = self . create_weight ( <NUM_LIT> , <NUM_LIT> , suffix = "<STR_LIT>" ) <EOL> self . W_g2_hl = self . create_weight ( <NUM_LIT> , <NUM_LIT> , suffix = "<STR_LIT>" ) <EOL> self . W_h_g = self . create_weight ( <NUM_LIT> , <NUM_LIT> , suffix = "<STR_LIT>" ) <EOL> self . W_h = self . create_weight ( <NUM_LIT> , <NUM_LIT> , suffix = "<STR_LIT:h>" ) <EOL> self . B_h = self . create_bias ( <NUM_LIT> , suffix = "<STR_LIT:h>" ) <EOL> self . h0 = self . create_vector ( <NUM_LIT> , "<STR_LIT>" ) <EOL> self . l0 = self . create_vector ( <NUM_LIT:2> , "<STR_LIT>" ) <EOL> self . l0 . set_value ( np . array ( [ - <NUM_LIT:1> , - <NUM_LIT:1> ] , dtype = FLOATX ) ) <EOL> self . W_l = self . create_weight ( <NUM_LIT> , <NUM_LIT:2> , suffix = "<STR_LIT:l>" ) <EOL> self . W_l . set_value ( self . W_l . get_value ( ) / <NUM_LIT:10> ) <EOL> self . B_l = self . create_bias ( <NUM_LIT:2> , suffix = "<STR_LIT:l>" ) <EOL> self . W_a = self . create_weight ( <NUM_LIT> , <NUM_LIT:10> , suffix = "<STR_LIT:a>" ) <EOL> self . B_a = self . create_bias ( <NUM_LIT:10> , suffix = "<STR_LIT:a>" ) <EOL> self . W = [ self . W_g0 , self . W_g1 , self . W_g2_hg , self . W_g2_hl , self . W_h_g , self . W_h , self . W_a ] <EOL> self . B = [ self . B_h , self . B_a ] <EOL> self . parameters = [ self . W_l ] <EOL> def get_network ( model = None , std = <NUM_LIT> , disable_reinforce = False , random_glimpse = False ) : <EOL> """<STR_LIT>""" <EOL> network = NeuralClassifier ( input_dim = <NUM_LIT> * <NUM_LIT> ) <EOL> network . stack_layer ( AttentionLayer ( std = std , disable_reinforce = disable_reinforce , random_glimpse = random_glimpse ) ) <EOL> if model and os . path . exists ( model ) : <EOL> network . load_params ( model ) <EOL> return network </s>
