<s> import theano <EOL> import theano . tensor as T <EOL> import numpy as np <EOL> from lasagne . layers import Layer , DenseLayer <EOL> from lasagne . theano_extensions import padding <EOL> import lasagne . init <EOL> import lasagne . nonlinearities <EOL> import similarities <EOL> import nonlinearities <EOL> import init <EOL> class Head ( Layer ) : <EOL> r"""<STR_LIT>""" <EOL> def __init__ ( self , controller , num_shifts = <NUM_LIT:3> , memory_shape = ( <NUM_LIT> , <NUM_LIT:20> ) , <EOL> W_hid_to_sign = None , <EOL> b_hid_to_sign = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_sign = nonlinearities . ClippedLinear ( low = - <NUM_LIT:1.> , high = <NUM_LIT:1.> ) , <EOL> W_hid_to_key = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_key = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_key = nonlinearities . ClippedLinear ( low = <NUM_LIT:0.> , high = <NUM_LIT:1.> ) , <EOL> W_hid_to_beta = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_beta = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_beta = lasagne . nonlinearities . rectify , <EOL> W_hid_to_gate = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_gate = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_gate = nonlinearities . hard_sigmoid , <EOL> W_hid_to_shift = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_shift = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_shift = lasagne . nonlinearities . softmax , <EOL> W_hid_to_gamma = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_gamma = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_gamma = lambda x : <NUM_LIT:1.> + lasagne . nonlinearities . rectify ( x ) , <EOL> weights_init = init . OneHot ( ) , <EOL> learn_init = False , <EOL> ** kwargs ) : <EOL> super ( Head , self ) . __init__ ( controller , ** kwargs ) <EOL> self . memory_shape = memory_shape <EOL> self . basename = kwargs . get ( '<STR_LIT:name>' , '<STR_LIT>' ) <EOL> self . learn_init = learn_init <EOL> if W_hid_to_sign is not None : <EOL> self . sign = DenseLayer ( controller , num_units = self . memory_shape [ <NUM_LIT:1> ] , <EOL> W = W_hid_to_sign , b = b_hid_to_sign , nonlinearity = nonlinearity_sign , <EOL> name = self . basename + '<STR_LIT>' ) <EOL> self . W_hid_to_sign , self . b_hid_to_sign = self . sign . W , self . sign . b <EOL> else : <EOL> self . sign = None <EOL> self . W_hid_to_sign , self . b_hid_to_sign = None , None <EOL> self . key = DenseLayer ( controller , num_units = self . memory_shape [ <NUM_LIT:1> ] , <EOL> W = W_hid_to_key , b = b_hid_to_key , nonlinearity = nonlinearity_key , <EOL> name = self . basename + '<STR_LIT>' ) <EOL> self . W_hid_to_key , self . b_hid_to_key = self . key . W , self . key . b <EOL> self . beta = DenseLayer ( controller , num_units = <NUM_LIT:1> , <EOL> W = W_hid_to_beta , b = b_hid_to_beta , nonlinearity = nonlinearity_beta , <EOL> name = self . basename + '<STR_LIT>' ) <EOL> self . W_hid_to_beta , self . b_hid_to_beta = self . beta . W , self . beta . b <EOL> self . gate = DenseLayer ( controller , num_units = <NUM_LIT:1> , <EOL> W = W_hid_to_gate , b = b_hid_to_gate , nonlinearity = nonlinearity_gate , <EOL> name = self . basename + '<STR_LIT>' ) <EOL> self . W_hid_to_gate , self . b_hid_to_gate = self . gate . W , self . gate . b <EOL> self . num_shifts = num_shifts <EOL> self . shift = DenseLayer ( controller , num_units = num_shifts , <EOL> W = W_hid_to_shift , b = b_hid_to_shift , nonlinearity = nonlinearity_shift , <EOL> name = self . basename + '<STR_LIT>' ) <EOL> self . W_hid_to_shift , self . b_hid_to_shift = self . shift . W , self . shift . b <EOL> self . gamma = DenseLayer ( controller , num_units = <NUM_LIT:1> , <EOL> W = W_hid_to_gamma , b = b_hid_to_gamma , nonlinearity = nonlinearity_gamma , <EOL> name = self . basename + '<STR_LIT>' ) <EOL> self . W_hid_to_gamma , self . b_hid_to_gamma = self . gamma . W , self . gamma . b <EOL> self . weights_init = self . add_param ( <EOL> weights_init , ( <NUM_LIT:1> , self . memory_shape [ <NUM_LIT:0> ] ) , <EOL> name = '<STR_LIT>' , trainable = learn_init , regularizable = False ) <EOL> def get_output_for ( self , h_t , w_tm1 , M_t , ** kwargs ) : <EOL> if self . sign is not None : <EOL> sign_t = self . sign . get_output_for ( h_t , ** kwargs ) <EOL> else : <EOL> sign_t = <NUM_LIT:1.> <EOL> k_t = self . key . get_output_for ( h_t , ** kwargs ) <EOL> beta_t = self . beta . get_output_for ( h_t , ** kwargs ) <EOL> g_t = self . gate . get_output_for ( h_t , ** kwargs ) <EOL> s_t = self . shift . get_output_for ( h_t , ** kwargs ) <EOL> gamma_t = self . gamma . get_output_for ( h_t , ** kwargs ) <EOL> beta_t = T . addbroadcast ( beta_t , <NUM_LIT:1> ) <EOL> betaK = beta_t * similarities . cosine_similarity ( sign_t * k_t , M_t ) <EOL> w_c = lasagne . nonlinearities . softmax ( betaK ) <EOL> g_t = T . addbroadcast ( g_t , <NUM_LIT:1> ) <EOL> w_g = g_t * w_c + ( <NUM_LIT:1.> - g_t ) * w_tm1 <EOL> w_g_padded = w_g . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' , '<STR_LIT:x>' , <NUM_LIT:1> ) <EOL> conv_filter = s_t . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' , '<STR_LIT:x>' , <NUM_LIT:1> ) <EOL> pad = ( self . num_shifts // <NUM_LIT:2> , ( self . num_shifts - <NUM_LIT:1> ) // <NUM_LIT:2> ) <EOL> w_g_padded = padding . pad ( w_g_padded , [ pad ] , batch_ndim = <NUM_LIT:3> ) <EOL> convolution = T . nnet . conv2d ( w_g_padded , conv_filter , <EOL> input_shape = ( self . input_shape [ <NUM_LIT:0> ] , <NUM_LIT:1> , <NUM_LIT:1> , self . memory_shape [ <NUM_LIT:0> ] + pad [ <NUM_LIT:0> ] + pad [ <NUM_LIT:1> ] ) , <EOL> filter_shape = ( self . input_shape [ <NUM_LIT:0> ] , <NUM_LIT:1> , <NUM_LIT:1> , self . num_shifts ) , <EOL> subsample = ( <NUM_LIT:1> , <NUM_LIT:1> ) , <EOL> border_mode = '<STR_LIT>' ) <EOL> w_tilde = convolution [ : , <NUM_LIT:0> , <NUM_LIT:0> , : ] <EOL> gamma_t = T . addbroadcast ( gamma_t , <NUM_LIT:1> ) <EOL> w = T . pow ( w_tilde + <NUM_LIT> , gamma_t ) <EOL> w /= T . sum ( w ) <EOL> return w <EOL> def get_params ( self , ** tags ) : <EOL> params = super ( Head , self ) . get_params ( ** tags ) <EOL> if self . sign is not None : <EOL> params += self . sign . get_params ( ** tags ) <EOL> params += self . key . get_params ( ** tags ) <EOL> params += self . beta . get_params ( ** tags ) <EOL> params += self . gate . get_params ( ** tags ) <EOL> params += self . shift . get_params ( ** tags ) <EOL> params += self . gamma . get_params ( ** tags ) <EOL> return params <EOL> class WriteHead ( Head ) : <EOL> r"""<STR_LIT>""" <EOL> def __init__ ( self , controller , num_shifts = <NUM_LIT:3> , memory_shape = ( <NUM_LIT> , <NUM_LIT:20> ) , <EOL> W_hid_to_sign = None , <EOL> b_hid_to_sign = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_sign = nonlinearities . ClippedLinear ( low = - <NUM_LIT:1.> , high = <NUM_LIT:1.> ) , <EOL> W_hid_to_key = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_key = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_key = nonlinearities . ClippedLinear ( low = <NUM_LIT:0.> , high = <NUM_LIT:1.> ) , <EOL> W_hid_to_beta = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_beta = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_beta = lasagne . nonlinearities . rectify , <EOL> W_hid_to_gate = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_gate = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_gate = nonlinearities . hard_sigmoid , <EOL> W_hid_to_shift = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_shift = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_shift = lasagne . nonlinearities . softmax , <EOL> W_hid_to_gamma = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_gamma = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_gamma = lambda x : <NUM_LIT:1.> + lasagne . nonlinearities . rectify ( x ) , <EOL> W_hid_to_erase = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_erase = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_erase = nonlinearities . hard_sigmoid , <EOL> W_hid_to_add = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_add = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_add = nonlinearities . ClippedLinear ( low = <NUM_LIT:0.> , high = <NUM_LIT:1.> ) , <EOL> W_hid_to_sign_add = None , <EOL> b_hid_to_sign_add = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_sign_add = nonlinearities . ClippedLinear ( low = - <NUM_LIT:1.> , high = <NUM_LIT:1.> ) , <EOL> weights_init = init . OneHot ( ) , <EOL> learn_init = False , <EOL> ** kwargs ) : <EOL> super ( WriteHead , self ) . __init__ ( controller , num_shifts = num_shifts , memory_shape = memory_shape , <EOL> W_hid_to_sign = W_hid_to_sign , b_hid_to_sign = b_hid_to_sign , nonlinearity_sign = nonlinearity_sign , <EOL> W_hid_to_key = W_hid_to_key , b_hid_to_key = b_hid_to_key , nonlinearity_key = nonlinearity_key , <EOL> W_hid_to_beta = W_hid_to_beta , b_hid_to_beta = b_hid_to_beta , nonlinearity_beta = nonlinearity_beta , <EOL> W_hid_to_gate = W_hid_to_gate , b_hid_to_gate = b_hid_to_gate , nonlinearity_gate = nonlinearity_gate , <EOL> W_hid_to_shift = W_hid_to_shift , b_hid_to_shift = b_hid_to_shift , nonlinearity_shift = nonlinearity_shift , <EOL> W_hid_to_gamma = W_hid_to_gamma , b_hid_to_gamma = b_hid_to_gamma , nonlinearity_gamma = nonlinearity_gamma , <EOL> weights_init = weights_init , learn_init = learn_init , ** kwargs ) <EOL> self . erase = DenseLayer ( controller , num_units = self . memory_shape [ <NUM_LIT:1> ] , <EOL> W = W_hid_to_erase , b = b_hid_to_erase , nonlinearity = nonlinearity_erase , <EOL> name = self . basename + '<STR_LIT>' ) <EOL> self . W_hid_to_erase , self . b_hid_to_erase = self . erase . W , self . erase . b <EOL> self . add = DenseLayer ( controller , num_units = self . memory_shape [ <NUM_LIT:1> ] , <EOL> W = W_hid_to_add , b = b_hid_to_add , nonlinearity = nonlinearity_add , <EOL> name = self . basename + '<STR_LIT>' ) <EOL> self . W_hid_to_add , self . b_hid_to_add = self . add . W , self . add . b <EOL> if W_hid_to_sign_add is not None : <EOL> self . sign_add = DenseLayer ( controller , num_units = self . memory_shape [ <NUM_LIT:1> ] , <EOL> W = W_hid_to_sign_add , b = b_hid_to_sign_add , nonlinearity = nonlinearity_sign_add , <EOL> name = self . basename + '<STR_LIT>' ) <EOL> self . W_hid_to_sign_add , self . b_hid_to_sign_add = self . sign_add . W , self . sign_add . b <EOL> else : <EOL> self . sign_add = None <EOL> self . W_hid_to_sign_add , self . b_hid_to_sign_add = None , None <EOL> def get_params ( self , ** tags ) : <EOL> params = super ( WriteHead , self ) . get_params ( ** tags ) <EOL> params += self . erase . get_params ( ** tags ) <EOL> params += self . add . get_params ( ** tags ) <EOL> if self . sign_add is not None : <EOL> params += self . sign_add . get_params ( ** tags ) <EOL> return params <EOL> class ReadHead ( Head ) : <EOL> r"""<STR_LIT>""" <EOL> def __init__ ( self , controller , num_shifts = <NUM_LIT:3> , memory_shape = ( <NUM_LIT> , <NUM_LIT:20> ) , <EOL> W_hid_to_sign = None , <EOL> b_hid_to_sign = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_sign = nonlinearities . ClippedLinear ( low = - <NUM_LIT:1.> , high = <NUM_LIT:1.> ) , <EOL> W_hid_to_key = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_key = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_key = nonlinearities . ClippedLinear ( low = <NUM_LIT:0.> , high = <NUM_LIT:1.> ) , <EOL> W_hid_to_beta = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_beta = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_beta = lasagne . nonlinearities . rectify , <EOL> W_hid_to_gate = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_gate = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_gate = T . nnet . hard_sigmoid , <EOL> W_hid_to_shift = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_shift = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_shift = lasagne . nonlinearities . softmax , <EOL> W_hid_to_gamma = lasagne . init . GlorotUniform ( ) , <EOL> b_hid_to_gamma = lasagne . init . Constant ( <NUM_LIT:0.> ) , <EOL> nonlinearity_gamma = lambda x : <NUM_LIT:1.> + lasagne . nonlinearities . rectify ( x ) , <EOL> weights_init = init . OneHot ( ) , <EOL> learn_init = False , <EOL> ** kwargs ) : <EOL> super ( ReadHead , self ) . __init__ ( controller , num_shifts = num_shifts , memory_shape = memory_shape , <EOL> W_hid_to_sign = W_hid_to_sign , b_hid_to_sign = b_hid_to_sign , nonlinearity_sign = nonlinearity_sign , <EOL> W_hid_to_key = W_hid_to_key , b_hid_to_key = b_hid_to_key , nonlinearity_key = nonlinearity_key , <EOL> W_hid_to_beta = W_hid_to_beta , b_hid_to_beta = b_hid_to_beta , nonlinearity_beta = nonlinearity_beta , <EOL> W_hid_to_gate = W_hid_to_gate , b_hid_to_gate = b_hid_to_gate , nonlinearity_gate = nonlinearity_gate , <EOL> W_hid_to_shift = W_hid_to_shift , b_hid_to_shift = b_hid_to_shift , nonlinearity_shift = nonlinearity_shift , <EOL> W_hid_to_gamma = W_hid_to_gamma , b_hid_to_gamma = b_hid_to_gamma , nonlinearity_gamma = nonlinearity_gamma , <EOL> weights_init = weights_init , learn_init = learn_init , ** kwargs ) </s>
