<s> """<STR_LIT>""" <EOL> import numpy <EOL> import theano <EOL> import theano . tensor as T <EOL> from theano import config <EOL> from theano . tensor . nnet import categorical_crossentropy <EOL> floatX = theano . config . floatX <EOL> class SimpleRNN ( object ) : <EOL> def __init__ ( self , input_dim , recurrent_dim ) : <EOL> w_xh = numpy . random . normal ( <NUM_LIT:0> , <NUM_LIT> , ( input_dim , recurrent_dim ) ) <EOL> w_hh = numpy . random . normal ( <NUM_LIT:0> , <NUM_LIT> , ( recurrent_dim , recurrent_dim ) ) <EOL> self . w_xh = theano . shared ( numpy . asarray ( w_xh , dtype = floatX ) , name = '<STR_LIT>' ) <EOL> self . w_hh = theano . shared ( numpy . asarray ( w_hh , dtype = floatX ) , name = '<STR_LIT>' ) <EOL> self . b_h = theano . shared ( numpy . zeros ( ( recurrent_dim , ) , dtype = floatX ) , name = '<STR_LIT>' ) <EOL> self . parameters = [ self . w_xh , self . w_hh , self . b_h ] <EOL> def _step ( self , input_t , previous ) : <EOL> return T . tanh ( T . dot ( previous , self . w_hh ) + input_t ) <EOL> def __call__ ( self , x ) : <EOL> x_w_xh = T . dot ( x , self . w_xh ) + self . b_h <EOL> result , updates = theano . scan ( self . _step , <EOL> sequences = [ x_w_xh ] , <EOL> outputs_info = [ T . zeros_like ( self . b_h ) ] ) <EOL> return result <EOL> w_ho_np = numpy . random . normal ( <NUM_LIT:0> , <NUM_LIT> , ( <NUM_LIT:15> , <NUM_LIT:1> ) ) <EOL> w_ho = theano . shared ( numpy . asarray ( w_ho_np , dtype = floatX ) , name = '<STR_LIT>' ) <EOL> b_o = theano . shared ( numpy . zeros ( ( <NUM_LIT:1> , ) , dtype = floatX ) , name = '<STR_LIT>' ) <EOL> x = T . matrix ( '<STR_LIT:x>' ) <EOL> my_rnn = SimpleRNN ( <NUM_LIT:1> , <NUM_LIT:15> ) <EOL> hidden = my_rnn ( x ) <EOL> prediction = T . dot ( hidden , w_ho ) + b_o <EOL> parameters = my_rnn . parameters + [ w_ho , b_o ] <EOL> l2 = sum ( ( p ** <NUM_LIT:2> ) . sum ( ) for p in parameters ) <EOL> mse = T . mean ( ( prediction [ : - <NUM_LIT:1> ] - x [ <NUM_LIT:1> : ] ) ** <NUM_LIT:2> ) <EOL> cost = mse + <NUM_LIT> * l2 <EOL> gradient = T . grad ( cost , wrt = parameters ) <EOL> lr = <NUM_LIT> <EOL> updates = [ ( par , par - lr * gra ) for par , gra in zip ( parameters , gradient ) ] <EOL> update_model = theano . function ( [ x ] , cost , updates = updates ) <EOL> get_cost = theano . function ( [ x ] , mse ) <EOL> predict = theano . function ( [ x ] , prediction ) <EOL> get_hidden = theano . function ( [ x ] , hidden ) <EOL> get_gradient = theano . function ( [ x ] , gradient ) <EOL> predict = theano . function ( [ x ] , prediction ) <EOL> x_t = T . vector ( ) <EOL> h_p = T . vector ( ) <EOL> preactivation = T . dot ( x_t , my_rnn . w_xh ) + my_rnn . b_h <EOL> h_t = my_rnn . _step ( preactivation , h_p ) <EOL> o_t = T . dot ( h_t , w_ho ) + b_o <EOL> single_step = theano . function ( [ x_t , h_p ] , [ o_t , h_t ] ) <EOL> def gauss_weight ( rng , ndim_in , ndim_out = None , sd = <NUM_LIT> ) : <EOL> if ndim_out is None : <EOL> ndim_out = ndim_in <EOL> W = rng . randn ( ndim_in , ndim_out ) * sd <EOL> return numpy . asarray ( W , dtype = config . floatX ) <EOL> def index_dot ( indices , w ) : <EOL> return w [ indices . flatten ( ) ] <EOL> class LstmLayer : <EOL> def __init__ ( self , rng , input , mask , n_in , n_h ) : <EOL> self . W_i = theano . shared ( gauss_weight ( rng , n_in , n_h ) , '<STR_LIT>' , borrow = True ) <EOL> self . W_f = theano . shared ( gauss_weight ( rng , n_in , n_h ) , '<STR_LIT>' , borrow = True ) <EOL> self . W_c = theano . shared ( gauss_weight ( rng , n_in , n_h ) , '<STR_LIT>' , borrow = True ) <EOL> self . W_o = theano . shared ( gauss_weight ( rng , n_in , n_h ) , '<STR_LIT>' , borrow = True ) <EOL> self . U_i = theano . shared ( gauss_weight ( rng , n_h ) , '<STR_LIT>' , borrow = True ) <EOL> self . U_f = theano . shared ( gauss_weight ( rng , n_h ) , '<STR_LIT>' , borrow = True ) <EOL> self . U_c = theano . shared ( gauss_weight ( rng , n_h ) , '<STR_LIT>' , borrow = True ) <EOL> self . U_o = theano . shared ( gauss_weight ( rng , n_h ) , '<STR_LIT>' , borrow = True ) <EOL> self . b_i = theano . shared ( numpy . zeros ( ( n_h , ) , dtype = config . floatX ) , <EOL> '<STR_LIT>' , borrow = True ) <EOL> self . b_f = theano . shared ( numpy . zeros ( ( n_h , ) , dtype = config . floatX ) , <EOL> '<STR_LIT>' , borrow = True ) <EOL> self . b_c = theano . shared ( numpy . zeros ( ( n_h , ) , dtype = config . floatX ) , <EOL> '<STR_LIT>' , borrow = True ) <EOL> self . b_o = theano . shared ( numpy . zeros ( ( n_h , ) , dtype = config . floatX ) , <EOL> '<STR_LIT>' , borrow = True ) <EOL> self . params = [ self . W_i , self . W_f , self . W_c , self . W_o , <EOL> self . U_i , self . U_f , self . U_c , self . U_o , <EOL> self . b_i , self . b_f , self . b_c , self . b_o ] <EOL> outputs_info = [ T . zeros ( ( input . shape [ <NUM_LIT:1> ] , n_h ) ) , <EOL> T . zeros ( ( input . shape [ <NUM_LIT:1> ] , n_h ) ) ] <EOL> rval , updates = theano . scan ( self . _step , <EOL> sequences = [ mask , input ] , <EOL> outputs_info = outputs_info ) <EOL> self . output = rval [ <NUM_LIT:0> ] <EOL> def _step ( self , m_ , x_ , h_ , c_ ) : <EOL> i_preact = ( index_dot ( x_ , self . W_i ) + <EOL> T . dot ( h_ , self . U_i ) + self . b_i ) <EOL> i = T . nnet . sigmoid ( i_preact ) <EOL> f_preact = ( index_dot ( x_ , self . W_f ) + <EOL> T . dot ( h_ , self . U_f ) + self . b_f ) <EOL> f = T . nnet . sigmoid ( f_preact ) <EOL> o_preact = ( index_dot ( x_ , self . W_o ) + <EOL> T . dot ( h_ , self . U_o ) + self . b_o ) <EOL> o = T . nnet . sigmoid ( o_preact ) <EOL> c_preact = ( index_dot ( x_ , self . W_c ) + <EOL> T . dot ( h_ , self . U_c ) + self . b_c ) <EOL> c = T . tanh ( c_preact ) <EOL> c = f * c_ + i * c <EOL> c = m_ [ : , None ] * c + ( <NUM_LIT:1.> - m_ ) [ : , None ] * c_ <EOL> h = o * T . tanh ( c ) <EOL> h = m_ [ : , None ] * h + ( <NUM_LIT:1.> - m_ ) [ : , None ] * h_ <EOL> return h , c <EOL> def sequence_categorical_crossentropy ( prediction , targets , mask ) : <EOL> prediction_flat = prediction . reshape ( ( ( prediction . shape [ <NUM_LIT:0> ] * <EOL> prediction . shape [ <NUM_LIT:1> ] ) , <EOL> prediction . shape [ <NUM_LIT:2> ] ) , ndim = <NUM_LIT:2> ) <EOL> targets_flat = targets . flatten ( ) <EOL> mask_flat = mask . flatten ( ) <EOL> ce = categorical_crossentropy ( prediction_flat , targets_flat ) <EOL> return T . sum ( ce * mask_flat ) <EOL> class LogisticRegression ( object ) : <EOL> def __init__ ( self , rng , input , n_in , n_out ) : <EOL> W = gauss_weight ( rng , n_in , n_out ) <EOL> self . W = theano . shared ( value = numpy . asarray ( W , dtype = theano . config . floatX ) , <EOL> name = '<STR_LIT>' , borrow = True ) <EOL> self . b = theano . shared ( value = numpy . zeros ( ( n_out , ) , <EOL> dtype = theano . config . floatX ) , <EOL> name = '<STR_LIT:b>' , borrow = True ) <EOL> energy = T . dot ( input , self . W ) + self . b <EOL> energy_exp = T . exp ( energy - T . max ( energy , axis = <NUM_LIT:2> , keepdims = True ) ) <EOL> pmf = energy_exp / energy_exp . sum ( axis = <NUM_LIT:2> , keepdims = True ) <EOL> self . p_y_given_x = pmf <EOL> self . params = [ self . W , self . b ] <EOL> batch_size = <NUM_LIT:100> <EOL> n_h = <NUM_LIT:50> <EOL> rng = numpy . random . RandomState ( <NUM_LIT> ) <EOL> x = T . lmatrix ( '<STR_LIT:x>' ) <EOL> mask = T . matrix ( '<STR_LIT>' ) <EOL> recurrent_layer = LstmLayer ( rng = rng , input = x , mask = mask , n_in = <NUM_LIT> , n_h = n_h ) <EOL> logreg_layer = LogisticRegression ( rng = rng , input = recurrent_layer . output [ : - <NUM_LIT:1> ] , <EOL> n_in = n_h , n_out = <NUM_LIT> ) <EOL> cost = sequence_categorical_crossentropy ( logreg_layer . p_y_given_x , <EOL> x [ <NUM_LIT:1> : ] , <EOL> mask [ <NUM_LIT:1> : ] ) / batch_size <EOL> params = logreg_layer . params + recurrent_layer . params <EOL> grads = T . grad ( cost , params ) <EOL> learning_rate = <NUM_LIT:0.1> <EOL> updates = [ <EOL> ( param_i , param_i - learning_rate * grad_i ) <EOL> for param_i , grad_i in zip ( params , grads ) <EOL> ] <EOL> update_model = theano . function ( [ x , mask ] , cost , updates = updates ) <EOL> evaluate_model = theano . function ( [ x , mask ] , cost ) <EOL> x_t = T . iscalar ( ) <EOL> h_p = T . vector ( ) <EOL> c_p = T . vector ( ) <EOL> h_t , c_t = recurrent_layer . _step ( T . ones ( <NUM_LIT:1> ) , x_t , h_p , c_p ) <EOL> energy = T . dot ( h_t , logreg_layer . W ) + logreg_layer . b <EOL> energy_exp = T . exp ( energy - T . max ( energy , axis = <NUM_LIT:1> , keepdims = True ) ) <EOL> output = energy_exp / energy_exp . sum ( axis = <NUM_LIT:1> , keepdims = True ) <EOL> single_step = theano . function ( [ x_t , h_p , c_p ] , [ output , h_t , c_t ] ) </s>
