<s> import theano <EOL> import theano . tensor as T <EOL> from theano . sandbox . rng_mrg import MRG_RandomStreams <EOL> from theano . tensor . nnet . conv import conv2d <EOL> from theano . tensor . signal . downsample import max_pool_2d <EOL> from theano . tensor . shared_randomstreams import RandomStreams <EOL> import numpy as np <EOL> from toolbox import * <EOL> from modelbase import * <EOL> class LM_gru ( ModelLMBase ) : <EOL> def __init__ ( self , data , hp ) : <EOL> super ( LM_gru , self ) . __init__ ( self . __class__ . __name__ , data , hp ) <EOL> self . n_h = <NUM_LIT> <EOL> self . dropout = <NUM_LIT:0.5> <EOL> self . params = Parameters ( ) <EOL> self . hiddenstates = Parameters ( ) <EOL> n_tokens = self . data [ '<STR_LIT>' ] <EOL> n_h = self . n_h <EOL> scale = hp . init_scale <EOL> gates = <NUM_LIT:3> <EOL> with self . hiddenstates : <EOL> b1_h = shared_zeros ( ( self . hp . batch_size , n_h ) ) <EOL> b2_h = shared_zeros ( ( self . hp . batch_size , n_h ) ) <EOL> if hp . load_model and os . path . isfile ( self . filename ) : <EOL> self . params . load ( self . filename ) <EOL> else : <EOL> with self . params : <EOL> W_emb = shared_normal ( ( n_tokens , n_h ) , scale = scale ) <EOL> W1 = shared_normal ( ( n_h , n_h * gates ) , scale = scale * <NUM_LIT> ) <EOL> V1 = shared_normal ( ( n_h , n_h * gates ) , scale = scale * <NUM_LIT> ) <EOL> b1 = shared_zeros ( ( n_h * gates ) ) <EOL> W2 = shared_normal ( ( n_h , n_h * gates ) , scale = scale * <NUM_LIT> ) <EOL> V2 = shared_normal ( ( n_h , n_h * gates ) , scale = scale * <NUM_LIT> ) <EOL> b2 = shared_zeros ( ( n_h * gates , ) ) <EOL> def lstm ( X , h , c , W , U , b ) : <EOL> g_on = T . dot ( X , W ) + T . dot ( h , U ) + b <EOL> i_on = T . nnet . sigmoid ( g_on [ : , : n_h ] ) <EOL> f_on = T . nnet . sigmoid ( g_on [ : , n_h : <NUM_LIT:2> * n_h ] ) <EOL> o_on = T . nnet . sigmoid ( g_on [ : , <NUM_LIT:2> * n_h : <NUM_LIT:3> * n_h ] ) <EOL> c = f_on * c + i_on * T . tanh ( g_on [ : , <NUM_LIT:3> * n_h : ] ) <EOL> h = o_on * T . tanh ( c ) <EOL> return h , c <EOL> def gru ( X , h , W , U , b ) : <EOL> z_t = T . nnet . sigmoid ( T . dot ( X , W [ : , : n_h ] ) + T . dot ( h , U [ : , : n_h ] ) + b [ : n_h ] ) <EOL> r_t = T . nnet . sigmoid ( T . dot ( X , W [ : , n_h : <NUM_LIT:2> * n_h ] ) + T . dot ( h , U [ : , n_h : <NUM_LIT:2> * n_h ] ) + b [ n_h : <NUM_LIT:2> * n_h ] ) <EOL> h_t = T . tanh ( T . dot ( X , W [ : , <NUM_LIT:2> * n_h : <NUM_LIT:3> * n_h ] ) + r_t * T . dot ( h , U [ : , <NUM_LIT:2> * n_h : <NUM_LIT:3> * n_h ] ) + b [ <NUM_LIT:2> * n_h : <NUM_LIT:3> * n_h ] ) <EOL> return ( <NUM_LIT:1> - z_t ) * h + z_t * h_t <EOL> def sgru ( X , h , W , U , b ) : <EOL> z_t = T . tanh ( T . dot ( X , W [ : , : n_h ] ) + T . dot ( h , U [ : , : n_h ] ) + b [ : n_h ] ) <EOL> h_t = T . tanh ( T . dot ( X , W [ : , <NUM_LIT:1> * n_h : <NUM_LIT:2> * n_h ] ) + T . dot ( h , U [ : , <NUM_LIT:1> * n_h : <NUM_LIT:2> * n_h ] ) + b [ <NUM_LIT:1> * n_h : <NUM_LIT:2> * n_h ] ) <EOL> return z_t * h_t <EOL> def model ( x , p , p_dropout ) : <EOL> input_size = x . shape [ <NUM_LIT:1> ] <EOL> h0 = p . W_emb [ x ] <EOL> h0 = dropout ( h0 , p_dropout ) <EOL> cost , h1 , h2 = [ <NUM_LIT:0.> , b1_h , b2_h ] <EOL> for t in xrange ( <NUM_LIT:0> , self . hp . seq_size ) : <EOL> if t >= self . hp . warmup_size : <EOL> pyx = softmax ( T . dot ( dropout ( h2 , p_dropout ) , T . transpose ( p . W_emb ) ) ) <EOL> cost += T . sum ( T . nnet . categorical_crossentropy ( pyx , theano_one_hot ( x [ t ] , n_tokens ) ) ) <EOL> h1 = gru ( h0 [ t ] , h1 , p . W1 , p . V1 , p . b1 ) <EOL> h2 = gru ( dropout ( h1 , p_dropout ) , h2 , p . W2 , p . V2 , p . b2 ) <EOL> h_updates = [ ( b1_h , h1 ) , ( b2_h , h2 ) ] <EOL> return cost , h_updates <EOL> cost , h_updates = model ( self . X , self . params , self . dropout ) <EOL> te_cost , te_h_updates = model ( self . X , self . params , <NUM_LIT:0.> ) <EOL> self . compile ( cost , te_cost , h_updates , te_h_updates ) </s>
