<s> import theano <EOL> from theano import tensor as T <EOL> from theano . sandbox . rng_mrg import MRG_RandomStreams as RandomStreams <EOL> import numpy as np <EOL> from load import mnist <EOL> from theano . tensor . nnet . conv import conv2d <EOL> from theano . tensor . signal . downsample import max_pool_2d <EOL> srng = RandomStreams ( ) <EOL> def floatX ( X ) : <EOL> return np . asarray ( X , dtype = theano . config . floatX ) <EOL> def init_weights ( shape ) : <EOL> return theano . shared ( floatX ( np . random . randn ( * shape ) * <NUM_LIT> ) ) <EOL> def rectify ( X ) : <EOL> return T . maximum ( X , <NUM_LIT:0.> ) <EOL> def softmax ( X ) : <EOL> e_x = T . exp ( X - X . max ( axis = <NUM_LIT:1> ) . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' ) ) <EOL> return e_x / e_x . sum ( axis = <NUM_LIT:1> ) . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' ) <EOL> def dropout ( X , p = <NUM_LIT:0.> ) : <EOL> if p > <NUM_LIT:0> : <EOL> retain_prob = <NUM_LIT:1> - p <EOL> X *= srng . binomial ( X . shape , p = retain_prob , dtype = theano . config . floatX ) <EOL> X /= retain_prob <EOL> return X <EOL> def RMSprop ( cost , params , lr = <NUM_LIT> , rho = <NUM_LIT> , epsilon = <NUM_LIT> ) : <EOL> grads = T . grad ( cost = cost , wrt = params ) <EOL> updates = [ ] <EOL> for p , g in zip ( params , grads ) : <EOL> acc = theano . shared ( p . get_value ( ) * <NUM_LIT:0.> ) <EOL> acc_new = rho * acc + ( <NUM_LIT:1> - rho ) * g ** <NUM_LIT:2> <EOL> gradient_scaling = T . sqrt ( acc_new + epsilon ) <EOL> g = g / gradient_scaling <EOL> updates . append ( ( acc , acc_new ) ) <EOL> updates . append ( ( p , p - lr * g ) ) <EOL> return updates <EOL> def model ( X , w , w2 , w3 , w4 , p_drop_conv , p_drop_hidden ) : <EOL> l1a = rectify ( conv2d ( X , w , border_mode = '<STR_LIT>' ) ) <EOL> l1 = max_pool_2d ( l1a , ( <NUM_LIT:2> , <NUM_LIT:2> ) ) <EOL> l1 = dropout ( l1 , p_drop_conv ) <EOL> l2a = rectify ( conv2d ( l1 , w2 ) ) <EOL> l2 = max_pool_2d ( l2a , ( <NUM_LIT:2> , <NUM_LIT:2> ) ) <EOL> l2 = dropout ( l2 , p_drop_conv ) <EOL> l3a = rectify ( conv2d ( l2 , w3 ) ) <EOL> l3b = max_pool_2d ( l3a , ( <NUM_LIT:2> , <NUM_LIT:2> ) ) <EOL> l3 = T . flatten ( l3b , outdim = <NUM_LIT:2> ) <EOL> l3 = dropout ( l3 , p_drop_conv ) <EOL> l4 = rectify ( T . dot ( l3 , w4 ) ) <EOL> l4 = dropout ( l4 , p_drop_hidden ) <EOL> pyx = softmax ( T . dot ( l4 , w_o ) ) <EOL> return l1 , l2 , l3 , l4 , pyx <EOL> trX , teX , trY , teY = mnist ( onehot = True ) <EOL> trX = trX . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT> ) <EOL> teX = teX . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT> ) <EOL> X = T . ftensor4 ( ) <EOL> Y = T . fmatrix ( ) <EOL> w = init_weights ( ( <NUM_LIT:32> , <NUM_LIT:1> , <NUM_LIT:3> , <NUM_LIT:3> ) ) <EOL> w2 = init_weights ( ( <NUM_LIT:64> , <NUM_LIT:32> , <NUM_LIT:3> , <NUM_LIT:3> ) ) <EOL> w3 = init_weights ( ( <NUM_LIT> , <NUM_LIT:64> , <NUM_LIT:3> , <NUM_LIT:3> ) ) <EOL> w4 = init_weights ( ( <NUM_LIT> * <NUM_LIT:3> * <NUM_LIT:3> , <NUM_LIT> ) ) <EOL> w_o = init_weights ( ( <NUM_LIT> , <NUM_LIT:10> ) ) <EOL> noise_l1 , noise_l2 , noise_l3 , noise_l4 , noise_py_x = model ( X , w , w2 , w3 , w4 , <NUM_LIT> , <NUM_LIT:0.5> ) <EOL> l1 , l2 , l3 , l4 , py_x = model ( X , w , w2 , w3 , w4 , <NUM_LIT:0.> , <NUM_LIT:0.> ) <EOL> y_x = T . argmax ( py_x , axis = <NUM_LIT:1> ) <EOL> cost = T . mean ( T . nnet . categorical_crossentropy ( noise_py_x , Y ) ) <EOL> params = [ w , w2 , w3 , w4 , w_o ] <EOL> updates = RMSprop ( cost , params , lr = <NUM_LIT> ) <EOL> train = theano . function ( inputs = [ X , Y ] , outputs = cost , updates = updates , allow_input_downcast = True ) <EOL> predict = theano . function ( inputs = [ X ] , outputs = y_x , allow_input_downcast = True ) <EOL> for i in range ( <NUM_LIT:100> ) : <EOL> for start , end in zip ( range ( <NUM_LIT:0> , len ( trX ) , <NUM_LIT> ) , range ( <NUM_LIT> , len ( trX ) , <NUM_LIT> ) ) : <EOL> cost = train ( trX [ start : end ] , trY [ start : end ] ) <EOL> print np . mean ( np . argmax ( teY , axis = <NUM_LIT:1> ) == predict ( teX ) ) </s>
