<s> """<STR_LIT>""" <EOL> from groundhog . datasets import LMIterator <EOL> from groundhog . trainer . SGD_adadelta import SGD <EOL> from groundhog . mainLoop import MainLoop <EOL> from groundhog . layers import MultiLayer , RecurrentMultiLayer , RecurrentMultiLayerInp , RecurrentMultiLayerShortPath , RecurrentMultiLayerShortPathInp , RecurrentMultiLayerShortPathInpAll , SoftmaxLayer , LastState , UnaryOp , Operator , Shift , GaussianNoise , SigmoidLayer <EOL> from groundhog . layers import maxpool , maxpool_ntimes , last , last_ntimes , tanh , sigmoid , rectifier , hard_sigmoid , hard_tanh <EOL> from groundhog . models import LM_Model <EOL> from theano . sandbox . scan import scan <EOL> import numpy <EOL> import theano <EOL> import theano . tensor as TT <EOL> linear = lambda x : x <EOL> theano . config . allow_gc = False <EOL> def get_text_data ( state ) : <EOL> def out_format ( x , y , r ) : <EOL> return { '<STR_LIT:x>' : x , '<STR_LIT:y>' : y , '<STR_LIT>' : r } <EOL> def out_format_valid ( x , y , r ) : <EOL> return { '<STR_LIT:x>' : x , '<STR_LIT:y>' : y , '<STR_LIT>' : r } <EOL> train_data = LMIterator ( <EOL> batch_size = state [ '<STR_LIT>' ] , <EOL> path = state [ '<STR_LIT:path>' ] , <EOL> stop = - <NUM_LIT:1> , <EOL> seq_len = state [ '<STR_LIT>' ] , <EOL> mode = "<STR_LIT:train>" , <EOL> chunks = state [ '<STR_LIT>' ] , <EOL> shift = state [ '<STR_LIT>' ] , <EOL> output_format = out_format , <EOL> can_fit = True ) <EOL> valid_data = LMIterator ( <EOL> batch_size = state [ '<STR_LIT>' ] , <EOL> path = state [ '<STR_LIT:path>' ] , <EOL> stop = - <NUM_LIT:1> , <EOL> use_infinite_loop = False , <EOL> allow_short_sequences = True , <EOL> seq_len = state [ '<STR_LIT>' ] , <EOL> mode = "<STR_LIT>" , <EOL> reset = state [ '<STR_LIT>' ] , <EOL> chunks = state [ '<STR_LIT>' ] , <EOL> shift = state [ '<STR_LIT>' ] , <EOL> output_format = out_format_valid , <EOL> can_fit = True ) <EOL> test_data = LMIterator ( <EOL> batch_size = state [ '<STR_LIT>' ] , <EOL> path = state [ '<STR_LIT:path>' ] , <EOL> stop = - <NUM_LIT:1> , <EOL> use_infinite_loop = False , <EOL> allow_short_sequences = True , <EOL> seq_len = state [ '<STR_LIT>' ] , <EOL> mode = "<STR_LIT:test>" , <EOL> chunks = state [ '<STR_LIT>' ] , <EOL> shift = state [ '<STR_LIT>' ] , <EOL> output_format = out_format_valid , <EOL> can_fit = True ) <EOL> if '<STR_LIT>' in state [ '<STR_LIT:path>' ] : <EOL> test_data = None <EOL> return train_data , valid_data , test_data <EOL> def jobman ( state , channel ) : <EOL> rng = numpy . random . RandomState ( state [ '<STR_LIT>' ] ) <EOL> if state [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> else : <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:50> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:50> <EOL> train_data , valid_data , test_data = get_text_data ( state ) <EOL> x = TT . lvector ( '<STR_LIT:x>' ) <EOL> y = TT . lvector ( '<STR_LIT:y>' ) <EOL> h0 = theano . shared ( numpy . zeros ( ( eval ( state [ '<STR_LIT>' ] ) [ - <NUM_LIT:1> ] , ) , dtype = '<STR_LIT>' ) ) <EOL> emb_words = MultiLayer ( <EOL> rng , <EOL> n_in = state [ '<STR_LIT>' ] , <EOL> n_hids = eval ( state [ '<STR_LIT>' ] ) , <EOL> activation = eval ( state [ '<STR_LIT>' ] ) , <EOL> init_fn = '<STR_LIT>' , <EOL> weight_noise = state [ '<STR_LIT>' ] , <EOL> rank_n_approx = state [ '<STR_LIT>' ] , <EOL> scale = state [ '<STR_LIT>' ] , <EOL> sparsity = state [ '<STR_LIT>' ] , <EOL> learn_bias = True , <EOL> bias_scale = eval ( state [ '<STR_LIT>' ] ) , <EOL> name = '<STR_LIT>' ) <EOL> rec = eval ( state [ '<STR_LIT>' ] ) ( <EOL> rng , <EOL> eval ( state [ '<STR_LIT>' ] ) , <EOL> activation = eval ( state [ '<STR_LIT>' ] ) , <EOL> bias_scale = eval ( state [ '<STR_LIT>' ] ) , <EOL> scale = eval ( state [ '<STR_LIT>' ] ) , <EOL> sparsity = eval ( state [ '<STR_LIT>' ] ) , <EOL> init_fn = eval ( state [ '<STR_LIT>' ] ) , <EOL> weight_noise = state [ '<STR_LIT>' ] , <EOL> name = '<STR_LIT>' ) <EOL> x_emb = emb_words ( x , no_noise_bias = state [ '<STR_LIT>' ] ) <EOL> reset = TT . scalar ( '<STR_LIT>' ) <EOL> rec_layer = rec ( x_emb , n_steps = x . shape [ <NUM_LIT:0> ] , <EOL> init_state = h0 * reset , <EOL> no_noise_bias = state [ '<STR_LIT>' ] , <EOL> truncate_gradient = state [ '<STR_LIT>' ] , <EOL> batch_size = <NUM_LIT:1> ) <EOL> output_layer = SoftmaxLayer ( <EOL> rng , <EOL> eval ( state [ '<STR_LIT>' ] ) [ - <NUM_LIT:1> ] , <EOL> state [ '<STR_LIT>' ] , <EOL> scale = state [ '<STR_LIT>' ] , <EOL> bias_scale = state [ '<STR_LIT>' ] , <EOL> init_fn = "<STR_LIT>" , <EOL> weight_noise = state [ '<STR_LIT>' ] , <EOL> sparsity = state [ '<STR_LIT>' ] , <EOL> sum_over_time = True , <EOL> name = '<STR_LIT>' ) <EOL> if state [ '<STR_LIT>' ] : <EOL> shortcut = MultiLayer ( <EOL> rng , <EOL> n_in = state [ '<STR_LIT>' ] , <EOL> n_hids = eval ( state [ '<STR_LIT>' ] ) , <EOL> activations = eval ( state [ '<STR_LIT>' ] ) , <EOL> init_fn = '<STR_LIT>' , <EOL> weight_noise = state [ '<STR_LIT>' ] , <EOL> scale = eval ( state [ '<STR_LIT>' ] ) , <EOL> sparsity = eval ( state [ '<STR_LIT>' ] ) , <EOL> learn_bias = eval ( state [ '<STR_LIT>' ] ) , <EOL> bias_scale = eval ( state [ '<STR_LIT>' ] ) , <EOL> name = '<STR_LIT>' ) <EOL> state [ '<STR_LIT>' ] = state [ '<STR_LIT>' ] <EOL> def update_lr ( obj , cost ) : <EOL> stp = obj . step <EOL> if isinstance ( obj . state [ '<STR_LIT>' ] , int ) and stp > obj . state [ '<STR_LIT>' ] : <EOL> time = float ( stp - obj . state [ '<STR_LIT>' ] ) <EOL> new_lr = obj . state [ '<STR_LIT>' ] / ( <NUM_LIT:1> + time / obj . state [ '<STR_LIT>' ] ) <EOL> obj . lr = new_lr <EOL> if state [ '<STR_LIT>' ] : <EOL> rec . add_schedule ( update_lr ) <EOL> if state [ '<STR_LIT>' ] : <EOL> train_model = output_layer ( rec_layer , <EOL> no_noise_bias = state [ '<STR_LIT>' ] , <EOL> additional_inputs = [ shortcut ( x ) ] ) . train ( target = y , <EOL> scale = numpy . float32 ( <NUM_LIT:1.> / state [ '<STR_LIT>' ] ) ) <EOL> else : <EOL> train_model = output_layer ( rec_layer , <EOL> no_noise_bias = state [ '<STR_LIT>' ] ) . train ( target = y , <EOL> scale = numpy . float32 ( <NUM_LIT:1.> / state [ '<STR_LIT>' ] ) ) <EOL> nw_h0 = rec_layer . out [ rec_layer . out . shape [ <NUM_LIT:0> ] - <NUM_LIT:1> ] <EOL> if state [ '<STR_LIT>' ] : <EOL> train_model . updates += [ ( h0 , nw_h0 ) ] <EOL> h0val = theano . shared ( numpy . zeros ( ( eval ( state [ '<STR_LIT>' ] ) [ - <NUM_LIT:1> ] , ) , dtype = '<STR_LIT>' ) ) <EOL> rec_layer = rec ( emb_words ( x , use_noise = False ) , <EOL> n_steps = x . shape [ <NUM_LIT:0> ] , <EOL> batch_size = <NUM_LIT:1> , <EOL> init_state = h0val * reset , <EOL> use_noise = False ) <EOL> nw_h0 = rec_layer . out [ rec_layer . out . shape [ <NUM_LIT:0> ] - <NUM_LIT:1> ] <EOL> if not state [ '<STR_LIT>' ] : <EOL> valid_model = output_layer ( rec_layer , <EOL> use_noise = False ) . validate ( target = y , sum_over_time = True ) <EOL> else : <EOL> valid_model = output_layer ( rec_layer , <EOL> additional_inputs = [ shortcut ( x , use_noise = False ) ] , <EOL> use_noise = False ) . validate ( target = y , sum_over_time = True ) <EOL> valid_updates = [ ] <EOL> if state [ '<STR_LIT>' ] : <EOL> valid_updates = [ ( h0val , nw_h0 ) ] <EOL> valid_fn = theano . function ( [ x , y , reset ] , valid_model . out , <EOL> name = '<STR_LIT>' , updates = valid_updates ) <EOL> def sample_fn ( word_tm1 , h_tm1 ) : <EOL> x_emb = emb_words ( word_tm1 , use_noise = False , one_step = True ) <EOL> h0_ = rec ( x_emb , state_before = h_tm1 , one_step = True , use_noise = False ) [ - <NUM_LIT:1> ] <EOL> word = output_layer . get_sample ( state_below = h0_ , temp = <NUM_LIT:1.> ) <EOL> return word , h0_ <EOL> [ samples , summaries ] , updates = scan ( sample_fn , <EOL> states = [ <EOL> TT . alloc ( numpy . int64 ( <NUM_LIT:0> ) , state [ '<STR_LIT>' ] ) , <EOL> TT . alloc ( numpy . float32 ( <NUM_LIT:0> ) , <NUM_LIT:1> , eval ( state [ '<STR_LIT>' ] ) [ - <NUM_LIT:1> ] ) ] , <EOL> n_steps = state [ '<STR_LIT>' ] , <EOL> name = '<STR_LIT>' ) <EOL> sample_fn = theano . function ( [ ] , [ samples ] , <EOL> updates = updates , profile = False , name = '<STR_LIT>' ) <EOL> dictionary = numpy . load ( state [ '<STR_LIT>' ] ) <EOL> if state [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> dictionary = dictionary [ '<STR_LIT>' ] <EOL> else : <EOL> dictionary = dictionary [ '<STR_LIT>' ] <EOL> def hook_fn ( ) : <EOL> sample = sample_fn ( ) [ <NUM_LIT:0> ] <EOL> print '<STR_LIT>' , <EOL> if state [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> print "<STR_LIT>" . join ( dictionary [ sample ] ) <EOL> else : <EOL> for si in sample : <EOL> print dictionary [ si ] , <EOL> print <EOL> model = LM_Model ( <EOL> cost_layer = train_model , <EOL> weight_noise_amount = state [ '<STR_LIT>' ] , <EOL> valid_fn = valid_fn , <EOL> clean_before_noise_fn = False , <EOL> noise_fn = None , <EOL> rng = rng ) <EOL> if state [ '<STR_LIT>' ] : <EOL> model . load ( state [ '<STR_LIT>' ] + '<STR_LIT>' ) <EOL> algo = SGD ( model , state , train_data ) <EOL> main = MainLoop ( train_data , <EOL> valid_data , <EOL> test_data , <EOL> model , <EOL> algo , <EOL> state , <EOL> channel , <EOL> train_cost = False , <EOL> hooks = hook_fn , <EOL> validate_postprocess = eval ( state [ '<STR_LIT>' ] ) ) <EOL> main . main ( ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> state = { } <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:100> <EOL> state [ '<STR_LIT:path>' ] = "<STR_LIT>" <EOL> state [ '<STR_LIT>' ] = "<STR_LIT>" <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = - <NUM_LIT:1> <EOL> state [ '<STR_LIT>' ] = float ( <NUM_LIT> ) <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = - <NUM_LIT:1> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = - <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = - <NUM_LIT:1> <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = "<STR_LIT>" <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:1> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:1.> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:1.> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = True <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = int ( <NUM_LIT> ) <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> * <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = False <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:200> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:100> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:1000> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:15> <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = False <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:1> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:1> <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:10> * <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> state [ '<STR_LIT>' ] = True <EOL> state [ '<STR_LIT>' ] = True <EOL> state [ '<STR_LIT>' ] = <NUM_LIT> <EOL> state [ '<STR_LIT>' ] = - <NUM_LIT:1> <EOL> state [ '<STR_LIT>' ] = <NUM_LIT:1> <EOL> state [ '<STR_LIT>' ] = False <EOL> jobman ( state , None ) </s>
