<s> import logging <EOL> from argparse import ArgumentParser <EOL> import numpy <EOL> import theano <EOL> from theano import tensor <EOL> from theano . sandbox . rng_mrg import MRG_RandomStreams <EOL> from blocks . algorithms import ( <EOL> GradientDescent , StepClipping , CompositeRule , Adam <EOL> ) <EOL> from blocks . model import Model <EOL> from blocks . extensions import FinishAfter , Timing , Printing <EOL> from blocks . extensions . monitoring import TrainingDataMonitoring <EOL> from blocks . main_loop import MainLoop <EOL> from blocks . extensions import SimpleExtension <EOL> from blocks . bricks import Linear , Tanh <EOL> from blocks . bricks . lookup import LookupTable <EOL> from blocks . bricks . recurrent import SimpleRecurrent <EOL> from blocks . bricks . cost import CategoricalCrossEntropy , MisclassificationRate <EOL> from blocks . graph import ComputationGraph <EOL> from blocks . initialization import Uniform , Constant <EOL> from fuel . streams import DataStream <EOL> from fuel . transformers import Padding <EOL> from dl_tutorials . blocks . extensions . plot import ( <EOL> PlotManager , Plotter <EOL> ) <EOL> from dl_tutorials . utils . sequences import sequence_map <EOL> from dl_tutorials . fuel . datasets . text_file import TextFile <EOL> from dl_tutorials . fuel . schemes import BatchwiseShuffledScheme <EOL> from dl_tutorials . utils . softmax import softmax <EOL> class TupleMapping ( object ) : <EOL> def __init__ ( self , fn , same_len_out = False , same_len_in = False ) : <EOL> self . fn = fn <EOL> self . same_len_out = same_len_out <EOL> self . same_len_in = same_len_in <EOL> def __call__ ( self , args ) : <EOL> if self . same_len_in : <EOL> rval = ( self . fn ( * args ) , ) <EOL> else : <EOL> rval = ( self . fn ( args [ <NUM_LIT:0> ] ) , ) <EOL> if self . same_len_out : <EOL> rval += args [ <NUM_LIT:1> : ] <EOL> return rval <EOL> class PrintSamples ( SimpleExtension ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , sampler , voc , ** kwargs ) : <EOL> kwargs . setdefault ( "<STR_LIT>" , True ) <EOL> kwargs . setdefault ( "<STR_LIT>" , True ) <EOL> kwargs . setdefault ( "<STR_LIT>" , True ) <EOL> kwargs . setdefault ( "<STR_LIT>" , True ) <EOL> kwargs . setdefault ( "<STR_LIT>" , True ) <EOL> self . sampler = sampler <EOL> self . voc = voc <EOL> super ( PrintSamples , self ) . __init__ ( ** kwargs ) <EOL> def do ( self , which_callback , * args ) : <EOL> samples = self . sampler ( ) <EOL> for sample in samples : <EOL> print ( '<STR_LIT>' . join ( [ self . voc [ idx ] for idx in sample ] ) ) <EOL> def main ( num_epochs = <NUM_LIT:100> ) : <EOL> x = tensor . matrix ( '<STR_LIT>' ) <EOL> m = tensor . matrix ( '<STR_LIT>' ) <EOL> x_int = x . astype ( dtype = '<STR_LIT>' ) . T <EOL> train_dataset = TextFile ( '<STR_LIT>' ) <EOL> train_dataset . indexables [ <NUM_LIT:0> ] = numpy . array ( sorted ( <EOL> train_dataset . indexables [ <NUM_LIT:0> ] , key = len <EOL> ) ) <EOL> n_voc = len ( train_dataset . dict . keys ( ) ) <EOL> init_probs = numpy . array ( <EOL> [ sum ( filter ( lambda idx : idx == w , <EOL> [ s [ <NUM_LIT:0> ] for s in train_dataset . indexables [ <EOL> train_dataset . sources . index ( '<STR_LIT>' ) ] ] <EOL> ) ) for w in xrange ( n_voc ) ] , <EOL> dtype = theano . config . floatX <EOL> ) <EOL> init_probs = init_probs / init_probs . sum ( ) <EOL> n_h = <NUM_LIT:100> <EOL> linear_embedding = LookupTable ( <EOL> length = n_voc , <EOL> dim = n_h , <EOL> weights_init = Uniform ( std = <NUM_LIT> ) , <EOL> biases_init = Constant ( <NUM_LIT:0.> ) <EOL> ) <EOL> linear_embedding . initialize ( ) <EOL> lstm_biases = numpy . zeros ( <NUM_LIT:4> * n_h ) . astype ( dtype = theano . config . floatX ) <EOL> lstm_biases [ n_h : ( <NUM_LIT:2> * n_h ) ] = <NUM_LIT> <EOL> rnn = SimpleRecurrent ( <EOL> dim = n_h , <EOL> activation = Tanh ( ) , <EOL> weights_init = Uniform ( std = <NUM_LIT> ) , <EOL> biases_init = Constant ( <NUM_LIT:0.> ) <EOL> ) <EOL> rnn . initialize ( ) <EOL> score_layer = Linear ( <EOL> input_dim = n_h , <EOL> output_dim = n_voc , <EOL> weights_init = Uniform ( std = <NUM_LIT> ) , <EOL> biases_init = Constant ( <NUM_LIT:0.> ) <EOL> ) <EOL> score_layer . initialize ( ) <EOL> embedding = ( linear_embedding . apply ( x_int [ : - <NUM_LIT:1> ] ) <EOL> * tensor . shape_padright ( m . T [ <NUM_LIT:1> : ] ) ) <EOL> rnn_out = rnn . apply ( inputs = embedding , mask = m . T [ <NUM_LIT:1> : ] ) <EOL> probs = softmax ( <EOL> sequence_map ( score_layer . apply , rnn_out , mask = m . T [ <NUM_LIT:1> : ] ) [ <NUM_LIT:0> ] <EOL> ) <EOL> idx_mask = m . T [ <NUM_LIT:1> : ] . nonzero ( ) <EOL> cost = CategoricalCrossEntropy ( ) . apply ( <EOL> x_int [ <NUM_LIT:1> : ] [ idx_mask [ <NUM_LIT:0> ] , idx_mask [ <NUM_LIT:1> ] ] , <EOL> probs [ idx_mask [ <NUM_LIT:0> ] , idx_mask [ <NUM_LIT:1> ] ] <EOL> ) <EOL> cost . name = '<STR_LIT>' <EOL> misclassification = MisclassificationRate ( ) . apply ( <EOL> x_int [ <NUM_LIT:1> : ] [ idx_mask [ <NUM_LIT:0> ] , idx_mask [ <NUM_LIT:1> ] ] , <EOL> probs [ idx_mask [ <NUM_LIT:0> ] , idx_mask [ <NUM_LIT:1> ] ] <EOL> ) <EOL> misclassification . name = '<STR_LIT>' <EOL> cg = ComputationGraph ( [ cost ] ) <EOL> params = cg . parameters <EOL> algorithm = GradientDescent ( <EOL> cost = cost , <EOL> params = params , <EOL> step_rule = Adam ( ) <EOL> ) <EOL> train_data_stream = Padding ( <EOL> data_stream = DataStream ( <EOL> dataset = train_dataset , <EOL> iteration_scheme = BatchwiseShuffledScheme ( <EOL> examples = train_dataset . num_examples , <EOL> batch_size = <NUM_LIT:10> , <EOL> ) <EOL> ) , <EOL> mask_sources = ( '<STR_LIT>' , ) <EOL> ) <EOL> model = Model ( cost ) <EOL> extensions = [ ] <EOL> extensions . append ( Timing ( ) ) <EOL> extensions . append ( FinishAfter ( after_n_epochs = num_epochs ) ) <EOL> extensions . append ( TrainingDataMonitoring ( <EOL> [ cost , misclassification ] , <EOL> prefix = '<STR_LIT:train>' , <EOL> after_epoch = True ) ) <EOL> batch_size = <NUM_LIT:10> <EOL> length = <NUM_LIT:30> <EOL> trng = MRG_RandomStreams ( <NUM_LIT> ) <EOL> u = trng . uniform ( size = ( length , batch_size , n_voc ) ) <EOL> gumbel_noise = - tensor . log ( - tensor . log ( u ) ) <EOL> init_samples = ( tensor . log ( init_probs ) . dimshuffle ( ( '<STR_LIT:x>' , <NUM_LIT:0> ) ) <EOL> + gumbel_noise [ <NUM_LIT:0> ] ) . argmax ( axis = - <NUM_LIT:1> ) <EOL> init_states = rnn . initial_state ( '<STR_LIT>' , batch_size ) <EOL> def sampling_step ( g_noise , states , samples_step ) : <EOL> embedding_step = linear_embedding . apply ( samples_step ) <EOL> next_states = rnn . apply ( inputs = embedding_step , <EOL> states = states , <EOL> iterate = False ) <EOL> probs_step = softmax ( score_layer . apply ( next_states ) ) <EOL> next_samples = ( tensor . log ( probs_step ) <EOL> + g_noise ) . argmax ( axis = - <NUM_LIT:1> ) <EOL> return next_states , next_samples <EOL> [ _ , samples ] , _ = theano . scan ( <EOL> fn = sampling_step , <EOL> sequences = [ gumbel_noise [ <NUM_LIT:1> : ] ] , <EOL> outputs_info = [ init_states , init_samples ] <EOL> ) <EOL> sampling = theano . function ( [ ] , samples . owner . inputs [ <NUM_LIT:0> ] . T ) <EOL> plotters = [ ] <EOL> plotters . append ( Plotter ( <EOL> channels = [ [ '<STR_LIT>' , '<STR_LIT>' ] ] , <EOL> titles = [ '<STR_LIT>' ] ) ) <EOL> extensions . append ( PlotManager ( '<STR_LIT>' , <EOL> plotters = plotters , <EOL> after_epoch = True , <EOL> after_training = True ) ) <EOL> extensions . append ( Printing ( ) ) <EOL> extensions . append ( PrintSamples ( sampler = sampling , <EOL> voc = train_dataset . inv_dict ) ) <EOL> main_loop = MainLoop ( model = model , <EOL> data_stream = train_data_stream , <EOL> algorithm = algorithm , <EOL> extensions = extensions ) <EOL> main_loop . run ( ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> logging . basicConfig ( level = logging . INFO ) <EOL> parser = ArgumentParser ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> parser . add_argument ( "<STR_LIT>" , type = int , default = <NUM_LIT:100> , <EOL> help = "<STR_LIT>" ) <EOL> args = parser . parse_args ( ) <EOL> main ( num_epochs = args . num_epochs ) </s>
