<s> import logging <EOL> from collections import Counter <EOL> from theano import tensor <EOL> from toolz import merge <EOL> from blocks . algorithms import ( GradientDescent , StepClipping , AdaDelta , <EOL> CompositeRule ) <EOL> from blocks . extensions import FinishAfter , Printing <EOL> from blocks . extensions . monitoring import TrainingDataMonitoring <EOL> from blocks . filter import VariableFilter <EOL> from blocks . graph import ComputationGraph , apply_noise , apply_dropout <EOL> from blocks . initialization import IsotropicGaussian , Orthogonal , Constant <EOL> from blocks . main_loop import MainLoop <EOL> from blocks . model import Model <EOL> from blocks . select import Selector <EOL> from machine_translation . checkpoint import CheckpointNMT , LoadNMT <EOL> from machine_translation . model import BidirectionalEncoder , Decoder <EOL> from machine_translation . sampling import BleuValidator , Sampler <EOL> try : <EOL> from blocks_extras . extensions . plot import Plot <EOL> BOKEH_AVAILABLE = True <EOL> except ImportError : <EOL> BOKEH_AVAILABLE = False <EOL> logger = logging . getLogger ( __name__ ) <EOL> def main ( config , tr_stream , dev_stream , use_bokeh = False ) : <EOL> logger . info ( '<STR_LIT>' ) <EOL> source_sentence = tensor . lmatrix ( '<STR_LIT:source>' ) <EOL> source_sentence_mask = tensor . matrix ( '<STR_LIT>' ) <EOL> target_sentence = tensor . lmatrix ( '<STR_LIT:target>' ) <EOL> target_sentence_mask = tensor . matrix ( '<STR_LIT>' ) <EOL> sampling_input = tensor . lmatrix ( '<STR_LIT:input>' ) <EOL> logger . info ( '<STR_LIT>' ) <EOL> encoder = BidirectionalEncoder ( <EOL> config [ '<STR_LIT>' ] , config [ '<STR_LIT>' ] , config [ '<STR_LIT>' ] ) <EOL> decoder = Decoder ( <EOL> config [ '<STR_LIT>' ] , config [ '<STR_LIT>' ] , config [ '<STR_LIT>' ] , <EOL> config [ '<STR_LIT>' ] * <NUM_LIT:2> ) <EOL> cost = decoder . cost ( <EOL> encoder . apply ( source_sentence , source_sentence_mask ) , <EOL> source_sentence_mask , target_sentence , target_sentence_mask ) <EOL> logger . info ( '<STR_LIT>' ) <EOL> cg = ComputationGraph ( cost ) <EOL> logger . info ( '<STR_LIT>' ) <EOL> encoder . weights_init = decoder . weights_init = IsotropicGaussian ( <EOL> config [ '<STR_LIT>' ] ) <EOL> encoder . biases_init = decoder . biases_init = Constant ( <NUM_LIT:0> ) <EOL> encoder . push_initialization_config ( ) <EOL> decoder . push_initialization_config ( ) <EOL> encoder . bidir . prototype . weights_init = Orthogonal ( ) <EOL> decoder . transition . weights_init = Orthogonal ( ) <EOL> encoder . initialize ( ) <EOL> decoder . initialize ( ) <EOL> if config [ '<STR_LIT>' ] < <NUM_LIT:1.0> : <EOL> logger . info ( '<STR_LIT>' ) <EOL> dropout_inputs = [ x for x in cg . intermediary_variables <EOL> if x . name == '<STR_LIT>' ] <EOL> cg = apply_dropout ( cg , dropout_inputs , config [ '<STR_LIT>' ] ) <EOL> if config [ '<STR_LIT>' ] > <NUM_LIT:0.0> : <EOL> logger . info ( '<STR_LIT>' ) <EOL> enc_params = Selector ( encoder . lookup ) . get_params ( ) . values ( ) <EOL> enc_params += Selector ( encoder . fwd_fork ) . get_params ( ) . values ( ) <EOL> enc_params += Selector ( encoder . back_fork ) . get_params ( ) . values ( ) <EOL> dec_params = Selector ( <EOL> decoder . sequence_generator . readout ) . get_params ( ) . values ( ) <EOL> dec_params += Selector ( <EOL> decoder . sequence_generator . fork ) . get_params ( ) . values ( ) <EOL> dec_params += Selector ( decoder . state_init ) . get_params ( ) . values ( ) <EOL> cg = apply_noise ( cg , enc_params + dec_params , config [ '<STR_LIT>' ] ) <EOL> shapes = [ param . get_value ( ) . shape for param in cg . parameters ] <EOL> logger . info ( "<STR_LIT>" ) <EOL> for shape , count in Counter ( shapes ) . most_common ( ) : <EOL> logger . info ( '<STR_LIT>' . format ( shape , count ) ) <EOL> logger . info ( "<STR_LIT>" . format ( len ( shapes ) ) ) <EOL> enc_dec_param_dict = merge ( Selector ( encoder ) . get_parameters ( ) , <EOL> Selector ( decoder ) . get_parameters ( ) ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> for name , value in enc_dec_param_dict . items ( ) : <EOL> logger . info ( '<STR_LIT>' . format ( value . get_value ( ) . shape , name ) ) <EOL> logger . info ( "<STR_LIT>" <EOL> . format ( len ( enc_dec_param_dict ) ) ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> training_model = Model ( cost ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> extensions = [ <EOL> FinishAfter ( after_n_batches = config [ '<STR_LIT>' ] ) , <EOL> TrainingDataMonitoring ( [ cost ] , after_batch = True ) , <EOL> Printing ( after_batch = True ) , <EOL> CheckpointNMT ( config [ '<STR_LIT>' ] , <EOL> every_n_batches = config [ '<STR_LIT>' ] ) <EOL> ] <EOL> if config [ '<STR_LIT>' ] >= <NUM_LIT:1> or config [ '<STR_LIT>' ] is not None : <EOL> logger . info ( "<STR_LIT>" ) <EOL> sampling_representation = encoder . apply ( <EOL> sampling_input , tensor . ones ( sampling_input . shape ) ) <EOL> generated = decoder . generate ( sampling_input , sampling_representation ) <EOL> search_model = Model ( generated ) <EOL> _ , samples = VariableFilter ( <EOL> bricks = [ decoder . sequence_generator ] , name = "<STR_LIT>" ) ( <EOL> ComputationGraph ( generated [ <NUM_LIT:1> ] ) ) <EOL> if config [ '<STR_LIT>' ] >= <NUM_LIT:1> : <EOL> logger . info ( "<STR_LIT>" ) <EOL> extensions . append ( <EOL> Sampler ( model = search_model , data_stream = tr_stream , <EOL> hook_samples = config [ '<STR_LIT>' ] , <EOL> every_n_batches = config [ '<STR_LIT>' ] , <EOL> src_vocab_size = config [ '<STR_LIT>' ] ) ) <EOL> if config [ '<STR_LIT>' ] is not None : <EOL> logger . info ( "<STR_LIT>" ) <EOL> extensions . append ( <EOL> BleuValidator ( sampling_input , samples = samples , config = config , <EOL> model = search_model , data_stream = dev_stream , <EOL> normalize = config [ '<STR_LIT>' ] , <EOL> every_n_batches = config [ '<STR_LIT>' ] ) ) <EOL> if config [ '<STR_LIT>' ] : <EOL> extensions . append ( LoadNMT ( config [ '<STR_LIT>' ] ) ) <EOL> if use_bokeh and BOKEH_AVAILABLE : <EOL> extensions . append ( <EOL> Plot ( '<STR_LIT>' , channels = [ [ '<STR_LIT>' ] ] , <EOL> after_batch = True ) ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> algorithm = GradientDescent ( <EOL> cost = cost , parameters = cg . parameters , <EOL> step_rule = CompositeRule ( [ StepClipping ( config [ '<STR_LIT>' ] ) , <EOL> eval ( config [ '<STR_LIT>' ] ) ( ) ] ) <EOL> ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> main_loop = MainLoop ( <EOL> model = training_model , <EOL> algorithm = algorithm , <EOL> data_stream = tr_stream , <EOL> extensions = extensions <EOL> ) <EOL> main_loop . run ( ) </s>
