<s> import numpy as np <EOL> import sys <EOL> import os <EOL> import utils <EOL> import models <EOL> import conf <EOL> from time import time <EOL> from itertools import izip <EOL> def _process_corpus ( net , dataset , mode = '<STR_LIT:train>' , learning_rate = None ) : <EOL> assert mode in ( '<STR_LIT:train>' , '<STR_LIT:test>' ) <EOL> counter = <NUM_LIT:0> <EOL> total_neg_log_prob = <NUM_LIT:0> <EOL> t0 = time ( ) <EOL> max_output_length = <NUM_LIT:0> <EOL> net . reset_state ( ) <EOL> for inputs , outputs , pauses in izip ( dataset [ "<STR_LIT>" ] , dataset [ "<STR_LIT>" ] , dataset . get ( "<STR_LIT>" , [ None ] * dataset [ "<STR_LIT>" ] ) ) : <EOL> if mode == '<STR_LIT:train>' : <EOL> neg_log_probs = net . train ( inputs , outputs , pauses , learning_rate ) <EOL> elif mode == '<STR_LIT:test>' : <EOL> neg_log_probs = net . neg_log_prob ( inputs , outputs , pauses ) <EOL> total_neg_log_prob += np . sum ( neg_log_probs ) <EOL> counter += dataset [ "<STR_LIT>" ] <EOL> if conf . SHOW_WPS : <EOL> elapsed_time = time ( ) - t0 <EOL> if elapsed_time > <NUM_LIT:0> : <EOL> output = "<STR_LIT>" % ( float ( counter ) / dataset [ "<STR_LIT>" ] * <NUM_LIT:100> , <EOL> counter / elapsed_time ) <EOL> max_output_length = max ( max_output_length , len ( output ) ) <EOL> sys . stdout . write ( output + "<STR_LIT:\r>" ) <EOL> sys . stdout . flush ( ) <EOL> if conf . SHOW_WPS : <EOL> sys . stdout . write ( "<STR_LIT:U+0020>" * max_output_length + "<STR_LIT:\r>" ) <EOL> sys . stdout . flush ( ) <EOL> ppl = np . exp ( total_neg_log_prob / counter ) <EOL> return ppl <EOL> def _train ( net , training_data , validation_data , model_name , learning_rate , max_epochs , min_improvement ) : <EOL> min_learning_rate = <NUM_LIT> <EOL> best_validation_ppl = np . inf <EOL> divide = False <EOL> for epoch in range ( <NUM_LIT:1> , max_epochs + <NUM_LIT:1> ) : <EOL> epoch_start = time ( ) <EOL> print "<STR_LIT>" % epoch <EOL> print "<STR_LIT>" % learning_rate <EOL> train_ppl = _process_corpus ( net , training_data , mode = '<STR_LIT:train>' , learning_rate = learning_rate ) <EOL> print "<STR_LIT>" % train_ppl <EOL> validation_ppl = _process_corpus ( net , validation_data , mode = '<STR_LIT:test>' ) <EOL> print "<STR_LIT>" % validation_ppl <EOL> print "<STR_LIT>" % ( time ( ) - epoch_start ) <EOL> if np . log ( validation_ppl ) * min_improvement > np . log ( best_validation_ppl ) : <EOL> if not divide : <EOL> divide = True <EOL> print "<STR_LIT>" <EOL> if validation_ppl > best_validation_ppl : <EOL> print "<STR_LIT>" <EOL> net = utils . load_model ( "<STR_LIT>" + model_name ) <EOL> else : <EOL> if validation_ppl < best_validation_ppl : <EOL> print "<STR_LIT>" <EOL> net . save ( "<STR_LIT>" + model_name , final = True ) <EOL> break <EOL> else : <EOL> print "<STR_LIT>" <EOL> best_validation_ppl = validation_ppl <EOL> final = learning_rate / <NUM_LIT> < min_learning_rate or epoch == max_epochs <EOL> net . save ( "<STR_LIT>" + model_name , final ) <EOL> if divide : <EOL> learning_rate /= <NUM_LIT> <EOL> if learning_rate < min_learning_rate : <EOL> break <EOL> print "<STR_LIT:->" * <NUM_LIT:30> <EOL> print "<STR_LIT>" <EOL> print "<STR_LIT>" % best_validation_ppl <EOL> def train ( model_name , p1_train_data , p1_dev_data , p2_train_data , p2_dev_data ) : <EOL> training_data = np . load ( p1_train_data ) <EOL> validation_data = np . load ( p1_dev_data ) <EOL> assert training_data [ "<STR_LIT>" ] == validation_data [ "<STR_LIT>" ] <EOL> assert training_data [ "<STR_LIT>" ] == validation_data [ "<STR_LIT>" ] <EOL> assert training_data [ "<STR_LIT>" ] == validation_data [ "<STR_LIT>" ] <EOL> print "<STR_LIT>" <EOL> print "<STR_LIT>" % utils . get_vocabulary_size ( validation_data [ "<STR_LIT>" ] ) <EOL> print "<STR_LIT>" % training_data [ "<STR_LIT>" ] <EOL> print "<STR_LIT>" % validation_data [ "<STR_LIT>" ] <EOL> net = models . T_LSTM ( ) <EOL> net . initialize ( hidden_size = conf . PHASE1 [ "<STR_LIT>" ] , <EOL> projection_size = conf . PHASE1 [ "<STR_LIT>" ] , <EOL> in_vocabulary = training_data [ "<STR_LIT>" ] , <EOL> out_vocabulary = training_data [ "<STR_LIT>" ] , <EOL> batch_size = training_data [ "<STR_LIT>" ] , <EOL> hidden_activation = conf . PHASE1 [ "<STR_LIT>" ] , <EOL> bptt_steps = conf . PHASE1 [ "<STR_LIT>" ] , <EOL> use_pauses = False ) <EOL> _train ( net , training_data , validation_data , model_name , conf . PHASE1 [ "<STR_LIT>" ] , conf . PHASE1 [ "<STR_LIT>" ] , conf . PHASE1 [ "<STR_LIT>" ] ) <EOL> if not os . path . isfile ( p2_train_data ) or not os . path . isfile ( p2_train_data ) : <EOL> print "<STR_LIT>" <EOL> return <EOL> training_data = np . load ( p2_train_data ) <EOL> validation_data = np . load ( p2_dev_data ) <EOL> assert training_data [ "<STR_LIT>" ] == validation_data [ "<STR_LIT>" ] == net . batch_size <EOL> assert training_data [ "<STR_LIT>" ] == validation_data [ "<STR_LIT>" ] == net . in_vocabulary <EOL> assert training_data [ "<STR_LIT>" ] == validation_data [ "<STR_LIT>" ] == net . out_vocabulary <EOL> print "<STR_LIT>" <EOL> print "<STR_LIT>" % training_data [ "<STR_LIT>" ] <EOL> print "<STR_LIT>" % validation_data [ "<STR_LIT>" ] <EOL> print "<STR_LIT>" % ( "<STR_LIT>" if conf . PHASE2 [ "<STR_LIT>" ] else "<STR_LIT>" ) <EOL> t_lstm = net <EOL> net = models . TA_LSTM ( ) <EOL> net . initialize ( hidden_size = conf . PHASE2 [ "<STR_LIT>" ] , <EOL> t_lstm = t_lstm , <EOL> out_vocabulary = training_data [ "<STR_LIT>" ] , <EOL> batch_size = training_data [ "<STR_LIT>" ] , <EOL> hidden_activation = conf . PHASE2 [ "<STR_LIT>" ] , <EOL> bptt_steps = conf . PHASE2 [ "<STR_LIT>" ] , <EOL> use_pauses = conf . PHASE2 [ "<STR_LIT>" ] ) <EOL> _train ( net , training_data , validation_data , model_name , conf . PHASE2 [ "<STR_LIT>" ] , conf . PHASE2 [ "<STR_LIT>" ] , conf . PHASE2 [ "<STR_LIT>" ] ) </s>
