<s> """<STR_LIT>""" <EOL> usage = '<STR_LIT>' <EOL> import cPickle <EOL> import gzip <EOL> import os <EOL> import sys <EOL> import time <EOL> import re <EOL> import codecs <EOL> import argparse <EOL> import datetime <EOL> import numpy as np <EOL> import theano <EOL> import theano . tensor as T <EOL> import model_nnlm <EOL> import io_vocab <EOL> import io_read_ngram <EOL> import io_model <EOL> from train_util import * <EOL> def process_command_line ( ) : <EOL> """<STR_LIT>""" <EOL> parser = argparse . ArgumentParser ( description = usage ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , metavar = '<STR_LIT>' , type = str , help = '<STR_LIT>' ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , metavar = '<STR_LIT>' , type = str , help = '<STR_LIT>' ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , metavar = '<STR_LIT>' , type = str , help = '<STR_LIT>' ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , metavar = '<STR_LIT>' , type = int , help = '<STR_LIT>' ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , metavar = '<STR_LIT>' , type = int , help = '<STR_LIT>' ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , metavar = '<STR_LIT>' , type = str , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , type = int , <EOL> default = <NUM_LIT> , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , type = str , <EOL> default = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> type = float , default = <NUM_LIT:0.1> , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , type = int , default = <NUM_LIT> , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> type = int , default = <NUM_LIT:1000> , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , type = int , default = <NUM_LIT:0> , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , type = str , default = '<STR_LIT:relu>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , type = int , default = <NUM_LIT:1> , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , type = int , default = <NUM_LIT:5> , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , type = int , <EOL> default = <NUM_LIT:5> , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> type = str , default = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> type = str , default = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , type = str , <EOL> default = None , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = "<STR_LIT>" , type = str , default = None , help = "<STR_LIT>" ) <EOL> args = parser . parse_args ( ) <EOL> return args <EOL> class TrainNNJMModel ( TrainModel ) : <EOL> def loadModelParams ( self , ngram_size , src_window , model ) : <EOL> self . ngram_size = ngram_size <EOL> self . src_window = src_window <EOL> self . model = model <EOL> self . model_param_loaded = True <EOL> def getBatchData ( self ) : <EOL> src_lang = self . src_lang <EOL> tgt_lang = self . tgt_lang <EOL> tgt_vocab_size = self . tgt_vocab_size <EOL> ngram_size = self . ngram_size <EOL> is_shuffle = self . is_shuffle <EOL> chunk_size = self . chunk_size <EOL> src_window = self . src_window <EOL> opt = self . opt <EOL> ( data_x , data_y ) = io_read_ngram . get_joint_ngrams ( self . src_f , self . tgt_f , self . align_f , tgt_vocab_size , ngram_size , src_window , opt , num_read_lines = chunk_size ) <EOL> return ( data_x , data_y ) <EOL> def loadValidSet ( self , valid_data_package ) : <EOL> self . valid_set_x , self . valid_set_y = valid_data_package <EOL> self . shared_valid_set_x , self . shared_valid_set_y = io_read_ngram . shared_dataset ( valid_data_package ) <EOL> self . shared_valid_set_y = T . cast ( self . shared_valid_set_y , '<STR_LIT>' ) <EOL> self . valid_set_loaded = True <EOL> def loadTestSet ( self , test_data_package ) : <EOL> self . test_set_x , self . test_set_y = test_data_package <EOL> self . shared_test_set_x , self . shared_test_set_y = io_read_ngram . shared_dataset ( test_data_package ) <EOL> self . shared_test_set_y = T . cast ( self . shared_test_set_y , '<STR_LIT>' ) <EOL> self . test_set_loaded = True <EOL> def loadBatchData ( self , isInitialLoad = False ) : <EOL> src_lang = self . src_lang <EOL> tgt_lang = self . tgt_lang <EOL> tgt_vocab_size = self . tgt_vocab_size <EOL> ngram_size = self . ngram_size <EOL> chunk_size = self . chunk_size <EOL> src_window = self . src_window <EOL> opt = self . opt <EOL> ( self . data_x , self . data_y ) = io_read_ngram . get_joint_ngrams ( self . src_f , self . tgt_f , self . align_f , tgt_vocab_size , ngram_size , src_window , opt , num_read_lines = chunk_size ) <EOL> if isInitialLoad == False : <EOL> assert ( type ( self . model ) == model_nnlm . ModelNNLM ) <EOL> return self . model . updateTrainModelInput ( self . data_x , self . data_y ) <EOL> def displayFirstNExamples ( self , n ) : <EOL> if self . src_window < <NUM_LIT:0> : <EOL> return <EOL> src_vocab , src_vocab_size = io_vocab . load_vocab ( self . src_vocab_file ) <EOL> tgt_vocab , tgt_vocab_size = io_vocab . load_vocab ( self . tgt_vocab_file ) <EOL> src_inverse_vocab = io_vocab . inverse_vocab ( src_vocab ) <EOL> tgt_inverse_vocab = io_vocab . inverse_vocab ( tgt_vocab ) <EOL> assert ( n <= self . chunk_size ) <EOL> for i in xrange ( n ) : <EOL> example_x = self . data_x [ i ] <EOL> example_y = self . data_y [ i ] <EOL> sent_idx = example_x [ - <NUM_LIT:1> ] <EOL> src_window_vector = example_x [ : self . src_window * <NUM_LIT:2> + <NUM_LIT:1> ] <EOL> tgt_gram_vector = example_x [ self . src_window * <NUM_LIT:2> + <NUM_LIT:1> : ] <EOL> src_window_words = io_vocab . getWordsFromIndeces ( src_window_vector , src_inverse_vocab , self . tgt_vocab_size ) <EOL> tgt_gram_words = io_vocab . getWordsFromIndeces ( tgt_gram_vector , tgt_inverse_vocab , <NUM_LIT:0> ) <EOL> output = "<STR_LIT>" <EOL> count = <NUM_LIT:0> <EOL> for w in src_window_words : <EOL> count += <NUM_LIT:1> <EOL> if count == self . src_window + <NUM_LIT:1> : <EOL> output += "<STR_LIT:[>" + w + "<STR_LIT>" <EOL> else : <EOL> output += w + "<STR_LIT:U+0020>" <EOL> output += "<STR_LIT>" <EOL> output += "<STR_LIT:U+0020>" . join ( tgt_gram_words ) + "<STR_LIT:U+0020>" <EOL> output += "<STR_LIT>" + tgt_inverse_vocab [ example_y ] <EOL> print output <EOL> def trainOnBatch ( self , train_model , i , batch_size , num_train_batches , num_train_samples , learning_rate ) : <EOL> ngram_start_id = i * batch_size <EOL> ngram_end_id = ( i + <NUM_LIT:1> ) * batch_size if i < ( num_train_batches - <NUM_LIT:1> ) else num_train_samples <EOL> outputs = train_model ( ngram_start_id , ngram_end_id , learning_rate ) <EOL> return outputs <EOL> def buildModels ( self ) : <EOL> assert ( hasattr ( self , '<STR_LIT>' ) ) <EOL> print "<STR_LIT>" <EOL> train_model = self . model . getTrainModel ( self . data_x , self . data_y ) <EOL> print "<STR_LIT>" <EOL> valid_model = self . model . getValidationModel ( self . shared_valid_set_x , self . shared_valid_set_y , self . batch_size ) <EOL> print "<STR_LIT>" <EOL> test_model = self . model . getTestModel ( self . shared_test_set_x , self . shared_test_set_y , self . batch_size ) <EOL> print "<STR_LIT>" <EOL> return ( train_model , valid_model , test_model ) <EOL> def validate ( self , model , num_ngrams , batch_size , num_batches ) : <EOL> """<STR_LIT>""" <EOL> loss = <NUM_LIT:0.0> <EOL> for i in xrange ( num_batches ) : <EOL> ngram_start_id = i * batch_size <EOL> ngram_end_id = ( i + <NUM_LIT:1> ) * batch_size if i < ( num_batches - <NUM_LIT:1> ) else num_ngrams <EOL> loss -= model ( ngram_start_id , ngram_end_id ) <EOL> loss /= num_ngrams <EOL> perp = np . exp ( loss ) <EOL> return ( loss , perp ) <EOL> def test ( self , model , num_ngrams , batch_size , num_batches ) : <EOL> """<STR_LIT>""" <EOL> loss = <NUM_LIT:0.0> <EOL> for i in xrange ( num_batches ) : <EOL> ngram_start_id = i * batch_size <EOL> ngram_end_id = ( i + <NUM_LIT:1> ) * batch_size if i < ( num_batches - <NUM_LIT:1> ) else num_ngrams <EOL> loss -= model ( ngram_start_id , ngram_end_id ) <EOL> loss /= num_ngrams <EOL> perp = np . exp ( loss ) <EOL> return ( loss , perp ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> args = process_command_line ( ) <EOL> print "<STR_LIT>" % ( os . getpid ( ) ) <EOL> print_cml_args ( args ) <EOL> batch_size = <NUM_LIT> <EOL> emb_dim = args . emb_dim <EOL> hidden_layers = [ int ( x ) for x in re . split ( '<STR_LIT:->' , args . hidden_layers ) ] <EOL> train_file = args . train_file <EOL> learning_rate = args . learning_rate <EOL> ngram_size = args . ngram_size <EOL> valid_freq = args . valid_freq <EOL> opt = args . opt <EOL> chunk_size = args . chunk <EOL> act_func = args . act_func <EOL> src_lang = args . src_lang <EOL> tgt_lang = args . tgt_lang <EOL> vocab_size = args . vocab_size <EOL> finetune_epoch = args . finetune <EOL> n_epochs = args . n_epochs <EOL> src_window = args . src_window <EOL> pretrain_file = args . pretrain_file <EOL> assert src_lang != '<STR_LIT>' and tgt_lang != '<STR_LIT>' <EOL> src_vocab_file = args . vocab_file + '<STR_LIT:.>' + str ( args . vocab_size ) + '<STR_LIT>' + src_lang <EOL> tgt_vocab_file = args . vocab_file + '<STR_LIT:.>' + str ( args . vocab_size ) + '<STR_LIT>' + tgt_lang <EOL> ( src_vocab_map , src_vocab_size ) = io_vocab . load_vocab ( src_vocab_file ) <EOL> ( tgt_vocab_map , tgt_vocab_size ) = io_vocab . load_vocab ( tgt_vocab_file ) <EOL> src_valid_file = args . valid_file + '<STR_LIT:.>' + str ( args . vocab_size ) + '<STR_LIT>' + src_lang <EOL> tgt_valid_file = args . valid_file + '<STR_LIT:.>' + str ( args . vocab_size ) + '<STR_LIT>' + tgt_lang <EOL> ( valid_set_x , valid_set_y ) = io_read_ngram . get_all_joint_ngrams ( src_valid_file , tgt_valid_file , args . valid_file + '<STR_LIT>' , <EOL> tgt_vocab_size , ngram_size , src_window , opt ) <EOL> src_test_file = args . test_file + '<STR_LIT:.>' + str ( args . vocab_size ) + '<STR_LIT>' + src_lang <EOL> tgt_test_file = args . test_file + '<STR_LIT:.>' + str ( args . vocab_size ) + '<STR_LIT>' + tgt_lang <EOL> ( test_set_x , test_set_y ) = io_read_ngram . get_all_joint_ngrams ( src_test_file , tgt_test_file , args . test_file + '<STR_LIT>' , <EOL> tgt_vocab_size , ngram_size , src_window , opt ) <EOL> if src_window >= <NUM_LIT:0> : <EOL> context_size = <NUM_LIT:2> * src_window + ngram_size <EOL> else : <EOL> context_size = ngram_size - <NUM_LIT:1> <EOL> in_vocab_size = src_vocab_size + tgt_vocab_size <EOL> out_vocab_size = tgt_vocab_size <EOL> print "<STR_LIT>" <EOL> nnlm_model = model_nnlm . ModelNNLM ( context_size , in_vocab_size , emb_dim , hidden_layers , act_func , out_vocab_size , pretrain_file ) <EOL> nnlm_model . buildModel ( ) <EOL> print "<STR_LIT>" <EOL> train_model = TrainNNJMModel ( ) <EOL> train_model . loadVocab ( src_lang , tgt_lang , tgt_vocab_size , src_vocab_file , tgt_vocab_file ) <EOL> train_model . loadValidSet ( ( valid_set_x , valid_set_y ) ) <EOL> train_model . loadTestSet ( ( test_set_x , test_set_y ) ) <EOL> train_model . loadModelParams ( ngram_size , src_window , nnlm_model ) <EOL> train_model . loadTrainParams ( train_file , batch_size , learning_rate , opt , valid_freq , finetune_epoch , chunk_size , vocab_size , n_epochs ) <EOL> print "<STR_LIT>" <EOL> train_model . train ( ) <EOL> if args . save_model_file is not None : <EOL> io_model . save_model ( args . save_model_file , nnlm_model . classifier ) </s>
