<s> import os <EOL> import logging <EOL> import numpy as np <EOL> from quagga import Model <EOL> from quagga . utils import List <EOL> from quagga . cuda import cudart <EOL> from urllib import urlretrieve <EOL> from quagga . matrix import Matrix <EOL> from quagga . context import Context <EOL> from quagga . blocks import DotBlock <EOL> from collections import defaultdict <EOL> from quagga . blocks import LstmBlock <EOL> from quagga . blocks import RepeatBlock <EOL> from quagga . connector import Connector <EOL> from quagga . optimizers import Optimizer <EOL> from quagga . blocks import SequencerBlock <EOL> from quagga . blocks import SoftmaxCeBlock <EOL> from quagga . blocks import RowSlicingBlock <EOL> from quagga . blocks import ParameterContainer <EOL> from quagga . utils . initializers import Constant <EOL> from quagga . blocks import HorizontalStackBlock <EOL> from quagga . utils . initializers import Orthogonal <EOL> from quagga . optimizers . observers import Hdf5Saver <EOL> from quagga . optimizers . observers import ValidTracker , LossForValidTracker <EOL> from quagga . optimizers . observers import TrainLossTracker <EOL> from quagga . optimizers . policies import FixedValuePolicy <EOL> from quagga . optimizers . policies import FixedValuePolicy <EOL> from quagga . optimizers . stopping_criteria import MaxIterCriterion <EOL> from quagga . optimizers . steps import NagStep , SparseSgdStep , SgdStep <EOL> def get_logger ( file_name ) : <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> handler = logging . FileHandler ( file_name , mode = '<STR_LIT:w>' ) <EOL> handler . setFormatter ( logging . Formatter ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> logger . addHandler ( handler ) <EOL> logger . setLevel ( logging . INFO ) <EOL> return logger <EOL> def load_ptb_dataset ( ) : <EOL> train_file_path = '<STR_LIT>' <EOL> valid_file_path = '<STR_LIT>' <EOL> test_file_path = '<STR_LIT>' <EOL> if not os . path . exists ( train_file_path ) : <EOL> urlretrieve ( '<STR_LIT>' , train_file_path ) <EOL> if not os . path . exists ( valid_file_path ) : <EOL> urlretrieve ( '<STR_LIT>' , valid_file_path ) <EOL> if not os . path . exists ( test_file_path ) : <EOL> urlretrieve ( '<STR_LIT>' , test_file_path ) <EOL> vocab = { } <EOL> idx_to_word = [ ] <EOL> ptb_train = [ ] <EOL> with open ( train_file_path ) as f : <EOL> for line in f : <EOL> sentence = [ '<STR_LIT>' ] + line . strip ( ) . split ( ) + [ '<STR_LIT>' ] <EOL> for word in sentence : <EOL> if word not in vocab : <EOL> vocab [ word ] = len ( idx_to_word ) <EOL> idx_to_word . append ( word ) <EOL> ptb_train . append ( [ vocab [ word ] for word in sentence ] ) <EOL> ptb_valid = [ ] <EOL> with open ( valid_file_path ) as f : <EOL> for line in f : <EOL> sentence = [ '<STR_LIT>' ] + line . strip ( ) . split ( ) + [ '<STR_LIT>' ] <EOL> ptb_valid . append ( [ vocab [ word ] for word in sentence ] ) <EOL> ptb_test = [ ] <EOL> with open ( test_file_path ) as f : <EOL> for line in f : <EOL> sentence = [ '<STR_LIT>' ] + line . strip ( ) . split ( ) + [ '<STR_LIT>' ] <EOL> ptb_test . append ( [ vocab [ word ] for word in sentence ] ) <EOL> return ptb_train , ptb_valid , ptb_test , vocab , idx_to_word <EOL> class HomogeneousDataGenerator ( object ) : <EOL> def __init__ ( self , sentences , batch_size , sentence_max_len , randomize = False , infinite = False ) : <EOL> sentences = [ s for s in sentences if len ( s ) <= sentence_max_len ] <EOL> self . b_size = batch_size <EOL> self . flatten_sentences = [ w for s in sentences for w in s ] <EOL> self . offsets = defaultdict ( list ) <EOL> for s_offsets in self . __get_sentence_offsets ( sentences ) : <EOL> self . offsets [ s_offsets [ <NUM_LIT:1> ] - s_offsets [ <NUM_LIT:0> ] ] . append ( s_offsets ) <EOL> if randomize : <EOL> self . rng = np . random . RandomState ( <NUM_LIT> ) <EOL> self . infinite = infinite <EOL> def __iter__ ( self ) : <EOL> while True : <EOL> for batch_offsets in self . __iter ( ) : <EOL> yield batch_offsets <EOL> if not self . infinite : <EOL> break <EOL> def __iter ( self ) : <EOL> randomize = hasattr ( self , '<STR_LIT>' ) <EOL> if randomize : <EOL> for s_offsets in self . offsets . itervalues ( ) : <EOL> self . rng . shuffle ( s_offsets ) <EOL> progress = defaultdict ( int ) <EOL> available_lengths = self . offsets . keys ( ) <EOL> if randomize : <EOL> get_sent_len = lambda : self . rng . choice ( available_lengths ) <EOL> else : <EOL> get_sent_len = lambda : available_lengths [ <NUM_LIT:0> ] <EOL> batch_offsets = [ ] <EOL> b_size = self . b_size <EOL> k = get_sent_len ( ) <EOL> while available_lengths : <EOL> batch_offsets . extend ( self . offsets [ k ] [ progress [ k ] : progress [ k ] + b_size ] ) <EOL> progress [ k ] += b_size <EOL> if len ( batch_offsets ) == self . b_size : <EOL> yield batch_offsets <EOL> batch_offsets = [ ] <EOL> b_size = self . b_size <EOL> k = get_sent_len ( ) <EOL> else : <EOL> b_size = self . b_size - len ( batch_offsets ) <EOL> i = available_lengths . index ( k ) <EOL> del available_lengths [ i ] <EOL> if not available_lengths : <EOL> break <EOL> if i == <NUM_LIT:0> : <EOL> k = available_lengths [ <NUM_LIT:0> ] <EOL> elif i >= len ( available_lengths ) - <NUM_LIT:1> : <EOL> k = available_lengths [ - <NUM_LIT:1> ] <EOL> else : <EOL> k = available_lengths [ i + self . rng . choice ( [ - <NUM_LIT:1> , <NUM_LIT:1> ] ) ] <EOL> if batch_offsets : <EOL> yield batch_offsets <EOL> @ staticmethod <EOL> def __get_sentence_offsets ( sentences ) : <EOL> sentence_offsets = [ ] <EOL> offset = <NUM_LIT:0> <EOL> for s in sentences : <EOL> sentence_offsets . append ( ( offset , offset + len ( s ) ) ) <EOL> offset += len ( s ) <EOL> return sentence_offsets <EOL> class PtbMiniBatchesGenerator ( object ) : <EOL> def __init__ ( self , ptb_train , ptb_valid , batch_size , sentence_max_len , device_id ) : <EOL> self . blocking_contexts = None <EOL> self . context = Context ( device_id ) <EOL> device_id = self . context . device_id <EOL> self . train_offsets = HomogeneousDataGenerator ( ptb_train , batch_size , sentence_max_len , randomize = True , infinite = True ) <EOL> self . valid_offsets = HomogeneousDataGenerator ( ptb_valid , batch_size , sentence_max_len ) <EOL> train_sentences = np . array ( [ self . train_offsets . flatten_sentences ] ) <EOL> valid_sentences = np . array ( [ self . valid_offsets . flatten_sentences ] ) <EOL> self . train_sents = Matrix . from_npa ( train_sentences , '<STR_LIT:int>' , device_id ) <EOL> self . valid_sents = Matrix . from_npa ( valid_sentences , '<STR_LIT:int>' , device_id ) <EOL> self . _sent_lengths = np . empty ( ( batch_size , <NUM_LIT:1> ) , dtype = np . int32 , order = '<STR_LIT:F>' ) [ ... ] <EOL> self . sent_lengths = Matrix . from_npa ( self . _sent_lengths , device_id = device_id ) <EOL> sentence_batch = Matrix . empty ( batch_size , sentence_max_len , '<STR_LIT:int>' , device_id ) <EOL> self . sentence_batch = Connector ( sentence_batch , self . context ) <EOL> self . sentence_batch . sync_fill ( <NUM_LIT:0> ) <EOL> self . _mask = Matrix . empty ( sentence_batch . nrows , self . sentence_batch . ncols , '<STR_LIT:float>' , device_id ) <EOL> self . mask = List ( [ Connector ( self . _mask [ : , i ] ) for i in xrange ( sentence_max_len ) ] , self . sentence_batch . ncols ) <EOL> self . train_offsets_iterator = iter ( self . train_offsets ) <EOL> self . valid_offsets_iterator = iter ( self . valid_offsets ) <EOL> self . training_mode = True <EOL> def set_training_mode ( self ) : <EOL> self . training_mode = True <EOL> def set_testing_mode ( self ) : <EOL> self . training_mode = False <EOL> def fprop ( self ) : <EOL> if self . training_mode : <EOL> offsets = next ( self . train_offsets_iterator ) <EOL> sents = self . train_sents <EOL> else : <EOL> try : <EOL> offsets = next ( self . valid_offsets_iterator ) <EOL> sents = self . valid_sents <EOL> except StopIteration as e : <EOL> self . valid_offsets_iterator = iter ( self . valid_offsets ) <EOL> raise e <EOL> self . context . wait ( * self . blocking_contexts ) <EOL> self . _sent_lengths = self . _sent_lengths . base [ : len ( offsets ) ] <EOL> self . sentence_batch . nrows = len ( offsets ) <EOL> for k , offset in enumerate ( offsets ) : <EOL> self . sentence_batch [ k ] . assign ( self . context , sents [ : , offset [ <NUM_LIT:0> ] : offset [ <NUM_LIT:1> ] ] ) <EOL> self . _sent_lengths [ k ] = offset [ <NUM_LIT:1> ] - offset [ <NUM_LIT:0> ] <EOL> max_sent_len = int ( np . max ( self . _sent_lengths ) ) <EOL> self . sentence_batch . last_modification_context = self . context <EOL> self . sentence_batch . ncols = max_sent_len <EOL> self . sent_lengths . assign_npa ( self . context , self . _sent_lengths ) <EOL> self . _mask . mask_column_numbers_row_wise ( self . context , self . sent_lengths ) <EOL> for e in self . mask : <EOL> e . last_modification_context = self . context <EOL> self . sentence_batch . fprop ( ) <EOL> self . mask . fprop ( ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> ptb_train , ptb_valid , ptb_test , vocab , idx_to_word = load_ptb_dataset ( ) <EOL> print len ( ptb_train ) , len ( ptb_valid ) , len ( ptb_test ) , len ( idx_to_word ) <EOL> get_orth_W = Orthogonal ( <NUM_LIT> , <NUM_LIT> ) <EOL> get_orth_R = Orthogonal ( <NUM_LIT> , <NUM_LIT> ) <EOL> p = ParameterContainer ( embd_W = { '<STR_LIT>' : Orthogonal ( len ( vocab ) , <NUM_LIT> ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> } , <EOL> lstm_fwd_c0 = { '<STR_LIT>' : Constant ( <NUM_LIT:1> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> '<STR_LIT>' : False } , <EOL> lstm_fwd_h0 = { '<STR_LIT>' : Constant ( <NUM_LIT:1> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> '<STR_LIT>' : False } , <EOL> lstm_fwd_W = { '<STR_LIT>' : lambda : np . hstack ( ( get_orth_W ( ) , get_orth_W ( ) , get_orth_W ( ) , get_orth_W ( ) ) ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> } , <EOL> lstm_fwd_R = { '<STR_LIT>' : lambda : np . hstack ( ( get_orth_R ( ) , get_orth_R ( ) , get_orth_R ( ) , get_orth_R ( ) ) ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> } , <EOL> lstm_bwd_c0 = { '<STR_LIT>' : Constant ( <NUM_LIT:1> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> '<STR_LIT>' : False } , <EOL> lstm_bwd_h0 = { '<STR_LIT>' : Constant ( <NUM_LIT:1> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> '<STR_LIT>' : False } , <EOL> lstm_bwd_W = { '<STR_LIT>' : lambda : np . hstack ( ( get_orth_W ( ) , get_orth_W ( ) , get_orth_W ( ) , get_orth_W ( ) ) ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> } , <EOL> lstm_bwd_R = { '<STR_LIT>' : lambda : np . hstack ( ( get_orth_R ( ) , get_orth_R ( ) , get_orth_R ( ) , get_orth_R ( ) ) ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> } , <EOL> sce_dot_block_W = { '<STR_LIT>' : Orthogonal ( <NUM_LIT> , len ( vocab ) ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> } , <EOL> sce_dot_block_b = { '<STR_LIT>' : Constant ( <NUM_LIT:1> , len ( vocab ) ) , <EOL> '<STR_LIT>' : <NUM_LIT:0> } ) <EOL> data_block = PtbMiniBatchesGenerator ( ptb_train , ptb_valid , batch_size = <NUM_LIT:64> , sentence_max_len = <NUM_LIT:100> , device_id = <NUM_LIT:0> ) <EOL> seq_embd_block = RowSlicingBlock ( p [ '<STR_LIT>' ] , data_block . sentence_batch ) <EOL> output = List ( seq_embd_block . output [ : - <NUM_LIT:1> ] , seq_embd_block . output . length - <NUM_LIT:1> ) <EOL> c_fwd_repeat_block = RepeatBlock ( p [ '<STR_LIT>' ] , data_block . sentence_batch . nrows , axis = <NUM_LIT:0> , device_id = <NUM_LIT:0> ) <EOL> h_fwd_repeat_block = RepeatBlock ( p [ '<STR_LIT>' ] , data_block . sentence_batch . nrows , axis = <NUM_LIT:0> , device_id = <NUM_LIT:0> ) <EOL> fwd_lstm_block = SequencerBlock ( block_class = LstmBlock , <EOL> params = [ p [ '<STR_LIT>' ] , p [ '<STR_LIT>' ] , <NUM_LIT:0.5> ] , <EOL> sequences = [ output , data_block . mask ] , <EOL> output_names = [ '<STR_LIT:h>' ] , <EOL> prev_names = [ '<STR_LIT:c>' , '<STR_LIT:h>' ] , <EOL> paddings = [ c_fwd_repeat_block . output , h_fwd_repeat_block . output ] , <EOL> reverse = False , <EOL> device_id = <NUM_LIT:0> ) <EOL> output = List ( seq_embd_block . output [ <NUM_LIT:1> : ] , seq_embd_block . output . length - <NUM_LIT:1> ) <EOL> c_bwd_repeat_block = RepeatBlock ( p [ '<STR_LIT>' ] , data_block . sentence_batch . nrows , axis = <NUM_LIT:0> , device_id = <NUM_LIT:0> ) <EOL> h_bwd_repeat_block = RepeatBlock ( p [ '<STR_LIT>' ] , data_block . sentence_batch . nrows , axis = <NUM_LIT:0> , device_id = <NUM_LIT:0> ) <EOL> bwd_lstm_block = SequencerBlock ( block_class = LstmBlock , <EOL> params = [ p [ '<STR_LIT>' ] , p [ '<STR_LIT>' ] , <NUM_LIT:0.5> ] , <EOL> sequences = [ output , data_block . mask ] , <EOL> output_names = [ '<STR_LIT:h>' ] , <EOL> prev_names = [ '<STR_LIT:c>' , '<STR_LIT:h>' ] , <EOL> paddings = [ c_bwd_repeat_block . output , h_bwd_repeat_block . output ] , <EOL> reverse = True , <EOL> device_id = <NUM_LIT:0> ) <EOL> seq_hstack = SequencerBlock ( block_class = HorizontalStackBlock , <EOL> params = [ ] , <EOL> sequences = [ List ( [ h_fwd_repeat_block . output ] + fwd_lstm_block . h [ : ] , fwd_lstm_block . h . length + <NUM_LIT:1> ) , <EOL> List ( bwd_lstm_block . h [ : ] + [ h_bwd_repeat_block . output ] , bwd_lstm_block . h . length + <NUM_LIT:1> ) ] , <EOL> output_names = [ '<STR_LIT>' ] , <EOL> device_id = <NUM_LIT:0> ) <EOL> seq_dot_block = SequencerBlock ( block_class = DotBlock , <EOL> params = [ p [ '<STR_LIT>' ] , p [ '<STR_LIT>' ] ] , <EOL> sequences = [ seq_hstack . output ] , <EOL> output_names = [ '<STR_LIT>' ] , <EOL> device_id = <NUM_LIT:0> ) <EOL> sentence_batch = List ( [ Connector ( data_block . sentence_batch [ : , i ] ) for i in xrange ( data_block . sentence_batch . ncols ) ] , data_block . sentence_batch . ncols ) <EOL> seq_sce_block = SequencerBlock ( block_class = SoftmaxCeBlock , <EOL> params = [ ] , <EOL> sequences = [ seq_dot_block . output , sentence_batch , data_block . mask ] , <EOL> device_id = <NUM_LIT:0> ) <EOL> model = Model ( [ p , data_block , seq_embd_block , <EOL> c_fwd_repeat_block , h_fwd_repeat_block , fwd_lstm_block , <EOL> c_bwd_repeat_block , h_bwd_repeat_block , bwd_lstm_block , <EOL> seq_hstack , seq_dot_block , seq_sce_block ] ) <EOL> logger = get_logger ( '<STR_LIT>' ) <EOL> momentum_policy = FixedValuePolicy ( <NUM_LIT> ) <EOL> train_loss_tracker = TrainLossTracker ( model , <NUM_LIT:100> , logger ) <EOL> valid_tracker = ValidTracker ( model , <NUM_LIT> , logger ) <EOL> loss_tracker = LossForValidTracker ( logger ) <EOL> valid_tracker . add_observer ( loss_tracker ) <EOL> saver = Hdf5Saver ( p . trainable_parameters , <NUM_LIT> , '<STR_LIT>' , logger ) <EOL> trainable_parameters = dict ( p . trainable_parameters ) <EOL> sparse_sgd_step = SparseSgdStep ( [ trainable_parameters [ '<STR_LIT>' ] ] , FixedValuePolicy ( <NUM_LIT> ) ) <EOL> del trainable_parameters [ '<STR_LIT>' ] <EOL> nag_step = NagStep ( trainable_parameters . values ( ) , FixedValuePolicy ( <NUM_LIT> ) , momentum_policy ) <EOL> data_block . blocking_contexts = nag_step . blocking_contexts + sparse_sgd_step . blocking_contexts <EOL> criterion = MaxIterCriterion ( <NUM_LIT> ) <EOL> optimizer = Optimizer ( criterion , model ) <EOL> optimizer . add_observer ( sparse_sgd_step ) <EOL> optimizer . add_observer ( nag_step ) <EOL> optimizer . add_observer ( train_loss_tracker ) <EOL> optimizer . add_observer ( valid_tracker ) <EOL> optimizer . add_observer ( saver ) <EOL> optimizer . add_observer ( criterion ) <EOL> optimizer . optimize ( ) <EOL> for device_id in xrange ( cudart . cuda_get_device_count ( ) ) : <EOL> cudart . cuda_set_device ( device_id ) <EOL> cudart . cuda_device_synchronize ( ) </s>
