<s> """<STR_LIT>""" <EOL> from __future__ import print_function <EOL> import os <EOL> import sys <EOL> import timeit <EOL> import numpy <EOL> import theano <EOL> import theano . tensor as T <EOL> from theano . tensor . shared_randomstreams import RandomStreams <EOL> from logistic_sgd import LogisticRegression , load_data <EOL> from mlp import HiddenLayer <EOL> from dA import dA <EOL> class SdA ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( <EOL> self , <EOL> numpy_rng , <EOL> theano_rng = None , <EOL> n_ins = <NUM_LIT> , <EOL> hidden_layers_sizes = [ <NUM_LIT> , <NUM_LIT> ] , <EOL> n_outs = <NUM_LIT:10> , <EOL> corruption_levels = [ <NUM_LIT:0.1> , <NUM_LIT:0.1> ] <EOL> ) : <EOL> """<STR_LIT>""" <EOL> self . sigmoid_layers = [ ] <EOL> self . dA_layers = [ ] <EOL> self . params = [ ] <EOL> self . n_layers = len ( hidden_layers_sizes ) <EOL> assert self . n_layers > <NUM_LIT:0> <EOL> if not theano_rng : <EOL> theano_rng = RandomStreams ( numpy_rng . randint ( <NUM_LIT:2> ** <NUM_LIT:30> ) ) <EOL> self . x = T . matrix ( '<STR_LIT:x>' ) <EOL> self . y = T . ivector ( '<STR_LIT:y>' ) <EOL> for i in range ( self . n_layers ) : <EOL> if i == <NUM_LIT:0> : <EOL> input_size = n_ins <EOL> else : <EOL> input_size = hidden_layers_sizes [ i - <NUM_LIT:1> ] <EOL> if i == <NUM_LIT:0> : <EOL> layer_input = self . x <EOL> else : <EOL> layer_input = self . sigmoid_layers [ - <NUM_LIT:1> ] . output <EOL> sigmoid_layer = HiddenLayer ( rng = numpy_rng , <EOL> input = layer_input , <EOL> n_in = input_size , <EOL> n_out = hidden_layers_sizes [ i ] , <EOL> activation = T . nnet . sigmoid ) <EOL> self . sigmoid_layers . append ( sigmoid_layer ) <EOL> self . params . extend ( sigmoid_layer . params ) <EOL> dA_layer = dA ( numpy_rng = numpy_rng , <EOL> theano_rng = theano_rng , <EOL> input = layer_input , <EOL> n_visible = input_size , <EOL> n_hidden = hidden_layers_sizes [ i ] , <EOL> W = sigmoid_layer . W , <EOL> bhid = sigmoid_layer . b ) <EOL> self . dA_layers . append ( dA_layer ) <EOL> self . logLayer = LogisticRegression ( <EOL> input = self . sigmoid_layers [ - <NUM_LIT:1> ] . output , <EOL> n_in = hidden_layers_sizes [ - <NUM_LIT:1> ] , <EOL> n_out = n_outs <EOL> ) <EOL> self . params . extend ( self . logLayer . params ) <EOL> self . finetune_cost = self . logLayer . negative_log_likelihood ( self . y ) <EOL> self . errors = self . logLayer . errors ( self . y ) <EOL> def pretraining_functions ( self , train_set_x , batch_size ) : <EOL> '''<STR_LIT>''' <EOL> index = T . lscalar ( '<STR_LIT:index>' ) <EOL> corruption_level = T . scalar ( '<STR_LIT>' ) <EOL> learning_rate = T . scalar ( '<STR_LIT>' ) <EOL> batch_begin = index * batch_size <EOL> batch_end = batch_begin + batch_size <EOL> pretrain_fns = [ ] <EOL> for dA in self . dA_layers : <EOL> cost , updates = dA . get_cost_updates ( corruption_level , <EOL> learning_rate ) <EOL> fn = theano . function ( <EOL> inputs = [ <EOL> index , <EOL> theano . In ( corruption_level , value = <NUM_LIT> ) , <EOL> theano . In ( learning_rate , value = <NUM_LIT:0.1> ) <EOL> ] , <EOL> outputs = cost , <EOL> updates = updates , <EOL> givens = { <EOL> self . x : train_set_x [ batch_begin : batch_end ] <EOL> } <EOL> ) <EOL> pretrain_fns . append ( fn ) <EOL> return pretrain_fns <EOL> def build_finetune_functions ( self , datasets , batch_size , learning_rate ) : <EOL> '''<STR_LIT>''' <EOL> ( train_set_x , train_set_y ) = datasets [ <NUM_LIT:0> ] <EOL> ( valid_set_x , valid_set_y ) = datasets [ <NUM_LIT:1> ] <EOL> ( test_set_x , test_set_y ) = datasets [ <NUM_LIT:2> ] <EOL> n_valid_batches = valid_set_x . get_value ( borrow = True ) . shape [ <NUM_LIT:0> ] <EOL> n_valid_batches //= batch_size <EOL> n_test_batches = test_set_x . get_value ( borrow = True ) . shape [ <NUM_LIT:0> ] <EOL> n_test_batches //= batch_size <EOL> index = T . lscalar ( '<STR_LIT:index>' ) <EOL> gparams = T . grad ( self . finetune_cost , self . params ) <EOL> updates = [ <EOL> ( param , param - gparam * learning_rate ) <EOL> for param , gparam in zip ( self . params , gparams ) <EOL> ] <EOL> train_fn = theano . function ( <EOL> inputs = [ index ] , <EOL> outputs = self . finetune_cost , <EOL> updates = updates , <EOL> givens = { <EOL> self . x : train_set_x [ <EOL> index * batch_size : ( index + <NUM_LIT:1> ) * batch_size <EOL> ] , <EOL> self . y : train_set_y [ <EOL> index * batch_size : ( index + <NUM_LIT:1> ) * batch_size <EOL> ] <EOL> } , <EOL> name = '<STR_LIT:train>' <EOL> ) <EOL> test_score_i = theano . function ( <EOL> [ index ] , <EOL> self . errors , <EOL> givens = { <EOL> self . x : test_set_x [ <EOL> index * batch_size : ( index + <NUM_LIT:1> ) * batch_size <EOL> ] , <EOL> self . y : test_set_y [ <EOL> index * batch_size : ( index + <NUM_LIT:1> ) * batch_size <EOL> ] <EOL> } , <EOL> name = '<STR_LIT:test>' <EOL> ) <EOL> valid_score_i = theano . function ( <EOL> [ index ] , <EOL> self . errors , <EOL> givens = { <EOL> self . x : valid_set_x [ <EOL> index * batch_size : ( index + <NUM_LIT:1> ) * batch_size <EOL> ] , <EOL> self . y : valid_set_y [ <EOL> index * batch_size : ( index + <NUM_LIT:1> ) * batch_size <EOL> ] <EOL> } , <EOL> name = '<STR_LIT>' <EOL> ) <EOL> def valid_score ( ) : <EOL> return [ valid_score_i ( i ) for i in range ( n_valid_batches ) ] <EOL> def test_score ( ) : <EOL> return [ test_score_i ( i ) for i in range ( n_test_batches ) ] <EOL> return train_fn , valid_score , test_score <EOL> def test_SdA ( finetune_lr = <NUM_LIT:0.1> , pretraining_epochs = <NUM_LIT:15> , <EOL> pretrain_lr = <NUM_LIT> , training_epochs = <NUM_LIT:1000> , <EOL> dataset = '<STR_LIT>' , batch_size = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> datasets = load_data ( dataset ) <EOL> train_set_x , train_set_y = datasets [ <NUM_LIT:0> ] <EOL> valid_set_x , valid_set_y = datasets [ <NUM_LIT:1> ] <EOL> test_set_x , test_set_y = datasets [ <NUM_LIT:2> ] <EOL> n_train_batches = train_set_x . get_value ( borrow = True ) . shape [ <NUM_LIT:0> ] <EOL> n_train_batches //= batch_size <EOL> numpy_rng = numpy . random . RandomState ( <NUM_LIT> ) <EOL> print ( '<STR_LIT>' ) <EOL> sda = SdA ( <EOL> numpy_rng = numpy_rng , <EOL> n_ins = <NUM_LIT> * <NUM_LIT> , <EOL> hidden_layers_sizes = [ <NUM_LIT:1000> , <NUM_LIT:1000> , <NUM_LIT:1000> ] , <EOL> n_outs = <NUM_LIT:10> <EOL> ) <EOL> print ( '<STR_LIT>' ) <EOL> pretraining_fns = sda . pretraining_functions ( train_set_x = train_set_x , <EOL> batch_size = batch_size ) <EOL> print ( '<STR_LIT>' ) <EOL> start_time = timeit . default_timer ( ) <EOL> corruption_levels = [ <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] <EOL> for i in range ( sda . n_layers ) : <EOL> for epoch in range ( pretraining_epochs ) : <EOL> c = [ ] <EOL> for batch_index in range ( n_train_batches ) : <EOL> c . append ( pretraining_fns [ i ] ( index = batch_index , <EOL> corruption = corruption_levels [ i ] , <EOL> lr = pretrain_lr ) ) <EOL> print ( '<STR_LIT>' % ( i , epoch , numpy . mean ( c ) ) ) <EOL> end_time = timeit . default_timer ( ) <EOL> print ( ( '<STR_LIT>' + <EOL> os . path . split ( __file__ ) [ <NUM_LIT:1> ] + <EOL> '<STR_LIT>' % ( ( end_time - start_time ) / <NUM_LIT> ) ) , file = sys . stderr ) <EOL> print ( '<STR_LIT>' ) <EOL> train_fn , validate_model , test_model = sda . build_finetune_functions ( <EOL> datasets = datasets , <EOL> batch_size = batch_size , <EOL> learning_rate = finetune_lr <EOL> ) <EOL> print ( '<STR_LIT>' ) <EOL> patience = <NUM_LIT:10> * n_train_batches <EOL> patience_increase = <NUM_LIT> <EOL> improvement_threshold = <NUM_LIT> <EOL> validation_frequency = min ( n_train_batches , patience // <NUM_LIT:2> ) <EOL> best_validation_loss = numpy . inf <EOL> test_score = <NUM_LIT:0.> <EOL> start_time = timeit . default_timer ( ) <EOL> done_looping = False <EOL> epoch = <NUM_LIT:0> <EOL> while ( epoch < training_epochs ) and ( not done_looping ) : <EOL> epoch = epoch + <NUM_LIT:1> <EOL> for minibatch_index in range ( n_train_batches ) : <EOL> minibatch_avg_cost = train_fn ( minibatch_index ) <EOL> iter = ( epoch - <NUM_LIT:1> ) * n_train_batches + minibatch_index <EOL> if ( iter + <NUM_LIT:1> ) % validation_frequency == <NUM_LIT:0> : <EOL> validation_losses = validate_model ( ) <EOL> this_validation_loss = numpy . mean ( validation_losses ) <EOL> print ( '<STR_LIT>' % <EOL> ( epoch , minibatch_index + <NUM_LIT:1> , n_train_batches , <EOL> this_validation_loss * <NUM_LIT> ) ) <EOL> if this_validation_loss < best_validation_loss : <EOL> if ( <EOL> this_validation_loss < best_validation_loss * <EOL> improvement_threshold <EOL> ) : <EOL> patience = max ( patience , iter * patience_increase ) <EOL> best_validation_loss = this_validation_loss <EOL> best_iter = iter <EOL> test_losses = test_model ( ) <EOL> test_score = numpy . mean ( test_losses ) <EOL> print ( ( '<STR_LIT>' <EOL> '<STR_LIT>' ) % <EOL> ( epoch , minibatch_index + <NUM_LIT:1> , n_train_batches , <EOL> test_score * <NUM_LIT> ) ) <EOL> if patience <= iter : <EOL> done_looping = True <EOL> break <EOL> end_time = timeit . default_timer ( ) <EOL> print ( <EOL> ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> ) <EOL> % ( best_validation_loss * <NUM_LIT> , best_iter + <NUM_LIT:1> , test_score * <NUM_LIT> ) <EOL> ) <EOL> print ( ( '<STR_LIT>' + <EOL> os . path . split ( __file__ ) [ <NUM_LIT:1> ] + <EOL> '<STR_LIT>' % ( ( end_time - start_time ) / <NUM_LIT> ) ) , file = sys . stderr ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> test_SdA ( ) </s>
