<s> from ipdb import set_trace as pause <EOL> from matplotlib import pyplot as plt <EOL> import numpy as np <EOL> from random import shuffle <EOL> from base import create_mnist_patternsets , Minibatch2 , Network , Patternset <EOL> from plotting import plot_errors <EOL> from utils . error import mean_squared_error <EOL> from utils . utils import deriv_sigmoid , deriv_tanh , imagesc , sigmoid , sumsq , tanh , vec_to_arr <EOL> class BackpropNetwork ( Network ) : <EOL> def __init__ ( self , layersizes , lrate = <NUM_LIT> , momentum = <NUM_LIT> , <EOL> act_fn = sigmoid , deriv_act_fn = deriv_sigmoid , <EOL> err_fn = mean_squared_error , <EOL> plot = True ) : <EOL> assert len ( layersizes ) >= <NUM_LIT:3> <EOL> self . layersizes = layersizes <EOL> self . w , self . d_w = self . init_weights ( scale = <NUM_LIT:1> ) , self . init_weights ( <NUM_LIT:0> ) <EOL> self . b , self . d_b = self . init_biases ( ) , self . init_biases ( ) <EOL> self . n_l = len ( self . layersizes ) <EOL> self . momentum = momentum <EOL> self . lrate = lrate <EOL> self . act_fn = act_fn <EOL> self . deriv_act_fn = deriv_act_fn <EOL> self . err_fn = err_fn <EOL> self . plot = plot <EOL> self . fignum_errors = <NUM_LIT:1> <EOL> self . fignum_layers = <NUM_LIT:2> <EOL> self . fignum_weights_01 = <NUM_LIT:3> <EOL> self . fignum_dweights_01 = <NUM_LIT:4> <EOL> self . fignum_weights_12 = <NUM_LIT:5> <EOL> self . fignum_dweights_12 = <NUM_LIT:6> <EOL> self . fignum_biases = <NUM_LIT:7> <EOL> self . fignum_dbiases = <NUM_LIT:8> <EOL> def init_weights ( self , scale = <NUM_LIT> ) : <EOL> return [ np . random . normal ( size = ( n1 , n2 ) , scale = scale ) if scale else np . zeros ( ( n1 , n2 ) ) <EOL> for n1 , n2 in zip ( self . layersizes , self . layersizes [ <NUM_LIT:1> : ] ) ] <EOL> def init_biases ( self ) : return [ np . zeros ( ( n , ) ) for n in self . layersizes ] <EOL> def test_trial ( self , act0 , tgt ) : <EOL> inps , acts = self . propagate_fwd_all ( act0 ) <EOL> act_k = acts [ - <NUM_LIT:1> ] <EOL> err = self . err_fn ( act_k , tgt ) <EOL> return err , act_k <EOL> def propagate_fwd_all ( self , act0 ) : <EOL> inps , acts = [ act0 ] * self . n_l , [ act0 ] * self . n_l <EOL> for l in range ( self . n_l - <NUM_LIT:1> ) : <EOL> inps [ l + <NUM_LIT:1> ] , acts [ l + <NUM_LIT:1> ] = self . propagate_fwd ( l , acts [ l ] ) <EOL> return inps , acts <EOL> def propagate_fwd ( self , lowlayer_idx , act ) : <EOL> w , b = self . w [ lowlayer_idx ] , self . b [ lowlayer_idx + <NUM_LIT:1> ] <EOL> return super ( BackpropNetwork , self ) . propagate_fwd ( act , w , b ) <EOL> def learn_trial ( self , act0 , target ) : <EOL> d_w , d_b = self . delta_weights ( act0 , target ) <EOL> for cur , new , old in zip ( self . w , d_w , self . d_w ) : cur += new + self . momentum * old <EOL> for cur , new , old in zip ( self . b , d_b , self . d_b ) : cur += new + self . momentum * old <EOL> self . d_w , self . d_b = d_w , d_b <EOL> def delta_weights ( self , act0 , tgt ) : <EOL> d_w , d_b = self . init_weights ( <NUM_LIT:0> ) , self . init_biases ( ) <EOL> inps , acts = self . propagate_fwd_all ( act0 ) <EOL> j , k = self . n_l - <NUM_LIT:2> , self . n_l - <NUM_LIT:1> <EOL> sens = [ None ] * self . n_l <EOL> d_w [ j ] , d_b [ k ] , sens [ k ] = self . delta_w_jk ( tgt - acts [ k ] , inps [ k ] , acts [ j ] ) <EOL> for i in reversed ( range ( self . n_l - <NUM_LIT:2> ) ) : <EOL> j , k = i + <NUM_LIT:1> , i + <NUM_LIT:2> <EOL> d_w [ i ] , d_b [ j ] , sens [ j ] = self . delta_w_ij ( inps [ j ] , self . w [ j ] , sens [ k ] , acts [ i ] ) <EOL> return d_w , d_b <EOL> def delta_w_jk ( self , err_k , inp_k , act_j ) : <EOL> """<STR_LIT>""" <EOL> sens_k = err_k * self . deriv_act_fn ( inp_k ) <EOL> d_w_jk = self . lrate * np . dot ( act_j . T , sens_k ) <EOL> d_b_k = np . mean ( self . lrate * sens_k , axis = <NUM_LIT:0> ) <EOL> return d_w_jk , d_b_k , sens_k <EOL> def delta_w_ij ( self , inp_j , w_jk , sens_k , act_i ) : <EOL> """<STR_LIT>""" <EOL> sens_j = self . deriv_act_fn ( inp_j ) * np . dot ( w_jk , sens_k . T ) . T <EOL> d_w_ij = self . lrate * np . dot ( act_i . T , sens_j ) <EOL> d_b_j = np . mean ( self . lrate * sens_j , axis = <NUM_LIT:0> ) <EOL> return d_w_ij , d_b_j , sens_j <EOL> def plot_errors ( self , train_errors ) : <EOL> if not self . plot : return <EOL> return plot_errors ( train_errors , fignum = self . fignum_errors ) <EOL> def arr_str ( arr ) : return np . array_str ( arr , precision = <NUM_LIT:2> ) <EOL> def test_epoch ( net , acts0 , targets , epochnum = - <NUM_LIT:1> , verbose = True ) : <EOL> errors , outs = net . test_trial ( acts0 , targets ) <EOL> if verbose : <EOL> for act0 , target , out , error in zip ( acts0 , targets , outs , errors ) : <EOL> print '<STR_LIT>' % ( arr_str ( act0 ) , arr_str ( target ) , arr_str ( out ) , error ) <EOL> mean_error = np . mean ( errors ) <EOL> print '<STR_LIT>' % ( epochnum , mean_error ) <EOL> return errors , mean_error <EOL> def xor ( * args , ** kwargs ) : <EOL> data = [ [ [ <NUM_LIT:0> , <NUM_LIT:0> ] , [ <NUM_LIT:0> ] ] , <EOL> [ [ <NUM_LIT:0> , <NUM_LIT:1> ] , [ <NUM_LIT:1> ] ] , <EOL> [ [ <NUM_LIT:1> , <NUM_LIT:0> ] , [ <NUM_LIT:1> ] ] , <EOL> [ [ <NUM_LIT:1> , <NUM_LIT:1> ] , [ <NUM_LIT:0> ] ] <EOL> ] <EOL> iset = Patternset ( [ vec_to_arr ( np . array ( a ) ) for a , t in data ] ) <EOL> oset = Patternset ( [ vec_to_arr ( np . array ( t ) ) for a , t in data ] ) <EOL> net = BackpropNetwork ( [ <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:1> ] , * args , ** kwargs ) <EOL> return net , iset , oset <EOL> def autoencoder ( n_inp_out , nhiddens , * args , ** kwargs ) : <EOL> layersizes = [ n_inp_out ] + nhiddens + [ n_inp_out ] <EOL> net = BackpropNetwork ( layersizes , * args , ** kwargs ) <EOL> return net <EOL> def rand_autoencoder ( nhiddens = None , * args , ** kwargs ) : <EOL> n_inp_out = <NUM_LIT:4> <EOL> npatterns = <NUM_LIT:10> <EOL> if nhiddens is None : nhiddens = [ <NUM_LIT:12> ] <EOL> pset = Patternset ( [ np . random . uniform ( size = ( <NUM_LIT:1> , n_inp_out ) ) for d in range ( npatterns ) ] ) <EOL> return autoencoder ( n_inp_out , nhiddens = nhiddens , * args , ** kwargs ) <EOL> def mnist_autoencoder ( nhiddens = None , * args , ** kwargs ) : <EOL> if nhiddens is None : nhiddens = [ <NUM_LIT> ] <EOL> return autoencoder ( <NUM_LIT> , nhiddens = nhiddens , * args , ** kwargs ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> np . random . seed ( ) <EOL> train_pset , valid_pset , test_pset = create_mnist_patternsets ( n_trainpatterns = <NUM_LIT:1000> , ravel = True , zero_to_negone = True ) <EOL> net = mnist_autoencoder ( nhiddens = [ <NUM_LIT:200> ] , lrate = <NUM_LIT> ) <EOL> train_iset , train_oset = train_pset , train_pset <EOL> nEpochs = <NUM_LIT> <EOL> n_in_minibatch = min ( len ( train_iset ) , <NUM_LIT:20> ) <EOL> report_every = <NUM_LIT:100> <EOL> train_errors = [ ] <EOL> for e in range ( nEpochs ) : <EOL> act0 , target = Minibatch2 ( train_iset , train_oset , n_in_minibatch ) . patterns <EOL> net . learn_trial ( act0 , target ) <EOL> if not e % report_every : <EOL> error , mean_error = test_epoch ( net , train_iset . patterns , train_oset . patterns , False ) <EOL> train_errors . append ( mean_error ) <EOL> net . plot_errors ( train_errors ) <EOL> if mean_error < <NUM_LIT> : break <EOL> final_error , final_mean_error = test_epoch ( net , train_iset . patterns , train_oset . patterns , True ) </s>
