<s> from collections import OrderedDict <EOL> import logging <EOL> import scipy <EOL> import numpy as np <EOL> from theano import tensor <EOL> from theano . tensor . signal . downsample import max_pool_2d , DownsampleFactorMax <EOL> from blocks . extensions import SimpleExtension <EOL> from blocks . extensions . monitoring import ( DataStreamMonitoring , <EOL> MonitoringExtension ) <EOL> from blocks . filter import VariableFilter <EOL> from blocks . graph import ComputationGraph <EOL> from blocks . monitoring . evaluators import DatasetEvaluator <EOL> from blocks . roles import AuxiliaryRole <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> class BnParamRole ( AuxiliaryRole ) : <EOL> pass <EOL> BNPARAM = BnParamRole ( ) <EOL> class ZCA ( object ) : <EOL> def __init__ ( self , n_components = None , data = None , filter_bias = <NUM_LIT:0.1> ) : <EOL> self . filter_bias = np . float32 ( filter_bias ) <EOL> self . P = None <EOL> self . P_inv = None <EOL> self . n_components = <NUM_LIT:0> <EOL> self . is_fit = False <EOL> if n_components and data : <EOL> self . fit ( n_components , data ) <EOL> def fit ( self , n_components , data ) : <EOL> if len ( data . shape ) == <NUM_LIT:2> : <EOL> self . reshape = None <EOL> else : <EOL> assert n_components == np . product ( data . shape [ <NUM_LIT:1> : ] ) , '<STR_LIT>' % np . product ( data . shape [ <NUM_LIT:1> : ] ) <EOL> self . reshape = data . shape [ <NUM_LIT:1> : ] <EOL> data = self . _flatten_data ( data ) <EOL> assert len ( data . shape ) == <NUM_LIT:2> <EOL> n , m = data . shape <EOL> self . mean = np . mean ( data , axis = <NUM_LIT:0> ) <EOL> bias = self . filter_bias * scipy . sparse . identity ( m , '<STR_LIT>' ) <EOL> cov = np . cov ( data , rowvar = <NUM_LIT:0> , bias = <NUM_LIT:1> ) + bias <EOL> eigs , eigv = scipy . linalg . eigh ( cov ) <EOL> assert not np . isnan ( eigs ) . any ( ) <EOL> assert not np . isnan ( eigv ) . any ( ) <EOL> assert eigs . min ( ) > <NUM_LIT:0> <EOL> if self . n_components : <EOL> eigs = eigs [ - self . n_components : ] <EOL> eigv = eigv [ : , - self . n_components : ] <EOL> sqrt_eigs = np . sqrt ( eigs ) <EOL> self . P = np . dot ( eigv * ( <NUM_LIT:1.0> / sqrt_eigs ) , eigv . T ) <EOL> assert not np . isnan ( self . P ) . any ( ) <EOL> self . P_inv = np . dot ( eigv * sqrt_eigs , eigv . T ) <EOL> self . P = np . float32 ( self . P ) <EOL> self . P_inv = np . float32 ( self . P_inv ) <EOL> self . is_fit = True <EOL> def apply ( self , data , remove_mean = True ) : <EOL> data = self . _flatten_data ( data ) <EOL> d = data - self . mean if remove_mean else data <EOL> return self . _reshape_data ( np . dot ( d , self . P ) ) <EOL> def inv ( self , data , add_mean = True ) : <EOL> d = np . dot ( self . _flatten_data ( data ) , self . P_inv ) <EOL> d += self . mean if add_mean else <NUM_LIT:0.> <EOL> return self . _reshape_data ( d ) <EOL> def _flatten_data ( self , data ) : <EOL> if self . reshape is None : <EOL> return data <EOL> assert data . shape [ <NUM_LIT:1> : ] == self . reshape <EOL> return data . reshape ( data . shape [ <NUM_LIT:0> ] , np . product ( data . shape [ <NUM_LIT:1> : ] ) ) <EOL> def _reshape_data ( self , data ) : <EOL> assert len ( data . shape ) == <NUM_LIT:2> <EOL> if self . reshape is None : <EOL> return data <EOL> return np . reshape ( data , ( data . shape [ <NUM_LIT:0> ] , ) + self . reshape ) <EOL> class ContrastNorm ( object ) : <EOL> def __init__ ( self , scale = <NUM_LIT> , epsilon = <NUM_LIT> ) : <EOL> self . scale = np . float32 ( scale ) <EOL> self . epsilon = np . float32 ( epsilon ) <EOL> def apply ( self , data , copy = False ) : <EOL> if copy : <EOL> data = np . copy ( data ) <EOL> data_shape = data . shape <EOL> if len ( data . shape ) > <NUM_LIT:2> : <EOL> data = data . reshape ( data . shape [ <NUM_LIT:0> ] , np . product ( data . shape [ <NUM_LIT:1> : ] ) ) <EOL> assert len ( data . shape ) == <NUM_LIT:2> , '<STR_LIT>' <EOL> data -= data . mean ( axis = <NUM_LIT:1> ) [ : , np . newaxis ] <EOL> norms = np . sqrt ( np . sum ( data ** <NUM_LIT:2> , axis = <NUM_LIT:1> ) ) / self . scale <EOL> norms [ norms < self . epsilon ] = np . float32 ( <NUM_LIT:1.> ) <EOL> data /= norms [ : , np . newaxis ] <EOL> if data_shape != data . shape : <EOL> data = data . reshape ( data_shape ) <EOL> return data <EOL> class TestMonitoring ( object ) : <EOL> def _get_bn_params ( self , output_vars ) : <EOL> cg = ComputationGraph ( output_vars ) <EOL> var_filter = VariableFilter ( roles = [ BNPARAM ] ) <EOL> bn_ps = var_filter ( cg . variables ) <EOL> if len ( bn_ps ) == <NUM_LIT:0> : <EOL> logger . warn ( '<STR_LIT>' + <EOL> '<STR_LIT>' ) <EOL> self . _bn = False <EOL> self . _counter = None <EOL> self . _counter_max = None <EOL> bn_share = [ ] <EOL> output_vars_replaced = output_vars <EOL> else : <EOL> self . _bn = True <EOL> assert len ( set ( [ p . name for p in bn_ps ] ) ) == len ( bn_ps ) , '<STR_LIT>' <EOL> logger . info ( '<STR_LIT>' % '<STR_LIT:U+002CU+0020>' . join ( [ p . name for p in bn_ps ] ) ) <EOL> def filter_share ( par ) : <EOL> lst = [ up for up in cg . updates if up . name == '<STR_LIT>' % par . name ] <EOL> assert len ( lst ) == <NUM_LIT:1> <EOL> return lst [ <NUM_LIT:0> ] <EOL> bn_share = map ( filter_share , bn_ps ) <EOL> output_vars_replaced = cg . replace ( zip ( bn_ps , bn_share ) ) . outputs <EOL> self . _counter = self . _param_from_updates ( cg . updates , '<STR_LIT>' ) <EOL> self . _counter_max = self . _param_from_updates ( cg . updates , '<STR_LIT>' ) <EOL> return bn_ps , bn_share , output_vars_replaced <EOL> def _param_from_updates ( self , updates , p_name ) : <EOL> var_filter = VariableFilter ( roles = [ BNPARAM ] ) <EOL> bn_ps = var_filter ( updates . keys ( ) ) <EOL> p = [ p for p in bn_ps if p . name == p_name ] <EOL> assert len ( p ) == <NUM_LIT:1> , '<STR_LIT>' % ( p_name , p_name ) <EOL> return p [ <NUM_LIT:0> ] <EOL> def reset_counter ( self ) : <EOL> if self . _bn : <EOL> self . _counter . set_value ( np . float32 ( <NUM_LIT:1> ) ) <EOL> def replicate_vars ( self , output_vars ) : <EOL> if isinstance ( output_vars , ( list , tuple ) ) : <EOL> return map ( self . replicate_vars , output_vars ) <EOL> assert not hasattr ( output_vars . tag , '<STR_LIT>' ) , '<STR_LIT>' % output_vars . name + '<STR_LIT>' + '<STR_LIT>' <EOL> new_var = <NUM_LIT:1> * output_vars <EOL> new_var . name = output_vars . name <EOL> return new_var <EOL> class ApproxTestMonitoring ( DataStreamMonitoring , TestMonitoring ) : <EOL> def __init__ ( self , output_vars , * args , ** kwargs ) : <EOL> output_vars = self . replicate_vars ( output_vars ) <EOL> _ , _ , replaced_vars = self . _get_bn_params ( output_vars ) <EOL> super ( ApproxTestMonitoring , self ) . __init__ ( replaced_vars , * args , <EOL> ** kwargs ) <EOL> def do ( self , which_callback , * args , ** kwargs ) : <EOL> assert not which_callback == "<STR_LIT>" , "<STR_LIT>" <EOL> self . reset_counter ( ) <EOL> super ( ApproxTestMonitoring , self ) . do ( which_callback , * args , ** kwargs ) <EOL> class FinalTestMonitoring ( SimpleExtension , MonitoringExtension , TestMonitoring ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , output_vars , train_data_stream , test_data_stream , <EOL> ** kwargs ) : <EOL> output_vars = self . replicate_vars ( output_vars ) <EOL> super ( FinalTestMonitoring , self ) . __init__ ( ** kwargs ) <EOL> self . trn_stream = train_data_stream <EOL> self . tst_stream = test_data_stream <EOL> bn_ps , bn_share , output_vars_replaced = self . _get_bn_params ( output_vars ) <EOL> if self . _bn : <EOL> updates = self . _get_updates ( bn_ps , bn_share ) <EOL> trn_evaluator = DatasetEvaluator ( bn_ps , updates = updates ) <EOL> else : <EOL> trn_evaluator = None <EOL> self . _trn_evaluator = trn_evaluator <EOL> self . _tst_evaluator = DatasetEvaluator ( output_vars_replaced ) <EOL> def _get_updates ( self , bn_ps , bn_share ) : <EOL> cg = ComputationGraph ( bn_ps ) <EOL> updates = OrderedDict ( [ ( up , cg . updates [ up ] ) for up in <EOL> cg . updates if up . name == '<STR_LIT>' or <EOL> up in bn_share ] ) <EOL> assert self . _counter == self . _param_from_updates ( cg . updates , '<STR_LIT>' ) <EOL> assert self . _counter_max == self . _param_from_updates ( cg . updates , <EOL> '<STR_LIT>' ) <EOL> assert len ( updates ) == len ( bn_ps ) + <NUM_LIT:1> , '<STR_LIT>' <EOL> return updates <EOL> def do ( self , which_callback , * args ) : <EOL> """<STR_LIT>""" <EOL> assert not which_callback == "<STR_LIT>" , "<STR_LIT>" <EOL> if self . _bn : <EOL> self . _counter_max . set_value ( np . float32 ( np . inf ) ) <EOL> self . reset_counter ( ) <EOL> self . _trn_evaluator . evaluate ( self . trn_stream ) <EOL> self . reset_counter ( ) <EOL> value_dict = self . _tst_evaluator . evaluate ( self . tst_stream ) <EOL> self . add_records ( self . main_loop . log , value_dict . items ( ) ) <EOL> class LRDecay ( SimpleExtension ) : <EOL> def __init__ ( self , lr , decay_first , decay_last , ** kwargs ) : <EOL> super ( LRDecay , self ) . __init__ ( ** kwargs ) <EOL> self . iter = <NUM_LIT:0> <EOL> self . decay_first = decay_first <EOL> self . decay_last = decay_last <EOL> self . lr = lr <EOL> self . lr_init = lr . get_value ( ) <EOL> def do ( self , which_callback , * args ) : <EOL> self . iter += <NUM_LIT:1> <EOL> if self . iter > self . decay_first : <EOL> ratio = <NUM_LIT:1.0> * ( self . decay_last - self . iter ) <EOL> ratio = np . maximum ( <NUM_LIT:0> , ratio / ( self . decay_last - self . decay_first ) ) <EOL> self . lr . set_value ( np . float32 ( ratio * self . lr_init ) ) <EOL> logger . info ( "<STR_LIT>" % ( self . iter , self . lr . get_value ( ) ) ) <EOL> def global_meanpool_2d ( x , num_filters ) : <EOL> mean = tensor . mean ( x . flatten ( <NUM_LIT:3> ) , axis = <NUM_LIT:2> ) <EOL> mean = mean . dimshuffle ( <NUM_LIT:0> , <NUM_LIT:1> , '<STR_LIT:x>' , '<STR_LIT:x>' ) <EOL> return mean , ( num_filters , <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> def pool_2d ( x , mode = "<STR_LIT>" , ws = ( <NUM_LIT:2> , <NUM_LIT:2> ) , stride = ( <NUM_LIT:2> , <NUM_LIT:2> ) ) : <EOL> import theano . sandbox . cuda as cuda <EOL> assert cuda . dnn . dnn_available ( ) <EOL> return cuda . dnn . dnn_pool ( x , ws = ws , stride = stride , mode = mode ) <EOL> def maxpool_2d ( z , in_dim , poolsize , poolstride ) : <EOL> z = max_pool_2d ( z , ds = poolsize , st = poolstride ) <EOL> output_size = tuple ( DownsampleFactorMax . out_shape ( in_dim , poolsize , <EOL> st = poolstride ) ) <EOL> return z , output_size </s>
