<s> """<STR_LIT>""" <EOL> import numpy as np <EOL> import warnings <EOL> from theano . compat import six <EOL> from theano import config <EOL> from theano import tensor as T <EOL> from pylearn2 . compat import OrderedDict <EOL> from pylearn2 . space import NullSpace <EOL> from pylearn2 . train_extensions import TrainExtension <EOL> from pylearn2 . utils import sharedX <EOL> from pylearn2 . utils import wraps <EOL> from pylearn2 . monitor import Monitor <EOL> class LearningRule ( ) : <EOL> """<STR_LIT>""" <EOL> def add_channels_to_monitor ( self , monitor , monitoring_dataset ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def get_updates ( self , learning_rate , grads , lr_scalers = None ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( str ( type ( self ) ) + "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> class Momentum ( LearningRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , init_momentum , nesterov_momentum = False ) : <EOL> assert init_momentum >= <NUM_LIT:0.> <EOL> assert init_momentum < <NUM_LIT:1.> <EOL> self . momentum = sharedX ( init_momentum , '<STR_LIT>' ) <EOL> self . nesterov_momentum = nesterov_momentum <EOL> def add_channels_to_monitor ( self , monitor , monitoring_dataset ) : <EOL> """<STR_LIT>""" <EOL> monitor . add_channel ( <EOL> name = '<STR_LIT>' , <EOL> ipt = None , <EOL> val = self . momentum , <EOL> data_specs = ( NullSpace ( ) , '<STR_LIT>' ) , <EOL> dataset = monitoring_dataset ) <EOL> def get_updates ( self , learning_rate , grads , lr_scalers = None ) : <EOL> """<STR_LIT>""" <EOL> updates = OrderedDict ( ) <EOL> for ( param , grad ) in six . iteritems ( grads ) : <EOL> vel = sharedX ( param . get_value ( ) * <NUM_LIT:0.> ) <EOL> assert param . dtype == vel . dtype <EOL> assert grad . dtype == param . dtype <EOL> if param . name is not None : <EOL> vel . name = '<STR_LIT>' + param . name <EOL> scaled_lr = learning_rate * lr_scalers . get ( param , <NUM_LIT:1.> ) <EOL> updates [ vel ] = self . momentum * vel - scaled_lr * grad <EOL> inc = updates [ vel ] <EOL> if self . nesterov_momentum : <EOL> inc = self . momentum * inc - scaled_lr * grad <EOL> assert inc . dtype == vel . dtype <EOL> updates [ param ] = param + inc <EOL> return updates <EOL> class MomentumAdjustor ( TrainExtension ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , final_momentum , start , saturate ) : <EOL> if saturate < start : <EOL> raise TypeError ( "<STR_LIT>" + <EOL> "<STR_LIT>" ) <EOL> self . __dict__ . update ( locals ( ) ) <EOL> del self . self <EOL> self . _initialized = False <EOL> self . _count = <NUM_LIT:0> <EOL> def setup ( self , model , dataset , algorithm ) : <EOL> """<STR_LIT>""" <EOL> monitor = Monitor . get_monitor ( model ) <EOL> self . _count = monitor . get_epochs_seen ( ) <EOL> self . _apply_momentum ( algorithm ) <EOL> def on_monitor ( self , model , dataset , algorithm ) : <EOL> """<STR_LIT>""" <EOL> self . _count += <NUM_LIT:1> <EOL> self . _apply_momentum ( algorithm ) <EOL> def _apply_momentum ( self , algorithm ) : <EOL> """<STR_LIT>""" <EOL> if not hasattr ( algorithm , '<STR_LIT>' ) : <EOL> raise ValueError ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> momentum = algorithm . learning_rule . momentum <EOL> if not self . _initialized : <EOL> self . _init_momentum = momentum . get_value ( ) <EOL> self . _initialized = True <EOL> momentum . set_value ( np . cast [ config . floatX ] ( self . current_momentum ( ) ) ) <EOL> def current_momentum ( self ) : <EOL> """<STR_LIT>""" <EOL> w = self . saturate - self . start <EOL> if w == <NUM_LIT:0> : <EOL> if self . _count >= self . start : <EOL> return self . final_momentum <EOL> return self . _init_momentum <EOL> alpha = float ( self . _count - self . start ) / float ( w ) <EOL> if alpha < <NUM_LIT:0.> : <EOL> alpha = <NUM_LIT:0.> <EOL> if alpha > <NUM_LIT:1.> : <EOL> alpha = <NUM_LIT:1.> <EOL> return self . _init_momentum * ( <NUM_LIT:1> - alpha ) + alpha * self . final_momentum <EOL> class AdaDelta ( LearningRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , decay = <NUM_LIT> ) : <EOL> assert decay >= <NUM_LIT:0.> <EOL> assert decay < <NUM_LIT:1.> <EOL> self . decay = decay <EOL> def get_updates ( self , learning_rate , grads , lr_scalers = None ) : <EOL> """<STR_LIT>""" <EOL> updates = OrderedDict ( ) <EOL> for param in grads . keys ( ) : <EOL> mean_square_grad = sharedX ( param . get_value ( ) * <NUM_LIT:0.> ) <EOL> mean_square_dx = sharedX ( param . get_value ( ) * <NUM_LIT:0.> ) <EOL> if param . name is not None : <EOL> mean_square_grad . name = '<STR_LIT>' + param . name <EOL> mean_square_dx . name = '<STR_LIT>' + param . name <EOL> new_mean_squared_grad = ( <EOL> self . decay * mean_square_grad + <EOL> ( <NUM_LIT:1> - self . decay ) * T . sqr ( grads [ param ] ) <EOL> ) <EOL> epsilon = lr_scalers . get ( param , <NUM_LIT:1.> ) * learning_rate <EOL> rms_dx_tm1 = T . sqrt ( mean_square_dx + epsilon ) <EOL> rms_grad_t = T . sqrt ( new_mean_squared_grad + epsilon ) <EOL> delta_x_t = - rms_dx_tm1 / rms_grad_t * grads [ param ] <EOL> new_mean_square_dx = ( <EOL> self . decay * mean_square_dx + <EOL> ( <NUM_LIT:1> - self . decay ) * T . sqr ( delta_x_t ) <EOL> ) <EOL> updates [ mean_square_grad ] = new_mean_squared_grad <EOL> updates [ mean_square_dx ] = new_mean_square_dx <EOL> updates [ param ] = param + delta_x_t <EOL> return updates <EOL> class AdaGrad ( LearningRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , max_scaling = <NUM_LIT> ) : <EOL> assert max_scaling > <NUM_LIT:0> <EOL> self . eps = <NUM_LIT:1.> / max_scaling <EOL> def get_updates ( self , learning_rate , grads , lr_scalers = None ) : <EOL> """<STR_LIT>""" <EOL> updates = OrderedDict ( ) <EOL> for param in grads . keys ( ) : <EOL> sum_square_grad = sharedX ( param . get_value ( ) * <NUM_LIT:0.> ) <EOL> if param . name is not None : <EOL> sum_square_grad . name = '<STR_LIT>' + param . name <EOL> new_sum_squared_grad = ( <EOL> sum_square_grad + T . sqr ( grads [ param ] ) <EOL> ) <EOL> epsilon = lr_scalers . get ( param , <NUM_LIT:1.> ) * learning_rate <EOL> scale = T . maximum ( self . eps , T . sqrt ( new_sum_squared_grad ) ) <EOL> delta_x_t = ( - epsilon / scale * grads [ param ] ) <EOL> updates [ sum_square_grad ] = new_sum_squared_grad <EOL> updates [ param ] = param + delta_x_t <EOL> return updates <EOL> class RMSProp ( LearningRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , decay = <NUM_LIT> , max_scaling = <NUM_LIT> ) : <EOL> assert <NUM_LIT:0.> <= decay < <NUM_LIT:1.> <EOL> assert max_scaling > <NUM_LIT:0> <EOL> self . decay = sharedX ( decay , '<STR_LIT>' ) <EOL> self . epsilon = <NUM_LIT:1.> / max_scaling <EOL> self . mean_square_grads = OrderedDict ( ) <EOL> @ wraps ( LearningRule . add_channels_to_monitor ) <EOL> def add_channels_to_monitor ( self , monitor , monitoring_dataset ) : <EOL> """<STR_LIT>""" <EOL> channel_mapping = { <EOL> '<STR_LIT>' : T . min , <EOL> '<STR_LIT>' : T . max , <EOL> '<STR_LIT>' : T . mean <EOL> } <EOL> for mean_square_grad in self . mean_square_grads . values ( ) : <EOL> for suffix , op in channel_mapping . items ( ) : <EOL> monitor . add_channel ( <EOL> name = ( mean_square_grad . name + suffix ) , <EOL> ipt = None , <EOL> val = op ( mean_square_grad ) , <EOL> data_specs = ( NullSpace ( ) , '<STR_LIT>' ) , <EOL> dataset = monitoring_dataset ) <EOL> return <EOL> def get_updates ( self , learning_rate , grads , lr_scalers = None ) : <EOL> """<STR_LIT>""" <EOL> updates = OrderedDict ( ) <EOL> for param in grads : <EOL> mean_square_grad = sharedX ( param . get_value ( ) * <NUM_LIT:0.> ) <EOL> if param . name is None : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> mean_square_grad . name = '<STR_LIT>' + param . name <EOL> if param . name in self . mean_square_grads : <EOL> warnings . warn ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % param . name ) <EOL> self . mean_square_grads [ param . name ] = mean_square_grad <EOL> new_mean_squared_grad = ( self . decay * mean_square_grad + <EOL> ( <NUM_LIT:1> - self . decay ) * T . sqr ( grads [ param ] ) ) <EOL> scaled_lr = lr_scalers . get ( param , <NUM_LIT:1.> ) * learning_rate <EOL> rms_grad_t = T . sqrt ( new_mean_squared_grad ) <EOL> rms_grad_t = T . maximum ( rms_grad_t , self . epsilon ) <EOL> delta_x_t = - scaled_lr * grads [ param ] / rms_grad_t <EOL> updates [ mean_square_grad ] = new_mean_squared_grad <EOL> updates [ param ] = param + delta_x_t <EOL> return updates </s>
