<s> """<STR_LIT>""" <EOL> from __future__ import division <EOL> __authors__ = "<STR_LIT>" <EOL> __copyright__ = "<STR_LIT>" <EOL> __credits__ = [ "<STR_LIT>" ] <EOL> __license__ = "<STR_LIT>" <EOL> __maintainer__ = "<STR_LIT>" <EOL> __email__ = "<STR_LIT>" <EOL> import logging <EOL> import warnings <EOL> import numpy as np <EOL> from theano import config <EOL> from theano import function <EOL> from theano . compat . python2x import OrderedDict <EOL> from theano . gof . op import get_debug_values <EOL> from pylearn2 . monitor import Monitor <EOL> from pylearn2 . space import CompositeSpace , NullSpace <EOL> from pylearn2 . train_extensions import TrainExtension <EOL> from pylearn2 . training_algorithms . training_algorithm import TrainingAlgorithm <EOL> from pylearn2 . training_algorithms . learning_rule import Momentum <EOL> from pylearn2 . training_algorithms . learning_rule import MomentumAdjustor as LRMomentumAdjustor <EOL> from pylearn2 . utils . iteration import is_stochastic , has_uniform_batch_size <EOL> from pylearn2 . utils import py_integer_types , py_float_types <EOL> from pylearn2 . utils import safe_zip <EOL> from pylearn2 . utils import serial <EOL> from pylearn2 . utils import sharedX <EOL> from pylearn2 . utils . data_specs import DataSpecsMapping <EOL> from pylearn2 . utils . timing import log_timing <EOL> from pylearn2 . utils . rng import make_np_rng <EOL> log = logging . getLogger ( __name__ ) <EOL> class SGD ( TrainingAlgorithm ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , learning_rate , cost = None , batch_size = None , <EOL> monitoring_batch_size = None , monitoring_batches = None , <EOL> monitoring_dataset = None , monitor_iteration_mode = '<STR_LIT>' , <EOL> termination_criterion = None , update_callbacks = None , <EOL> learning_rule = None , init_momentum = None , <EOL> set_batch_size = False , <EOL> train_iteration_mode = None , batches_per_iter = None , <EOL> theano_function_mode = None , monitoring_costs = None , <EOL> seed = [ <NUM_LIT> , <NUM_LIT:10> , <NUM_LIT:5> ] , discriminator_steps = <NUM_LIT:1> ) : <EOL> self . discriminator_steps = discriminator_steps <EOL> if isinstance ( cost , ( list , tuple , set ) ) : <EOL> raise TypeError ( "<STR_LIT>" + <EOL> "<STR_LIT>" + <EOL> "<STR_LIT>" ) <EOL> if init_momentum : <EOL> warnings . warn ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> self . learning_rule = Momentum ( init_momentum ) <EOL> else : <EOL> self . learning_rule = learning_rule <EOL> self . learning_rate = sharedX ( learning_rate , '<STR_LIT>' ) <EOL> self . cost = cost <EOL> self . batch_size = batch_size <EOL> self . set_batch_size = set_batch_size <EOL> self . batches_per_iter = batches_per_iter <EOL> self . _set_monitoring_dataset ( monitoring_dataset ) <EOL> self . monitoring_batch_size = monitoring_batch_size <EOL> self . monitoring_batches = monitoring_batches <EOL> self . monitor_iteration_mode = monitor_iteration_mode <EOL> if monitoring_dataset is None : <EOL> if monitoring_batch_size is not None : <EOL> raise ValueError ( "<STR_LIT>" + <EOL> "<STR_LIT>" ) <EOL> if monitoring_batches is not None : <EOL> raise ValueError ( "<STR_LIT>" + <EOL> "<STR_LIT>" ) <EOL> self . termination_criterion = termination_criterion <EOL> self . _register_update_callbacks ( update_callbacks ) <EOL> if train_iteration_mode is None : <EOL> train_iteration_mode = '<STR_LIT>' <EOL> self . train_iteration_mode = train_iteration_mode <EOL> self . first = True <EOL> self . rng = make_np_rng ( seed , which_method = [ "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> self . theano_function_mode = theano_function_mode <EOL> self . monitoring_costs = monitoring_costs <EOL> def setup ( self , model , dataset ) : <EOL> """<STR_LIT>""" <EOL> self . i = <NUM_LIT:0> <EOL> if self . cost is None : <EOL> self . cost = model . get_default_cost ( ) <EOL> inf_params = [ param for param in model . get_params ( ) <EOL> if np . any ( np . isinf ( param . get_value ( ) ) ) ] <EOL> if len ( inf_params ) > <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" + str ( inf_params ) ) <EOL> if any ( [ np . any ( np . isnan ( param . get_value ( ) ) ) <EOL> for param in model . get_params ( ) ] ) : <EOL> nan_params = [ param for param in model . get_params ( ) <EOL> if np . any ( np . isnan ( param . get_value ( ) ) ) ] <EOL> raise ValueError ( "<STR_LIT>" + str ( nan_params ) ) <EOL> self . model = model <EOL> self . _synchronize_batch_size ( model ) <EOL> model . _test_batch_size = self . batch_size <EOL> self . monitor = Monitor . get_monitor ( model ) <EOL> self . monitor . _sanity_check ( ) <EOL> if getattr ( model , "<STR_LIT>" , False ) and any ( dataset . get_design_matrix ( ) . shape [ <NUM_LIT:0> ] % self . batch_size != <NUM_LIT:0> for <EOL> dataset in self . monitoring_dataset . values ( ) ) and not has_uniform_batch_size ( self . monitor_iteration_mode ) : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> data_specs = self . cost . get_data_specs ( self . model ) <EOL> mapping = DataSpecsMapping ( data_specs ) <EOL> space_tuple = mapping . flatten ( data_specs [ <NUM_LIT:0> ] , return_tuple = True ) <EOL> source_tuple = mapping . flatten ( data_specs [ <NUM_LIT:1> ] , return_tuple = True ) <EOL> theano_args = [ ] <EOL> for space , source in safe_zip ( space_tuple , source_tuple ) : <EOL> name = '<STR_LIT>' % ( self . __class__ . __name__ , source ) <EOL> arg = space . make_theano_batch ( name = name , <EOL> batch_size = self . batch_size ) <EOL> theano_args . append ( arg ) <EOL> theano_args = tuple ( theano_args ) <EOL> nested_args = mapping . nest ( theano_args ) <EOL> fixed_var_descr = self . cost . get_fixed_var_descr ( model , nested_args ) <EOL> self . on_load_batch = fixed_var_descr . on_load_batch <EOL> cost_value = self . cost . expr ( model , nested_args , <EOL> ** fixed_var_descr . fixed_vars ) <EOL> if cost_value is not None and cost_value . name is None : <EOL> cost_value . name = '<STR_LIT>' <EOL> learning_rate = self . learning_rate <EOL> if self . monitoring_dataset is not None : <EOL> if ( self . monitoring_batch_size is None and <EOL> self . monitoring_batches is None ) : <EOL> self . monitoring_batch_size = self . batch_size <EOL> self . monitoring_batches = self . batches_per_iter <EOL> self . monitor . setup ( dataset = self . monitoring_dataset , <EOL> cost = self . cost , <EOL> batch_size = self . monitoring_batch_size , <EOL> num_batches = self . monitoring_batches , <EOL> extra_costs = self . monitoring_costs , <EOL> mode = self . monitor_iteration_mode ) <EOL> dataset_name = self . monitoring_dataset . keys ( ) [ <NUM_LIT:0> ] <EOL> monitoring_dataset = self . monitoring_dataset [ dataset_name ] <EOL> self . monitor . add_channel ( name = '<STR_LIT>' , <EOL> ipt = None , <EOL> val = learning_rate , <EOL> data_specs = ( NullSpace ( ) , '<STR_LIT>' ) , <EOL> dataset = monitoring_dataset ) <EOL> if self . learning_rule : <EOL> self . learning_rule . add_channels_to_monitor ( <EOL> self . monitor , <EOL> monitoring_dataset ) <EOL> params = list ( model . get_params ( ) ) <EOL> assert len ( params ) > <NUM_LIT:0> <EOL> for i , param in enumerate ( params ) : <EOL> if param . name is None : <EOL> param . name = '<STR_LIT>' % i <EOL> self . params = params <EOL> grads , updates = self . cost . get_gradients ( model , nested_args , <EOL> ** fixed_var_descr . fixed_vars ) <EOL> if not isinstance ( grads , OrderedDict ) : <EOL> raise TypeError ( str ( type ( self . cost ) ) + "<STR_LIT>" + <EOL> "<STR_LIT>" + str ( type ( grads ) ) + "<STR_LIT>" + <EOL> "<STR_LIT>" ) <EOL> for param in grads : <EOL> assert param in params <EOL> for param in params : <EOL> assert param in grads <EOL> lr_scalers = model . get_lr_scalers ( ) <EOL> for key in lr_scalers : <EOL> if key not in params : <EOL> raise ValueError ( "<STR_LIT>" + str ( key ) + "<STR_LIT>" ) <EOL> assert len ( updates . keys ( ) ) == <NUM_LIT:0> <EOL> def get_func ( learn_discriminator , learn_generator ) : <EOL> updates = OrderedDict ( ) <EOL> assert ( learn_discriminator or learn_generator ) and not ( learn_discriminator and learn_generator ) <EOL> if learn_discriminator : <EOL> cur_params = model . discriminator . get_params ( ) <EOL> else : <EOL> cur_params = model . generator . get_params ( ) <EOL> cur_grads = OrderedDict ( ) <EOL> for param in cur_params : <EOL> cur_grads [ param ] = grads [ param ] <EOL> for param in grads : <EOL> if grads [ param ] . name is None and cost_value is not None : <EOL> grads [ param ] . name = ( '<STR_LIT>' % <EOL> { '<STR_LIT>' : cost_value . name , <EOL> '<STR_LIT>' : param . name } ) <EOL> assert grads [ param ] . dtype == param . dtype <EOL> cur_lr_scalers = OrderedDict ( ) <EOL> for param in cur_params : <EOL> if param in lr_scalers : <EOL> lr_scaler = lr_scalers [ param ] <EOL> cur_lr_scalers [ param ] = lr_scaler <EOL> log . info ( '<STR_LIT>' ) <EOL> for param in cur_params : <EOL> param_name = param . name <EOL> if param_name is None : <EOL> param_name = '<STR_LIT>' <EOL> lr = learning_rate . get_value ( ) * cur_lr_scalers . get ( param , <NUM_LIT:1.> ) <EOL> log . info ( '<STR_LIT:\t>' + param_name + '<STR_LIT>' + str ( lr ) ) <EOL> if self . learning_rule : <EOL> updates . update ( self . learning_rule . get_updates ( <EOL> learning_rate , cur_grads , cur_lr_scalers ) ) <EOL> else : <EOL> updates . update ( dict ( safe_zip ( params , [ param - learning_rate * lr_scalers . get ( param , <NUM_LIT:1.> ) * grads [ param ] <EOL> for param in params ] ) ) ) <EOL> for param in cur_params : <EOL> if updates [ param ] . name is None : <EOL> updates [ param ] . name = '<STR_LIT>' + param . name + '<STR_LIT:)>' <EOL> model . modify_updates ( updates ) <EOL> for param in cur_params : <EOL> update = updates [ param ] <EOL> if update . name is None : <EOL> update . name = '<STR_LIT>' + param . name + '<STR_LIT>' <EOL> for update_val in get_debug_values ( update ) : <EOL> if np . any ( np . isinf ( update_val ) ) : <EOL> raise ValueError ( "<STR_LIT>" % <EOL> update . name ) <EOL> if np . any ( np . isnan ( update_val ) ) : <EOL> raise ValueError ( "<STR_LIT>" % <EOL> update . name ) <EOL> with log_timing ( log , '<STR_LIT>' ) : <EOL> return function ( theano_args , <EOL> updates = updates , <EOL> name = '<STR_LIT>' , <EOL> on_unused_input = '<STR_LIT:ignore>' , <EOL> mode = self . theano_function_mode ) <EOL> self . d_func = get_func ( <NUM_LIT:1> , <NUM_LIT:0> ) <EOL> self . g_func = get_func ( <NUM_LIT:0> , <NUM_LIT:1> ) <EOL> def train ( self , dataset ) : <EOL> """<STR_LIT>""" <EOL> if not hasattr ( self , '<STR_LIT>' ) : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> for param in self . params : <EOL> value = param . get_value ( borrow = True ) <EOL> if np . any ( np . isnan ( value ) ) or np . any ( np . isinf ( value ) ) : <EOL> raise Exception ( "<STR_LIT>" + param . name ) <EOL> self . first = False <EOL> rng = self . rng <EOL> if not is_stochastic ( self . train_iteration_mode ) : <EOL> rng = None <EOL> data_specs = self . cost . get_data_specs ( self . model ) <EOL> mapping = DataSpecsMapping ( data_specs ) <EOL> space_tuple = mapping . flatten ( data_specs [ <NUM_LIT:0> ] , return_tuple = True ) <EOL> source_tuple = mapping . flatten ( data_specs [ <NUM_LIT:1> ] , return_tuple = True ) <EOL> if len ( space_tuple ) == <NUM_LIT:0> : <EOL> raise NotImplementedError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % str ( data_specs ) ) <EOL> flat_data_specs = ( CompositeSpace ( space_tuple ) , source_tuple ) <EOL> iterator = dataset . iterator ( mode = self . train_iteration_mode , <EOL> batch_size = self . batch_size , <EOL> data_specs = flat_data_specs , return_tuple = True , <EOL> rng = rng , num_batches = self . batches_per_iter ) <EOL> on_load_batch = self . on_load_batch <EOL> i = self . i <EOL> for batch in iterator : <EOL> for callback in on_load_batch : <EOL> callback ( * batch ) <EOL> self . d_func ( * batch ) <EOL> i += <NUM_LIT:1> <EOL> if i == self . discriminator_steps : <EOL> self . g_func ( * batch ) <EOL> i = <NUM_LIT:0> <EOL> actual_batch_size = flat_data_specs [ <NUM_LIT:0> ] . np_batch_size ( batch ) <EOL> self . monitor . report_batch ( actual_batch_size ) <EOL> for callback in self . update_callbacks : <EOL> callback ( self ) <EOL> for param in self . params : <EOL> value = param . get_value ( borrow = True ) <EOL> if np . any ( np . isnan ( value ) ) or np . any ( np . isinf ( value ) ) : <EOL> raise Exception ( "<STR_LIT>" + param . name ) <EOL> self . i = i <EOL> def continue_learning ( self , model ) : <EOL> """<STR_LIT>""" <EOL> if self . termination_criterion is None : <EOL> return True <EOL> else : <EOL> return self . termination_criterion . continue_learning ( self . model ) <EOL> class MonitorBasedLRAdjuster ( TrainExtension ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , high_trigger = <NUM_LIT:1.> , shrink_amt = <NUM_LIT> , <EOL> low_trigger = <NUM_LIT> , grow_amt = <NUM_LIT> , <EOL> min_lr = <NUM_LIT> , max_lr = <NUM_LIT:1.> , <EOL> dataset_name = None , channel_name = None ) : <EOL> self . high_trigger = high_trigger <EOL> self . shrink_amt = shrink_amt <EOL> self . low_trigger = low_trigger <EOL> self . grow_amt = grow_amt <EOL> self . min_lr = min_lr <EOL> self . max_lr = max_lr <EOL> self . dataset_name = None <EOL> if channel_name is not None : <EOL> self . channel_name = channel_name <EOL> else : <EOL> if dataset_name is not None : <EOL> self . channel_name = dataset_name + '<STR_LIT>' <EOL> self . dataset_name = dataset_name <EOL> else : <EOL> self . channel_name = None <EOL> def on_monitor ( self , model , dataset , algorithm ) : <EOL> """<STR_LIT>""" <EOL> model = algorithm . model <EOL> lr = algorithm . learning_rate <EOL> current_learning_rate = lr . get_value ( ) <EOL> assert hasattr ( model , '<STR_LIT>' ) , ( "<STR_LIT>" <EOL> + str ( model ) ) <EOL> monitor = model . monitor <EOL> monitor_channel_specified = True <EOL> if self . channel_name is None : <EOL> monitor_channel_specified = False <EOL> channels = [ elem for elem in monitor . channels <EOL> if elem . endswith ( "<STR_LIT>" ) ] <EOL> if len ( channels ) < <NUM_LIT:1> : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> elif len ( channels ) > <NUM_LIT:1> : <EOL> datasets = algorithm . monitoring_dataset . keys ( ) <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" + <EOL> str ( datasets ) + "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> '<STR_LIT>' ) <EOL> else : <EOL> self . channel_name = channels [ <NUM_LIT:0> ] <EOL> warnings . warn ( '<STR_LIT>' <EOL> '<STR_LIT>' + <EOL> str ( self . channel_name ) + '<STR_LIT:.>' ) <EOL> try : <EOL> v = monitor . channels [ self . channel_name ] . val_record <EOL> except KeyError : <EOL> err_input = '<STR_LIT>' <EOL> if monitor_channel_specified : <EOL> if self . dataset_name : <EOL> err_input = '<STR_LIT>' + str ( <EOL> self . dataset_name ) + '<STR_LIT>' <EOL> else : <EOL> err_input = '<STR_LIT>' + str ( <EOL> self . channel_name ) + '<STR_LIT>' <EOL> err_message = '<STR_LIT>' + str ( self . channel_name ) + '<STR_LIT>' + '<STR_LIT>' + '<STR_LIT>' + '<STR_LIT>' + err_input <EOL> raise ValueError ( err_message ) <EOL> if len ( v ) < <NUM_LIT:1> : <EOL> if monitor . dataset is None : <EOL> assert len ( v ) == <NUM_LIT:0> <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if len ( v ) == <NUM_LIT:1> : <EOL> return <EOL> rval = current_learning_rate <EOL> log . info ( "<STR_LIT>" . format ( self . channel_name ) ) <EOL> if v [ - <NUM_LIT:1> ] > self . high_trigger * v [ - <NUM_LIT:2> ] : <EOL> rval *= self . shrink_amt <EOL> log . info ( "<STR_LIT>" % rval ) <EOL> elif v [ - <NUM_LIT:1> ] > self . low_trigger * v [ - <NUM_LIT:2> ] : <EOL> rval *= self . grow_amt <EOL> log . info ( "<STR_LIT>" % rval ) <EOL> rval = max ( self . min_lr , rval ) <EOL> rval = min ( self . max_lr , rval ) <EOL> lr . set_value ( np . cast [ lr . dtype ] ( rval ) ) <EOL> class PatienceBasedTermCrit ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , prop_decrease , initial_patience , <EOL> patience_increase = <NUM_LIT> , channel_name = None ) : <EOL> self . _channel_name = channel_name <EOL> self . prop_decrease = prop_decrease <EOL> self . patience = initial_patience <EOL> self . best_value = np . inf <EOL> self . patience_increase = patience_increase <EOL> def __call__ ( self , model ) : <EOL> """<STR_LIT>""" <EOL> monitor = model . monitor <EOL> if self . _channel_name is None : <EOL> if len ( monitor . channels ) != <NUM_LIT:1> : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> v = monitor . channels . values ( ) [ <NUM_LIT:0> ] . val_record <EOL> else : <EOL> v = monitor . channels [ self . _channel_name ] . val_record <EOL> if v [ - <NUM_LIT:1> ] < self . best_value * ( <NUM_LIT:1.> - self . prop_decrease ) : <EOL> self . patience = max ( self . patience , len ( v ) * self . patience_increase ) <EOL> self . best_value = v [ - <NUM_LIT:1> ] <EOL> return len ( v ) < self . patience <EOL> class AnnealedLearningRate ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , anneal_start ) : <EOL> self . _initialized = False <EOL> self . _count = <NUM_LIT:0> <EOL> self . _anneal_start = anneal_start <EOL> def __call__ ( self , algorithm ) : <EOL> """<STR_LIT>""" <EOL> if not self . _initialized : <EOL> self . _base = algorithm . learning_rate . get_value ( ) <EOL> self . _count += <NUM_LIT:1> <EOL> algorithm . learning_rate . set_value ( self . current_learning_rate ( ) ) <EOL> def current_learning_rate ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _base * min ( <NUM_LIT:1> , self . _anneal_start / self . _count ) <EOL> class ExponentialDecay ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , decay_factor , min_lr ) : <EOL> if isinstance ( decay_factor , str ) : <EOL> decay_factor = float ( decay_factor ) <EOL> if isinstance ( min_lr , str ) : <EOL> min_lr = float ( min_lr ) <EOL> assert isinstance ( decay_factor , float ) <EOL> assert isinstance ( min_lr , float ) <EOL> self . __dict__ . update ( locals ( ) ) <EOL> del self . self <EOL> self . _count = <NUM_LIT:0> <EOL> self . _min_reached = False <EOL> def __call__ ( self , algorithm ) : <EOL> """<STR_LIT>""" <EOL> if self . _count == <NUM_LIT:0> : <EOL> self . _base_lr = algorithm . learning_rate . get_value ( ) <EOL> self . _count += <NUM_LIT:1> <EOL> if not self . _min_reached : <EOL> new_lr = self . _base_lr / ( self . decay_factor ** self . _count ) <EOL> if new_lr <= self . min_lr : <EOL> self . _min_reached = True <EOL> new_lr = self . min_lr <EOL> else : <EOL> new_lr = self . min_lr <EOL> new_lr = np . cast [ config . floatX ] ( new_lr ) <EOL> algorithm . learning_rate . set_value ( new_lr ) <EOL> class LinearDecay ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , start , saturate , decay_factor ) : <EOL> if isinstance ( decay_factor , str ) : <EOL> decay_factor = float ( decay_factor ) <EOL> if isinstance ( start , str ) : <EOL> start = float ( start ) <EOL> if isinstance ( saturate , str ) : <EOL> saturate = float ( saturate ) <EOL> assert isinstance ( decay_factor , float ) <EOL> assert isinstance ( start , ( py_integer_types , py_float_types ) ) <EOL> assert isinstance ( saturate , ( py_integer_types , py_float_types ) ) <EOL> assert saturate > start <EOL> assert start > <NUM_LIT:0> <EOL> self . __dict__ . update ( locals ( ) ) <EOL> del self . self <EOL> self . _count = <NUM_LIT:0> <EOL> def __call__ ( self , algorithm ) : <EOL> """<STR_LIT>""" <EOL> if self . _count == <NUM_LIT:0> : <EOL> self . _base_lr = algorithm . learning_rate . get_value ( ) <EOL> self . _step = ( ( self . _base_lr - self . _base_lr * self . decay_factor ) / <EOL> ( self . saturate - self . start + <NUM_LIT:1> ) ) <EOL> self . _count += <NUM_LIT:1> <EOL> if self . _count >= self . start : <EOL> if self . _count < self . saturate : <EOL> new_lr = self . _base_lr - self . _step * ( self . _count <EOL> - self . start + <NUM_LIT:1> ) <EOL> else : <EOL> new_lr = self . _base_lr * self . decay_factor <EOL> else : <EOL> new_lr = self . _base_lr <EOL> assert new_lr > <NUM_LIT:0> <EOL> new_lr = np . cast [ config . floatX ] ( new_lr ) <EOL> algorithm . learning_rate . set_value ( new_lr ) <EOL> def MomentumAdjustor ( final_momentum , start , saturate ) : <EOL> """<STR_LIT>""" <EOL> warnings . warn ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> return LRMomentumAdjustor ( final_momentum , start , saturate ) <EOL> class OneOverEpoch ( TrainExtension ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , start , half_life = None , min_lr = <NUM_LIT> ) : <EOL> self . __dict__ . update ( locals ( ) ) <EOL> del self . self <EOL> self . _initialized = False <EOL> self . _count = <NUM_LIT:0> <EOL> assert start >= <NUM_LIT:0> <EOL> if half_life is None : <EOL> self . half_life = start + <NUM_LIT:1> <EOL> else : <EOL> assert half_life > <NUM_LIT:0> <EOL> def on_monitor ( self , model , dataset , algorithm ) : <EOL> """<STR_LIT>""" <EOL> if not self . _initialized : <EOL> self . _init_lr = algorithm . learning_rate . get_value ( ) <EOL> if self . _init_lr < self . min_lr : <EOL> raise ValueError ( "<STR_LIT>" + <EOL> "<STR_LIT>" ) <EOL> self . _initialized = True <EOL> self . _count += <NUM_LIT:1> <EOL> algorithm . learning_rate . set_value ( np . cast [ config . floatX ] ( <EOL> self . current_lr ( ) ) ) <EOL> def current_lr ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _count < self . start : <EOL> scale = <NUM_LIT:1> <EOL> else : <EOL> scale = float ( self . half_life ) / float ( self . _count - self . start <EOL> + self . half_life ) <EOL> lr = self . _init_lr * scale <EOL> clipped = max ( self . min_lr , lr ) <EOL> return clipped <EOL> class LinearDecayOverEpoch ( TrainExtension ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , start , saturate , decay_factor ) : <EOL> self . __dict__ . update ( locals ( ) ) <EOL> del self . self <EOL> self . _initialized = False <EOL> self . _count = <NUM_LIT:0> <EOL> assert isinstance ( decay_factor , float ) <EOL> assert isinstance ( start , ( py_integer_types , py_float_types ) ) <EOL> assert isinstance ( saturate , ( py_integer_types , py_float_types ) ) <EOL> assert saturate > start <EOL> assert start >= <NUM_LIT:0> <EOL> assert saturate >= start <EOL> def on_monitor ( self , model , dataset , algorithm ) : <EOL> """<STR_LIT>""" <EOL> if not self . _initialized : <EOL> self . _init_lr = algorithm . learning_rate . get_value ( ) <EOL> self . _step = ( ( self . _init_lr - self . _init_lr * self . decay_factor ) / <EOL> ( self . saturate - self . start + <NUM_LIT:1> ) ) <EOL> self . _initialized = True <EOL> self . _count += <NUM_LIT:1> <EOL> algorithm . learning_rate . set_value ( np . cast [ config . floatX ] ( <EOL> self . current_lr ( ) ) ) <EOL> def current_lr ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _count >= self . start : <EOL> if self . _count < self . saturate : <EOL> new_lr = self . _init_lr - self . _step * ( self . _count <EOL> - self . start + <NUM_LIT:1> ) <EOL> else : <EOL> new_lr = self . _init_lr * self . decay_factor <EOL> else : <EOL> new_lr = self . _init_lr <EOL> assert new_lr > <NUM_LIT:0> <EOL> return new_lr <EOL> class _PolyakWorker ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , model ) : <EOL> avg_updates = OrderedDict ( ) <EOL> t = sharedX ( <NUM_LIT:1.> ) <EOL> self . param_to_mean = OrderedDict ( ) <EOL> for param in model . get_params ( ) : <EOL> mean = sharedX ( param . get_value ( ) ) <EOL> assert type ( mean ) == type ( param ) <EOL> self . param_to_mean [ param ] = mean <EOL> avg_updates [ mean ] = mean - ( mean - param ) / t <EOL> avg_updates [ t ] = t + <NUM_LIT:1.> <EOL> self . avg = function ( [ ] , updates = avg_updates ) <EOL> def __call__ ( self , algorithm ) : <EOL> """<STR_LIT>""" <EOL> self . avg ( ) <EOL> class PolyakAveraging ( TrainExtension ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , start , save_path = None , save_freq = <NUM_LIT:1> ) : <EOL> self . __dict__ . update ( locals ( ) ) <EOL> del self . self <EOL> self . _count = <NUM_LIT:0> <EOL> assert isinstance ( start , py_integer_types ) <EOL> assert start >= <NUM_LIT:0> <EOL> def on_monitor ( self , model , dataset , algorithm ) : <EOL> """<STR_LIT>""" <EOL> if self . _count == self . start : <EOL> self . _worker = _PolyakWorker ( model ) <EOL> algorithm . update_callbacks . append ( self . _worker ) <EOL> try : <EOL> model . add_polyak_channels ( self . _worker . param_to_mean , <EOL> algorithm . monitoring_dataset ) <EOL> except AttributeError : <EOL> pass <EOL> elif self . save_path is not None and self . _count > self . start and self . _count % self . save_freq == <NUM_LIT:0> : <EOL> saved_params = OrderedDict ( ) <EOL> for param in model . get_params ( ) : <EOL> saved_params [ param ] = param . get_value ( ) <EOL> param . set_value ( self . _worker . param_to_mean [ param ] . get_value ( ) ) <EOL> serial . save ( self . save_path , model ) <EOL> for param in model . get_params ( ) : <EOL> param . set_value ( saved_params [ param ] ) <EOL> self . _count += <NUM_LIT:1> </s>
