<s> """<STR_LIT>""" <EOL> __authors__ = "<STR_LIT>" <EOL> __copyright__ = "<STR_LIT>" <EOL> __credits__ = [ "<STR_LIT>" ] <EOL> __license__ = "<STR_LIT>" <EOL> __maintainer__ = "<STR_LIT>" <EOL> __email__ = "<STR_LIT>" <EOL> import logging <EOL> import warnings <EOL> import numpy as np <EOL> from theano import config <EOL> from pylearn2 . compat import OrderedDict , first_value <EOL> from pylearn2 . monitor import Monitor <EOL> from pylearn2 . optimization . batch_gradient_descent import BatchGradientDescent <EOL> from pylearn2 . utils . iteration import is_stochastic <EOL> from pylearn2 . training_algorithms . training_algorithm import TrainingAlgorithm <EOL> from pylearn2 . utils import safe_zip <EOL> from pylearn2 . train_extensions import TrainExtension <EOL> from pylearn2 . termination_criteria import TerminationCriterion <EOL> from pylearn2 . utils import sharedX <EOL> from pylearn2 . space import CompositeSpace , NullSpace <EOL> from pylearn2 . utils . data_specs import DataSpecsMapping <EOL> from pylearn2 . utils . rng import make_np_rng <EOL> logger = logging . getLogger ( __name__ ) <EOL> class BGD ( TrainingAlgorithm ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , cost = None , batch_size = None , batches_per_iter = None , <EOL> updates_per_batch = <NUM_LIT:10> , monitoring_batch_size = None , <EOL> monitoring_batches = None , monitoring_dataset = None , <EOL> termination_criterion = None , set_batch_size = False , <EOL> reset_alpha = True , conjugate = False , min_init_alpha = <NUM_LIT> , <EOL> reset_conjugate = True , line_search_mode = None , <EOL> verbose_optimization = False , scale_step = <NUM_LIT:1.> , <EOL> theano_function_mode = None , init_alpha = None , seed = None ) : <EOL> self . __dict__ . update ( locals ( ) ) <EOL> del self . self <EOL> if monitoring_dataset is None : <EOL> assert monitoring_batches is None <EOL> assert monitoring_batch_size is None <EOL> self . _set_monitoring_dataset ( monitoring_dataset ) <EOL> self . bSetup = False <EOL> self . termination_criterion = termination_criterion <EOL> self . rng = make_np_rng ( seed , [ <NUM_LIT> , <NUM_LIT:10> , <NUM_LIT:16> ] , <EOL> which_method = [ "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> def setup ( self , model , dataset ) : <EOL> """<STR_LIT>""" <EOL> self . model = model <EOL> if self . cost is None : <EOL> self . cost = model . get_default_cost ( ) <EOL> try : <EOL> if self . cost . is_stochastic ( ) : <EOL> raise TypeError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> except NotImplementedError : <EOL> warnings . warn ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if self . batch_size is None : <EOL> self . batch_size = model . force_batch_size <EOL> else : <EOL> batch_size = self . batch_size <EOL> if self . set_batch_size : <EOL> model . set_batch_size ( batch_size ) <EOL> elif hasattr ( model , '<STR_LIT>' ) : <EOL> if not ( model . force_batch_size is None or <EOL> model . force_batch_size <= <NUM_LIT:0> or <EOL> batch_size == model . force_batch_size ) : <EOL> raise ValueError ( "<STR_LIT>" + <EOL> "<STR_LIT>" % <EOL> ( batch_size , model . force_batch_size ) ) <EOL> self . monitor = Monitor . get_monitor ( model ) <EOL> self . monitor . set_theano_function_mode ( self . theano_function_mode ) <EOL> data_specs = self . cost . get_data_specs ( model ) <EOL> mapping = DataSpecsMapping ( data_specs ) <EOL> space_tuple = mapping . flatten ( data_specs [ <NUM_LIT:0> ] , return_tuple = True ) <EOL> source_tuple = mapping . flatten ( data_specs [ <NUM_LIT:1> ] , return_tuple = True ) <EOL> theano_args = [ ] <EOL> for space , source in safe_zip ( space_tuple , source_tuple ) : <EOL> name = '<STR_LIT>' % source <EOL> arg = space . make_theano_batch ( name = name ) <EOL> theano_args . append ( arg ) <EOL> theano_args = tuple ( theano_args ) <EOL> nested_args = mapping . nest ( theano_args ) <EOL> fixed_var_descr = self . cost . get_fixed_var_descr ( model , nested_args ) <EOL> self . on_load_batch = fixed_var_descr . on_load_batch <EOL> cost_value = self . cost . expr ( model , nested_args , <EOL> ** fixed_var_descr . fixed_vars ) <EOL> grads , grad_updates = self . cost . get_gradients ( <EOL> model , nested_args , ** fixed_var_descr . fixed_vars ) <EOL> assert isinstance ( grads , OrderedDict ) <EOL> assert isinstance ( grad_updates , OrderedDict ) <EOL> if cost_value is None : <EOL> raise ValueError ( "<STR_LIT>" + str ( self . cost ) + <EOL> "<STR_LIT>" + <EOL> "<STR_LIT>" ) <EOL> def capture ( f , mapping = mapping ) : <EOL> new_f = lambda * args : f ( mapping . flatten ( args , return_tuple = True ) ) <EOL> return new_f <EOL> obj_prereqs = [ capture ( f ) for f in fixed_var_descr . on_load_batch ] <EOL> if self . monitoring_dataset is not None : <EOL> if ( self . monitoring_batch_size is None and <EOL> self . monitoring_batches is None ) : <EOL> self . monitoring_batch_size = self . batch_size <EOL> self . monitoring_batches = self . batches_per_iter <EOL> self . monitor . setup ( <EOL> dataset = self . monitoring_dataset , <EOL> cost = self . cost , <EOL> batch_size = self . monitoring_batch_size , <EOL> num_batches = self . monitoring_batches , <EOL> obj_prereqs = obj_prereqs , <EOL> cost_monitoring_args = fixed_var_descr . fixed_vars ) <EOL> params = model . get_params ( ) <EOL> self . optimizer = BatchGradientDescent ( <EOL> objective = cost_value , <EOL> gradients = grads , <EOL> gradient_updates = grad_updates , <EOL> params = params , <EOL> param_constrainers = [ model . modify_updates ] , <EOL> lr_scalers = model . get_lr_scalers ( ) , <EOL> inputs = theano_args , <EOL> verbose = self . verbose_optimization , <EOL> max_iter = self . updates_per_batch , <EOL> reset_alpha = self . reset_alpha , <EOL> conjugate = self . conjugate , <EOL> reset_conjugate = self . reset_conjugate , <EOL> min_init_alpha = self . min_init_alpha , <EOL> line_search_mode = self . line_search_mode , <EOL> theano_function_mode = self . theano_function_mode , <EOL> init_alpha = self . init_alpha ) <EOL> if self . monitoring_dataset is not None : <EOL> self . monitor . add_channel ( <EOL> name = '<STR_LIT>' , <EOL> ipt = None , <EOL> val = self . optimizer . ave_step_size , <EOL> data_specs = ( NullSpace ( ) , '<STR_LIT>' ) , <EOL> dataset = first_value ( self . monitoring_dataset ) ) <EOL> self . monitor . add_channel ( <EOL> name = '<STR_LIT>' , <EOL> ipt = None , <EOL> val = self . optimizer . ave_grad_size , <EOL> data_specs = ( NullSpace ( ) , '<STR_LIT>' ) , <EOL> dataset = first_value ( self . monitoring_dataset ) ) <EOL> self . monitor . add_channel ( <EOL> name = '<STR_LIT>' , <EOL> ipt = None , <EOL> val = self . optimizer . ave_grad_mult , <EOL> data_specs = ( NullSpace ( ) , '<STR_LIT>' ) , <EOL> dataset = first_value ( self . monitoring_dataset ) ) <EOL> self . first = True <EOL> self . bSetup = True <EOL> def train ( self , dataset ) : <EOL> """<STR_LIT>""" <EOL> assert self . bSetup <EOL> model = self . model <EOL> rng = self . rng <EOL> train_iteration_mode = '<STR_LIT>' <EOL> if not is_stochastic ( train_iteration_mode ) : <EOL> rng = None <EOL> data_specs = self . cost . get_data_specs ( self . model ) <EOL> mapping = DataSpecsMapping ( data_specs ) <EOL> space_tuple = mapping . flatten ( data_specs [ <NUM_LIT:0> ] , return_tuple = True ) <EOL> source_tuple = mapping . flatten ( data_specs [ <NUM_LIT:1> ] , return_tuple = True ) <EOL> if len ( space_tuple ) == <NUM_LIT:0> : <EOL> raise NotImplementedError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % str ( data_specs ) ) <EOL> flat_data_specs = ( CompositeSpace ( space_tuple ) , source_tuple ) <EOL> iterator = dataset . iterator ( mode = train_iteration_mode , <EOL> batch_size = self . batch_size , <EOL> num_batches = self . batches_per_iter , <EOL> data_specs = flat_data_specs , <EOL> return_tuple = True , <EOL> rng = rng ) <EOL> mode = self . theano_function_mode <EOL> for data in iterator : <EOL> if ( '<STR_LIT>' in source_tuple and mode is not None <EOL> and hasattr ( mode , '<STR_LIT>' ) ) : <EOL> Y = data [ source_tuple . index ( '<STR_LIT>' ) ] <EOL> stry = str ( Y ) . replace ( '<STR_LIT:\n>' , '<STR_LIT:U+0020>' ) <EOL> mode . record . handle_line ( '<STR_LIT>' + stry + '<STR_LIT:\n>' ) <EOL> for on_load_batch in self . on_load_batch : <EOL> on_load_batch ( mapping . nest ( data ) ) <EOL> self . before_step ( model ) <EOL> self . optimizer . minimize ( * data ) <EOL> self . after_step ( model ) <EOL> actual_batch_size = flat_data_specs [ <NUM_LIT:0> ] . np_batch_size ( data ) <EOL> model . monitor . report_batch ( actual_batch_size ) <EOL> def continue_learning ( self , model ) : <EOL> """<STR_LIT>""" <EOL> if self . termination_criterion is None : <EOL> return True <EOL> else : <EOL> rval = self . termination_criterion . continue_learning ( self . model ) <EOL> assert rval in [ True , False , <NUM_LIT:0> , <NUM_LIT:1> ] <EOL> return rval <EOL> def before_step ( self , model ) : <EOL> """<STR_LIT>""" <EOL> if self . scale_step != <NUM_LIT:1.> : <EOL> self . params = list ( model . get_params ( ) ) <EOL> self . value = [ param . get_value ( ) for param in self . params ] <EOL> def after_step ( self , model ) : <EOL> """<STR_LIT>""" <EOL> if self . scale_step != <NUM_LIT:1> : <EOL> for param , value in safe_zip ( self . params , self . value ) : <EOL> value = ( <NUM_LIT:1.> - self . scale_step ) * value + self . scale_step * param . get_value ( ) <EOL> param . set_value ( value ) <EOL> class StepShrinker ( TrainExtension , TerminationCriterion ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , channel , scale , giveup_after , scale_up = <NUM_LIT:1.> , <EOL> max_scale = <NUM_LIT:1.> ) : <EOL> self . __dict__ . update ( locals ( ) ) <EOL> del self . self <EOL> self . continue_learning = True <EOL> self . first = True <EOL> self . prev = np . inf <EOL> def on_monitor ( self , model , dataset , algorithm ) : <EOL> """<STR_LIT>""" <EOL> monitor = model . monitor <EOL> if self . first : <EOL> self . first = False <EOL> self . monitor_channel = sharedX ( algorithm . scale_step ) <EOL> hack = monitor . channels . values ( ) [ <NUM_LIT:0> ] <EOL> monitor . add_channel ( '<STR_LIT>' , hack . graph_input , <EOL> self . monitor_channel , dataset = hack . dataset , <EOL> data_specs = hack . data_specs ) <EOL> channel = monitor . channels [ self . channel ] <EOL> v = channel . val_record <EOL> if len ( v ) == <NUM_LIT:1> : <EOL> return <EOL> latest = v [ - <NUM_LIT:1> ] <EOL> logger . info ( "<STR_LIT>" . format ( self . channel , latest ) ) <EOL> logger . info ( "<STR_LIT>" . format ( self . prev ) ) <EOL> cur = algorithm . scale_step <EOL> if latest >= self . prev : <EOL> logger . info ( "<STR_LIT>" <EOL> "<STR_LIT>" . format ( cur ) ) <EOL> cur *= self . scale <EOL> if cur < self . giveup_after : <EOL> logger . info ( "<STR_LIT>" ) <EOL> self . continue_learning = False <EOL> cur = self . giveup_after <EOL> logger . info ( "<STR_LIT>" . format ( cur ) ) <EOL> elif latest <= self . prev and self . scale_up != <NUM_LIT:1.> : <EOL> logger . info ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> cur *= self . scale_up <EOL> if cur > self . max_scale : <EOL> cur = self . max_scale <EOL> logger . info ( "<STR_LIT>" . format ( cur ) ) <EOL> algorithm . scale_step = cur <EOL> self . monitor_channel . set_value ( np . cast [ config . floatX ] ( cur ) ) <EOL> self . prev = latest <EOL> def __call__ ( self , model ) : <EOL> """<STR_LIT>""" <EOL> return self . continue_learning <EOL> class ScaleStep ( TrainExtension ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , scale , min_value ) : <EOL> self . scale = scale <EOL> self . min_value = min_value <EOL> self . first = True <EOL> def on_monitor ( self , model , dataset , algorithm ) : <EOL> """<STR_LIT>""" <EOL> if self . first : <EOL> monitor = model . monitor <EOL> self . first = False <EOL> self . monitor_channel = sharedX ( algorithm . scale_step ) <EOL> hack = monitor . channels . values ( ) [ <NUM_LIT:0> ] <EOL> monitor . add_channel ( '<STR_LIT>' , hack . graph_input , <EOL> self . monitor_channel , dataset = hack . dataset ) <EOL> cur = algorithm . scale_step <EOL> cur *= self . scale <EOL> cur = max ( cur , self . min_value ) <EOL> algorithm . scale_step = cur <EOL> self . monitor_channel . set_value ( np . cast [ config . floatX ] ( cur ) ) <EOL> class BacktrackingStepShrinker ( TrainExtension , TerminationCriterion ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , channel , scale , giveup_after , scale_up = <NUM_LIT:1.> , <EOL> max_scale = <NUM_LIT:1.> ) : <EOL> self . __dict__ . update ( locals ( ) ) <EOL> del self . self <EOL> self . continue_learning = True <EOL> self . first = True <EOL> self . prev = np . inf <EOL> def on_monitor ( self , model , dataset , algorithm ) : <EOL> """<STR_LIT>""" <EOL> monitor = model . monitor <EOL> if self . first : <EOL> self . first = False <EOL> self . monitor_channel = sharedX ( algorithm . scale_step ) <EOL> hack = monitor . channels . values ( ) [ <NUM_LIT:0> ] <EOL> monitor . add_channel ( '<STR_LIT>' , hack . graph_input , <EOL> self . monitor_channel , dataset = hack . dataset ) <EOL> channel = monitor . channels [ self . channel ] <EOL> v = channel . val_record <EOL> if len ( v ) == <NUM_LIT:1> : <EOL> return <EOL> latest = v [ - <NUM_LIT:1> ] <EOL> logger . info ( "<STR_LIT>" . format ( self . channel , latest ) ) <EOL> logger . info ( "<STR_LIT>" . format ( self . prev ) ) <EOL> cur = algorithm . scale_step <EOL> if latest >= self . prev : <EOL> logger . info ( "<STR_LIT>" <EOL> "<STR_LIT>" . format ( cur ) ) <EOL> cur *= self . scale <EOL> if cur < self . giveup_after : <EOL> logger . info ( "<STR_LIT>" ) <EOL> self . continue_learning = False <EOL> cur = self . giveup_after <EOL> logger . info ( "<STR_LIT>" . format ( cur ) ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> for p , v in safe_zip ( model . get_params ( ) , self . stored_values ) : <EOL> p . set_value ( v ) <EOL> latest = self . prev <EOL> elif latest <= self . prev and self . scale_up != <NUM_LIT:1.> : <EOL> logger . info ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> cur *= self . scale_up <EOL> if cur > self . max_scale : <EOL> cur = self . max_scale <EOL> logger . info ( "<STR_LIT>" . format ( cur ) ) <EOL> algorithm . scale_step = cur <EOL> self . monitor_channel . set_value ( np . cast [ config . floatX ] ( cur ) ) <EOL> self . prev = latest <EOL> self . stored_values = [ param . get_value ( ) for param in <EOL> model . get_params ( ) ] <EOL> def __call__ ( self , model ) : <EOL> """<STR_LIT>""" <EOL> return self . continue_learning </s>
