<s> """<STR_LIT>""" <EOL> import logging <EOL> import itertools <EOL> from abc import ABCMeta , abstractmethod <EOL> from collections import OrderedDict <EOL> from six . moves import reduce <EOL> from picklable_itertools . extras import equizip <EOL> import theano <EOL> from six import add_metaclass <EOL> from theano import tensor <EOL> from blocks . graph import ComputationGraph <EOL> from blocks . roles import add_role , ALGORITHM_HYPERPARAMETER , ALGORITHM_BUFFER <EOL> from blocks . theano_expressions import l2_norm <EOL> from blocks . utils import ( dict_subset , pack , shared_floatx , <EOL> shared_floatx_zeros_matching ) <EOL> logger = logging . getLogger ( __name__ ) <EOL> @ add_metaclass ( ABCMeta ) <EOL> class TrainingAlgorithm ( object ) : <EOL> """<STR_LIT>""" <EOL> @ abstractmethod <EOL> def initialize ( self , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> @ abstractmethod <EOL> def process_batch ( self , batch ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class DifferentiableCostMinimizer ( TrainingAlgorithm ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , cost , parameters ) : <EOL> self . cost = cost <EOL> self . parameters = parameters <EOL> self . _cost_computation_graph = ComputationGraph ( self . cost ) <EOL> self . _updates = [ ] <EOL> @ property <EOL> def inputs ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _cost_computation_graph . inputs <EOL> @ property <EOL> def updates ( self ) : <EOL> return self . _updates <EOL> @ updates . setter <EOL> def updates ( self , value ) : <EOL> self . _updates = value <EOL> def add_updates ( self , updates ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( updates , OrderedDict ) : <EOL> updates = list ( updates . items ( ) ) <EOL> if not isinstance ( updates , list ) : <EOL> raise ValueError <EOL> self . updates . extend ( updates ) <EOL> variable_mismatch_error = """<STR_LIT>""" <EOL> source_missing_error = """<STR_LIT>""" <EOL> class GradientDescent ( DifferentiableCostMinimizer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , step_rule = None , gradients = None , known_grads = None , <EOL> consider_constant = None , on_unused_sources = '<STR_LIT>' , <EOL> theano_func_kwargs = None , ** kwargs ) : <EOL> if gradients : <EOL> kwargs . setdefault ( "<STR_LIT>" , gradients . keys ( ) ) <EOL> super ( GradientDescent , self ) . __init__ ( ** kwargs ) <EOL> self . gradients = gradients <EOL> if not self . gradients : <EOL> logger . info ( "<STR_LIT>" ) <EOL> self . gradients = dict ( <EOL> equizip ( self . parameters , tensor . grad ( <EOL> self . cost , self . parameters , <EOL> known_grads = known_grads , <EOL> consider_constant = consider_constant ) ) ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> else : <EOL> if known_grads : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if consider_constant is not None : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> self . step_rule = step_rule if step_rule else Scale ( ) <EOL> self . total_gradient_norm = l2_norm ( <EOL> self . gradients . values ( ) ) . copy ( name = "<STR_LIT>" ) <EOL> self . steps , self . step_rule_updates = ( <EOL> self . step_rule . compute_steps ( self . gradients ) ) <EOL> self . total_step_norm = l2_norm ( <EOL> self . steps . values ( ) ) . copy ( name = "<STR_LIT>" ) <EOL> self . on_unused_sources = on_unused_sources <EOL> self . theano_func_kwargs = ( theano_func_kwargs if theano_func_kwargs <EOL> is not None else dict ( ) ) <EOL> def initialize ( self ) : <EOL> logger . info ( "<STR_LIT>" ) <EOL> all_updates = self . updates <EOL> for parameter in self . parameters : <EOL> all_updates . append ( ( parameter , parameter - self . steps [ parameter ] ) ) <EOL> all_updates += self . step_rule_updates <EOL> self . _function = theano . function ( <EOL> self . inputs , [ ] , updates = all_updates , ** self . theano_func_kwargs ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> def _validate_source_names ( self , batch ) : <EOL> in_names = [ v . name for v in self . inputs ] <EOL> if not set ( in_names ) . issubset ( set ( batch . keys ( ) ) ) : <EOL> raise ValueError ( "<STR_LIT>" + <EOL> source_missing_error . format ( <EOL> sources = batch . keys ( ) , <EOL> variables = in_names ) ) <EOL> if not set ( batch . keys ( ) ) . issubset ( set ( in_names ) ) : <EOL> if self . on_unused_sources == '<STR_LIT:ignore>' : <EOL> pass <EOL> elif self . on_unused_sources == '<STR_LIT>' : <EOL> if not hasattr ( self , '<STR_LIT>' ) : <EOL> logger . warn ( variable_mismatch_error . format ( <EOL> sources = batch . keys ( ) , <EOL> variables = in_names ) ) <EOL> self . _unused_source_warned = True <EOL> elif self . on_unused_sources == '<STR_LIT>' : <EOL> raise ValueError ( <EOL> "<STR_LIT>" + <EOL> variable_mismatch_error . format ( <EOL> sources = batch . keys ( ) , <EOL> variables = in_names ) ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" <EOL> . format ( self . on_unused_sources ) ) <EOL> def process_batch ( self , batch ) : <EOL> self . _validate_source_names ( batch ) <EOL> ordered_batch = [ batch [ v . name ] for v in self . inputs ] <EOL> self . _function ( * ordered_batch ) <EOL> @ add_metaclass ( ABCMeta ) <EOL> class StepRule ( object ) : <EOL> """<STR_LIT>""" <EOL> def compute_step ( self , parameter , previous_step ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def compute_steps ( self , previous_steps ) : <EOL> """<STR_LIT>""" <EOL> parameter_wise = [ self . compute_step ( parameter , <EOL> previous_steps [ parameter ] ) <EOL> for parameter in previous_steps ] <EOL> steps , updates = equizip ( * parameter_wise ) <EOL> steps = OrderedDict ( ( parameter , step ) for parameter , step <EOL> in equizip ( previous_steps . keys ( ) , steps ) ) <EOL> updates = list ( itertools . chain ( * updates ) ) <EOL> return steps , updates <EOL> class CompositeRule ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , components ) : <EOL> self . components = components <EOL> def compute_steps ( self , previous_steps ) : <EOL> steps = previous_steps <EOL> updates = [ ] <EOL> for rule in self . components : <EOL> steps , more_updates = rule . compute_steps ( steps ) <EOL> updates += more_updates <EOL> return steps , updates <EOL> class Scale ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , learning_rate = <NUM_LIT:1.0> ) : <EOL> self . learning_rate = shared_floatx ( learning_rate , "<STR_LIT>" ) <EOL> add_role ( self . learning_rate , ALGORITHM_HYPERPARAMETER ) <EOL> def compute_step ( self , parameter , previous_step ) : <EOL> return self . learning_rate * previous_step , [ ] <EOL> class BasicMomentum ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , momentum = <NUM_LIT:0.> ) : <EOL> self . momentum = shared_floatx ( momentum , "<STR_LIT>" ) <EOL> add_role ( self . momentum , ALGORITHM_HYPERPARAMETER ) <EOL> def compute_step ( self , parameter , previous_step ) : <EOL> velocity = shared_floatx_zeros_matching ( parameter , "<STR_LIT>" ) <EOL> add_role ( velocity , ALGORITHM_BUFFER ) <EOL> step = self . momentum * velocity + previous_step <EOL> updates = [ ( velocity , step ) ] <EOL> return step , updates <EOL> class Momentum ( CompositeRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , learning_rate = <NUM_LIT:1.0> , momentum = <NUM_LIT:0.> ) : <EOL> scale = Scale ( learning_rate = learning_rate ) <EOL> basic_momentum = BasicMomentum ( momentum = momentum ) <EOL> self . learning_rate = scale . learning_rate <EOL> self . momentum = basic_momentum . momentum <EOL> self . components = [ scale , basic_momentum ] <EOL> class AdaDelta ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , decay_rate = <NUM_LIT> , epsilon = <NUM_LIT> ) : <EOL> if not <NUM_LIT:0.0> <= decay_rate <= <NUM_LIT:1.0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> self . decay_rate = shared_floatx ( decay_rate , "<STR_LIT>" ) <EOL> add_role ( self . decay_rate , ALGORITHM_HYPERPARAMETER ) <EOL> self . epsilon = shared_floatx ( epsilon , "<STR_LIT>" ) <EOL> add_role ( self . epsilon , ALGORITHM_HYPERPARAMETER ) <EOL> def compute_step ( self , parameter , previous_step ) : <EOL> mean_square_step_tm1 = shared_floatx_zeros_matching ( <EOL> parameter , "<STR_LIT>" ) <EOL> add_role ( mean_square_step_tm1 , ALGORITHM_BUFFER ) <EOL> mean_square_delta_x_tm1 = shared_floatx_zeros_matching ( <EOL> parameter , "<STR_LIT>" ) <EOL> add_role ( mean_square_delta_x_tm1 , ALGORITHM_BUFFER ) <EOL> mean_square_step_t = ( <EOL> self . decay_rate * mean_square_step_tm1 + <EOL> ( <NUM_LIT:1> - self . decay_rate ) * tensor . sqr ( previous_step ) <EOL> ) <EOL> rms_delta_x_tm1 = tensor . sqrt ( mean_square_delta_x_tm1 + self . epsilon ) <EOL> rms_step_t = tensor . sqrt ( mean_square_step_t + self . epsilon ) <EOL> delta_x_t = rms_delta_x_tm1 / rms_step_t * previous_step <EOL> mean_square_delta_x_t = ( <EOL> self . decay_rate * mean_square_delta_x_tm1 + <EOL> ( <NUM_LIT:1> - self . decay_rate ) * tensor . sqr ( delta_x_t ) <EOL> ) <EOL> step = delta_x_t <EOL> updates = [ ( mean_square_step_tm1 , mean_square_step_t ) , <EOL> ( mean_square_delta_x_tm1 , mean_square_delta_x_t ) ] <EOL> return step , updates <EOL> class BasicRMSProp ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , decay_rate = <NUM_LIT> , max_scaling = <NUM_LIT> ) : <EOL> if not <NUM_LIT:0.0> <= decay_rate <= <NUM_LIT:1.0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if max_scaling <= <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> self . decay_rate = shared_floatx ( decay_rate , "<STR_LIT>" ) <EOL> add_role ( self . decay_rate , ALGORITHM_HYPERPARAMETER ) <EOL> self . epsilon = <NUM_LIT:1.> / max_scaling <EOL> def compute_step ( self , parameter , previous_step ) : <EOL> mean_square_step_tm1 = shared_floatx_zeros_matching ( <EOL> parameter , "<STR_LIT>" ) <EOL> add_role ( mean_square_step_tm1 , ALGORITHM_BUFFER ) <EOL> mean_square_step_t = ( <EOL> self . decay_rate * mean_square_step_tm1 + <EOL> ( <NUM_LIT:1> - self . decay_rate ) * tensor . sqr ( previous_step ) ) <EOL> add_role ( mean_square_step_t , ALGORITHM_BUFFER ) <EOL> rms_step_t = tensor . maximum ( <EOL> tensor . sqrt ( mean_square_step_t ) , self . epsilon ) <EOL> step = previous_step / rms_step_t <EOL> updates = [ ( mean_square_step_tm1 , mean_square_step_t ) ] <EOL> return step , updates <EOL> class RMSProp ( CompositeRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , learning_rate = <NUM_LIT:1.0> , decay_rate = <NUM_LIT> , max_scaling = <NUM_LIT> ) : <EOL> basic_rms_prop = BasicRMSProp ( decay_rate = decay_rate , <EOL> max_scaling = max_scaling ) <EOL> scale = Scale ( learning_rate = learning_rate ) <EOL> self . learning_rate = scale . learning_rate <EOL> self . decay_rate = basic_rms_prop . decay_rate <EOL> self . components = [ basic_rms_prop , scale ] <EOL> class StepClipping ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , threshold = None ) : <EOL> if threshold : <EOL> self . threshold = shared_floatx ( threshold , "<STR_LIT>" ) <EOL> add_role ( self . threshold , ALGORITHM_HYPERPARAMETER ) <EOL> def compute_steps ( self , previous_steps ) : <EOL> if not hasattr ( self , '<STR_LIT>' ) : <EOL> return previous_steps <EOL> norm = l2_norm ( previous_steps . values ( ) ) <EOL> multiplier = tensor . switch ( norm < self . threshold , <EOL> <NUM_LIT:1> , self . threshold / norm ) <EOL> steps = OrderedDict ( <EOL> ( parameter , step * multiplier ) <EOL> for parameter , step in previous_steps . items ( ) ) <EOL> return steps , [ ] <EOL> class VariableClipping ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , threshold , axis = None ) : <EOL> axis = pack ( axis ) if axis is not None else ( ) <EOL> self . axis = set ( axis ) <EOL> self . threshold = shared_floatx ( threshold , "<STR_LIT>" ) <EOL> add_role ( self . threshold , ALGORITHM_HYPERPARAMETER ) <EOL> if len ( axis ) != len ( self . axis ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> def compute_step ( self , parameter , previous_step ) : <EOL> if any ( ax >= previous_step . ndim for ax in self . axis ) : <EOL> raise ValueError ( "<STR_LIT>" . format ( <EOL> self . axis , parameter , previous_step . ndim ) ) <EOL> if len ( self . axis ) == <NUM_LIT:0> : <EOL> norms = l2_norm ( [ parameter - previous_step ] ) <EOL> else : <EOL> squares = tensor . sqr ( parameter - previous_step ) <EOL> norms = tensor . sqrt ( <EOL> reduce ( lambda t , a : t . sum ( axis = a , keepdims = True ) , <EOL> sorted ( self . axis ) , squares ) ) <EOL> shrinking_step = ( parameter - <EOL> ( self . threshold / norms ) * <EOL> ( parameter - previous_step ) ) <EOL> return tensor . switch ( norms > self . threshold , <EOL> shrinking_step , <EOL> previous_step ) , ( ) <EOL> class AdaGrad ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , learning_rate = <NUM_LIT> , epsilon = <NUM_LIT> ) : <EOL> self . learning_rate = shared_floatx ( learning_rate , "<STR_LIT>" ) <EOL> self . epsilon = shared_floatx ( epsilon , "<STR_LIT>" ) <EOL> add_role ( self . learning_rate , ALGORITHM_HYPERPARAMETER ) <EOL> add_role ( self . epsilon , ALGORITHM_HYPERPARAMETER ) <EOL> def compute_step ( self , parameter , previous_step ) : <EOL> name = '<STR_LIT>' <EOL> if parameter . name : <EOL> name += '<STR_LIT:_>' + parameter . name <EOL> ssq = shared_floatx_zeros_matching ( parameter , name = name ) <EOL> add_role ( ssq , ALGORITHM_BUFFER ) <EOL> ssq_t = ( tensor . sqr ( previous_step ) + ssq ) <EOL> step = ( self . learning_rate * previous_step / <EOL> ( tensor . sqrt ( ssq_t ) + self . epsilon ) ) <EOL> updates = [ ( ssq , ssq_t ) ] <EOL> return step , updates <EOL> class Adam ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , learning_rate = <NUM_LIT> , <EOL> beta1 = <NUM_LIT:0.1> , beta2 = <NUM_LIT> , epsilon = <NUM_LIT> , <EOL> decay_factor = ( <NUM_LIT:1> - <NUM_LIT> ) ) : <EOL> self . learning_rate = shared_floatx ( learning_rate , "<STR_LIT>" ) <EOL> self . beta1 = shared_floatx ( beta1 , "<STR_LIT>" ) <EOL> self . beta2 = shared_floatx ( beta2 , "<STR_LIT>" ) <EOL> self . epsilon = shared_floatx ( epsilon , "<STR_LIT>" ) <EOL> self . decay_factor = shared_floatx ( decay_factor , "<STR_LIT>" ) <EOL> for param in [ self . learning_rate , self . beta1 , self . beta2 , self . epsilon , <EOL> self . decay_factor ] : <EOL> add_role ( param , ALGORITHM_HYPERPARAMETER ) <EOL> def compute_step ( self , parameter , previous_step ) : <EOL> mean = shared_floatx_zeros_matching ( parameter , '<STR_LIT>' ) <EOL> add_role ( mean , ALGORITHM_BUFFER ) <EOL> variance = shared_floatx_zeros_matching ( parameter , '<STR_LIT>' ) <EOL> add_role ( variance , ALGORITHM_BUFFER ) <EOL> time = shared_floatx ( <NUM_LIT:0.> , '<STR_LIT:time>' ) <EOL> add_role ( time , ALGORITHM_BUFFER ) <EOL> t1 = time + <NUM_LIT:1> <EOL> learning_rate = ( self . learning_rate * <EOL> tensor . sqrt ( ( <NUM_LIT:1.> - ( <NUM_LIT:1.> - self . beta2 ) ** t1 ) ) / <EOL> ( <NUM_LIT:1.> - ( <NUM_LIT:1.> - self . beta1 ) ** t1 ) ) <EOL> beta_1t = <NUM_LIT:1> - ( <NUM_LIT:1> - self . beta1 ) * self . decay_factor ** ( t1 - <NUM_LIT:1> ) <EOL> mean_t = beta_1t * previous_step + ( <NUM_LIT:1.> - beta_1t ) * mean <EOL> variance_t = ( self . beta2 * tensor . sqr ( previous_step ) + <EOL> ( <NUM_LIT:1.> - self . beta2 ) * variance ) <EOL> step = ( learning_rate * mean_t / <EOL> ( tensor . sqrt ( variance_t ) + self . epsilon ) ) <EOL> updates = [ ( mean , mean_t ) , <EOL> ( variance , variance_t ) , <EOL> ( time , t1 ) ] <EOL> return step , updates <EOL> class RemoveNotFinite ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , scaler = <NUM_LIT:1> ) : <EOL> self . scaler = scaler <EOL> def compute_step ( self , parameter , previous_step ) : <EOL> step_sum = tensor . sum ( previous_step ) <EOL> not_finite = ( tensor . isnan ( step_sum ) + <EOL> tensor . isinf ( step_sum ) ) <EOL> step = tensor . switch ( <EOL> not_finite > <NUM_LIT:0> , ( <NUM_LIT:1> - self . scaler ) * parameter , previous_step ) <EOL> return step , [ ] <EOL> class Restrict ( StepRule ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , step_rule , variables ) : <EOL> self . step_rule = step_rule <EOL> self . variables = frozenset ( variables ) <EOL> def compute_steps ( self , previous_steps ) : <EOL> filtered_previous_steps = dict_subset ( previous_steps , self . variables ) <EOL> steps , updates = self . step_rule . compute_steps ( filtered_previous_steps ) <EOL> actual = OrderedDict ( ( parameter , steps [ parameter ] ) <EOL> if parameter in steps <EOL> else ( parameter , previous_steps [ parameter ] ) <EOL> for parameter in previous_steps ) <EOL> return actual , updates </s>
