<s> import theano <EOL> import theano . tensor as T <EOL> import numpy as np <EOL> from theano_utils import shared0s , floatX <EOL> def clip_norm ( g , c , n ) : <EOL> if c > <NUM_LIT:0> : <EOL> g = T . switch ( T . ge ( n , c ) , g * c / n , g ) <EOL> return g <EOL> def clip_norms ( gs , c ) : <EOL> norm = T . sqrt ( sum ( [ T . sum ( g ** <NUM_LIT:2> ) for g in gs ] ) ) <EOL> return [ clip_norm ( g , c , norm ) for g in gs ] <EOL> class Regularizer ( object ) : <EOL> def __init__ ( self , l1 = <NUM_LIT:0.> , l2 = <NUM_LIT:0.> , maxnorm = <NUM_LIT:0.> ) : <EOL> self . __dict__ . update ( locals ( ) ) <EOL> def max_norm ( self , p , maxnorm ) : <EOL> if maxnorm > <NUM_LIT:0> : <EOL> norms = T . sqrt ( T . sum ( T . sqr ( p ) , axis = <NUM_LIT:0> ) ) <EOL> desired = T . clip ( norms , <NUM_LIT:0> , maxnorm ) <EOL> p = p * ( desired / ( <NUM_LIT> + norms ) ) <EOL> return p <EOL> def gradient_regularize ( self , p , g ) : <EOL> g += p * self . l2 <EOL> g += T . sgn ( p ) * self . l1 <EOL> return g <EOL> def weight_regularize ( self , p ) : <EOL> p = self . max_norm ( p , self . maxnorm ) <EOL> return p <EOL> class Update ( object ) : <EOL> def __init__ ( self , regularizer = Regularizer ( ) , clipnorm = <NUM_LIT:0.> ) : <EOL> self . __dict__ . update ( locals ( ) ) <EOL> def get_updates ( self , params , grads ) : <EOL> raise NotImplementedError <EOL> class SGD ( Update ) : <EOL> def __init__ ( self , lr = <NUM_LIT> , * args , ** kwargs ) : <EOL> Update . __init__ ( self , * args , ** kwargs ) <EOL> self . __dict__ . update ( locals ( ) ) <EOL> def get_updates ( self , params , cost ) : <EOL> updates = [ ] <EOL> grads = T . grad ( cost , params ) <EOL> grads = clip_norms ( grads , self . clipnorm ) <EOL> for p , g in zip ( params , grads ) : <EOL> g = self . regularizer . gradient_regularize ( p , g ) <EOL> updated_p = p - self . lr * g <EOL> updated_p = self . regularizer . weight_regularize ( updated_p ) <EOL> updates . append ( ( p , updated_p ) ) <EOL> return updates <EOL> class Momentum ( Update ) : <EOL> def __init__ ( self , lr = <NUM_LIT> , momentum = <NUM_LIT> , * args , ** kwargs ) : <EOL> Update . __init__ ( self , * args , ** kwargs ) <EOL> self . __dict__ . update ( locals ( ) ) <EOL> def get_updates ( self , params , cost ) : <EOL> updates = [ ] <EOL> grads = T . grad ( cost , params ) <EOL> grads = clip_norms ( grads , self . clipnorm ) <EOL> for p , g in zip ( params , grads ) : <EOL> g = self . regularizer . gradient_regularize ( p , g ) <EOL> m = theano . shared ( p . get_value ( ) * <NUM_LIT:0.> ) <EOL> v = ( self . momentum * m ) - ( self . lr * g ) <EOL> updates . append ( ( m , v ) ) <EOL> updated_p = p + v <EOL> updated_p = self . regularizer . weight_regularize ( updated_p ) <EOL> updates . append ( ( p , updated_p ) ) <EOL> return updates <EOL> class NAG ( Update ) : <EOL> def __init__ ( self , lr = <NUM_LIT> , momentum = <NUM_LIT> , * args , ** kwargs ) : <EOL> Update . __init__ ( self , * args , ** kwargs ) <EOL> self . __dict__ . update ( locals ( ) ) <EOL> def get_updates ( self , params , cost ) : <EOL> updates = [ ] <EOL> grads = T . grad ( cost , params ) <EOL> grads = clip_norms ( grads , self . clipnorm ) <EOL> for p , g in zip ( params , grads ) : <EOL> g = self . regularizer . gradient_regularize ( p , g ) <EOL> m = theano . shared ( p . get_value ( ) * <NUM_LIT:0.> ) <EOL> v = ( self . momentum * m ) - ( self . lr * g ) <EOL> updates . append ( ( m , v ) ) <EOL> updated_p = p + self . momentum * v - self . lr * g <EOL> updated_p = self . regularizer . weight_regularize ( updated_p ) <EOL> updates . append ( ( p , updated_p ) ) <EOL> return updates <EOL> class RMSprop ( Update ) : <EOL> def __init__ ( self , lr = <NUM_LIT> , rho = <NUM_LIT> , epsilon = <NUM_LIT> , * args , ** kwargs ) : <EOL> Update . __init__ ( self , * args , ** kwargs ) <EOL> self . __dict__ . update ( locals ( ) ) <EOL> def get_updates ( self , params , cost ) : <EOL> updates = [ ] <EOL> grads = T . grad ( cost , params ) <EOL> grads = clip_norms ( grads , self . clipnorm ) <EOL> for p , g in zip ( params , grads ) : <EOL> g = self . regularizer . gradient_regularize ( p , g ) <EOL> acc = theano . shared ( p . get_value ( ) * <NUM_LIT:0.> ) <EOL> acc_new = self . rho * acc + ( <NUM_LIT:1> - self . rho ) * g ** <NUM_LIT:2> <EOL> updates . append ( ( acc , acc_new ) ) <EOL> updated_p = p - self . lr * ( g / T . sqrt ( acc_new + self . epsilon ) ) <EOL> updated_p = self . regularizer . weight_regularize ( updated_p ) <EOL> updates . append ( ( p , updated_p ) ) <EOL> return updates <EOL> class Adam ( Update ) : <EOL> def __init__ ( self , lr = <NUM_LIT> , b1 = <NUM_LIT:0.1> , b2 = <NUM_LIT> , e = <NUM_LIT> , * args , ** kwargs ) : <EOL> Update . __init__ ( self , * args , ** kwargs ) <EOL> self . __dict__ . update ( locals ( ) ) <EOL> def get_updates ( self , params , cost ) : <EOL> updates = [ ] <EOL> grads = T . grad ( cost , params ) <EOL> grads = clip_norms ( grads , self . clipnorm ) <EOL> i = theano . shared ( floatX ( <NUM_LIT:0.> ) ) <EOL> i_t = i + <NUM_LIT:1.> <EOL> fix1 = <NUM_LIT:1.> - self . b1 ** ( i_t ) <EOL> fix2 = <NUM_LIT:1.> - self . b2 ** ( i_t ) <EOL> lr_t = self . lr * ( T . sqrt ( fix2 ) / fix1 ) <EOL> for p , g in zip ( params , grads ) : <EOL> m = theano . shared ( p . get_value ( ) * <NUM_LIT:0.> ) <EOL> v = theano . shared ( p . get_value ( ) * <NUM_LIT:0.> ) <EOL> m_t = ( self . b1 * g ) + ( ( <NUM_LIT:1.> - self . b1 ) * m ) <EOL> v_t = ( self . b2 * T . sqr ( g ) ) + ( ( <NUM_LIT:1.> - self . b2 ) * v ) <EOL> g_t = m_t / ( T . sqrt ( v_t ) + self . e ) <EOL> g_t = self . regularizer . gradient_regularize ( p , g_t ) <EOL> p_t = p - ( lr_t * g_t ) <EOL> p_t = self . regularizer . weight_regularize ( p_t ) <EOL> updates . append ( ( m , m_t ) ) <EOL> updates . append ( ( v , v_t ) ) <EOL> updates . append ( ( p , p_t ) ) <EOL> updates . append ( ( i , i_t ) ) <EOL> return updates <EOL> class Adagrad ( Update ) : <EOL> def __init__ ( self , lr = <NUM_LIT> , epsilon = <NUM_LIT> , * args , ** kwargs ) : <EOL> Update . __init__ ( self , * args , ** kwargs ) <EOL> self . __dict__ . update ( locals ( ) ) <EOL> def get_updates ( self , params , cost ) : <EOL> updates = [ ] <EOL> grads = T . grad ( cost , params ) <EOL> grads = clip_norms ( grads , self . clipnorm ) <EOL> for p , g in zip ( params , grads ) : <EOL> g = self . regularizer . gradient_regularize ( p , g ) <EOL> acc = theano . shared ( p . get_value ( ) * <NUM_LIT:0.> ) <EOL> acc_t = acc + g ** <NUM_LIT:2> <EOL> updates . append ( ( acc , acc_t ) ) <EOL> p_t = p - ( self . lr / T . sqrt ( acc_t + self . epsilon ) ) * g <EOL> p_t = self . regularizer . weight_regularize ( p_t ) <EOL> updates . append ( ( p , p_t ) ) <EOL> return updates <EOL> class Adadelta ( Update ) : <EOL> def __init__ ( self , lr = <NUM_LIT:0.5> , rho = <NUM_LIT> , epsilon = <NUM_LIT> , * args , ** kwargs ) : <EOL> Update . __init__ ( self , * args , ** kwargs ) <EOL> self . __dict__ . update ( locals ( ) ) <EOL> def get_updates ( self , params , cost ) : <EOL> updates = [ ] <EOL> grads = T . grad ( cost , params ) <EOL> grads = clip_norms ( grads , self . clipnorm ) <EOL> for p , g in zip ( params , grads ) : <EOL> g = self . regularizer . gradient_regularize ( p , g ) <EOL> acc = theano . shared ( p . get_value ( ) * <NUM_LIT:0.> ) <EOL> acc_delta = theano . shared ( p . get_value ( ) * <NUM_LIT:0.> ) <EOL> acc_new = self . rho * acc + ( <NUM_LIT:1> - self . rho ) * g ** <NUM_LIT:2> <EOL> updates . append ( ( acc , acc_new ) ) <EOL> update = g * T . sqrt ( acc_delta + self . epsilon ) / T . sqrt ( acc_new + self . epsilon ) <EOL> updated_p = p - self . lr * update <EOL> updated_p = self . regularizer . weight_regularize ( updated_p ) <EOL> updates . append ( ( p , updated_p ) ) <EOL> acc_delta_new = self . rho * acc_delta + ( <NUM_LIT:1> - self . rho ) * update ** <NUM_LIT:2> <EOL> updates . append ( ( acc_delta , acc_delta_new ) ) <EOL> return updates </s>
