<s> from distribution import Distribution <EOL> import ipdb <EOL> import numpy as np <EOL> class SGD ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , model_distribution , learning_rate = <NUM_LIT> , weight_decay = <NUM_LIT> , momentum = <NUM_LIT> , is_ascent = True ) : <EOL> assert ( isinstance ( model_distribution , Distribution ) ) <EOL> self . model_distribution = model_distribution <EOL> assert ( learning_rate > <NUM_LIT:0> ) <EOL> self . learning_rate = learning_rate <EOL> self . weigth_decay = weight_decay <EOL> self . momentum = momentum <EOL> self . ascent_factor = <NUM_LIT:1> if is_ascent else - <NUM_LIT:1> <EOL> self . d_weights = np . zeros ( shape = self . model_distribution . weights . shape ) <EOL> self . d_bias_visible = np . zeros ( shape = self . model_distribution . bias_visible . shape ) <EOL> self . d_bias_hidden = np . zeros ( shape = self . model_distribution . bias_hidden . shape ) <EOL> def optimize ( self , visible_0_states , hidden_0_states , hidden_0_probs , hidden_k_probs , hidden_k_states , <EOL> visible_k_probs , visible_k_states ) : <EOL> """<STR_LIT>""" <EOL> assert ( all ( dim == visible_0_states . ndim for dim in ( visible_k_probs . ndim , visible_k_states . ndim ) ) ) <EOL> assert ( all ( dim == hidden_0_states . ndim for dim in ( hidden_0_probs . ndim , hidden_k_probs . ndim , <EOL> hidden_k_states . ndim ) ) ) <EOL> assert ( self . model_distribution . size_visible in visible_0_states . shape ) <EOL> assert ( self . model_distribution . size_visible in visible_k_states . shape ) <EOL> assert ( self . model_distribution . size_visible in visible_k_probs . shape ) <EOL> assert ( self . model_distribution . size_hidden in hidden_0_states . shape ) <EOL> assert ( self . model_distribution . size_hidden in hidden_0_probs . shape ) <EOL> assert ( self . model_distribution . size_hidden in hidden_k_probs . shape ) <EOL> assert ( self . model_distribution . size_hidden in hidden_k_states . shape ) <EOL> if visible_k_probs . ndim > <NUM_LIT:1> : <EOL> assert ( all ( batch == len ( visible_0_states ) for batch in [ len ( hidden_0_states ) , len ( hidden_0_probs ) , <EOL> len ( hidden_k_probs ) , len ( hidden_k_states ) , <EOL> len ( visible_k_probs ) , len ( visible_k_states ) ] ) ) <EOL> d_weight_update = ( np . dot ( visible_0_states . T , hidden_0_probs ) - <EOL> np . dot ( visible_k_states . T , hidden_k_probs ) ) / float ( len ( visible_0_states ) ) <EOL> d_weight_update = self . ascent_factor * self . learning_rate * d_weight_update - self . weigth_decay * self . model_distribution . weights + self . momentum * self . d_weights <EOL> d_bias_hidden_update = np . mean ( hidden_0_probs - hidden_k_probs , axis = <NUM_LIT:0> ) <EOL> d_bias_hidden_update = self . ascent_factor * self . learning_rate * d_bias_hidden_update - self . weigth_decay * self . model_distribution . bias_hidden + self . momentum * self . d_bias_hidden <EOL> d_bias_visible_update = np . mean ( visible_0_states - visible_k_states , axis = <NUM_LIT:0> ) <EOL> d_bias_visible_update = self . ascent_factor * self . learning_rate * d_bias_visible_update - self . weigth_decay * self . model_distribution . bias_visible + self . momentum * self . d_bias_visible <EOL> self . d_weights = np . copy ( d_weight_update ) <EOL> self . d_bias_hidden = np . copy ( d_bias_hidden_update ) <EOL> self . d_bias_visible = np . copy ( d_bias_visible_update ) <EOL> return d_weight_update , d_bias_hidden_update , d_bias_visible_update <EOL> class DynamicSGD ( SGD ) : <EOL> def __init__ ( self , model_distribution , learning_rate = <NUM_LIT> , weight_decay = <NUM_LIT> , momentum = <NUM_LIT> , is_ascent = True ) : <EOL> SGD . __init__ ( self , model_distribution , learning_rate , weight_decay , momentum , is_ascent ) <EOL> self . d_vis_vis = np . zeros ( shape = model_distribution . vis_vis_weights . shape ) <EOL> self . d_vis_hid = np . zeros ( shape = model_distribution . vis_hid_weights . shape ) <EOL> def optimize ( self , visible_0_states , hidden_0_states , hidden_0_probs , hidden_k_probs , hidden_k_states , <EOL> visible_k_probs , visible_k_states , visible_lagged ) : <EOL> assert ( visible_lagged . shape [ <NUM_LIT:1> : ] == ( self . model_distribution . size_visible , self . model_distribution . m_lag_visible ) ) <EOL> assert ( len ( hidden_0_states ) == len ( visible_k_states ) ) <EOL> assert ( len ( visible_0_states ) == len ( visible_k_states ) ) <EOL> assert ( len ( visible_lagged ) == len ( visible_k_states ) ) <EOL> batch_size = len ( hidden_0_states ) <EOL> d_vis_vis = np . zeros ( shape = self . d_vis_vis . shape ) <EOL> d_vis_hid = np . zeros ( shape = self . d_vis_hid . shape ) <EOL> for lag in range ( self . model_distribution . m_lag_visible ) : <EOL> d_vis_vis [ : , : , lag ] = <NUM_LIT:1.> / batch_size * np . dot ( ( visible_0_states - visible_k_states ) . T , visible_lagged [ : , : , lag ] ) <EOL> for lag in range ( self . model_distribution . n_lag_hidden ) : <EOL> d_vis_hid [ : , : , lag ] = <NUM_LIT:1.> / batch_size * np . dot ( visible_lagged [ : , : , lag ] . T , hidden_0_states ) <EOL> d_vis_vis = self . ascent_factor * self . learning_rate * d_vis_vis - self . weigth_decay * self . model_distribution . vis_vis_weights + self . momentum * self . d_vis_vis <EOL> d_vis_hid = self . ascent_factor * self . learning_rate * d_vis_hid - self . weigth_decay * self . model_distribution . vis_hid_weights + self . momentum * self . d_vis_hid <EOL> self . d_vis_vis = np . copy ( d_vis_vis ) <EOL> self . d_vis_hid = np . copy ( d_vis_hid ) <EOL> d_weight_update , d_bias_hidden_update , d_bias_visible_update = super ( DynamicSGD , self ) . optimize ( visible_0_states , hidden_0_states , <EOL> hidden_0_probs , hidden_k_probs , hidden_k_states , <EOL> visible_k_probs , visible_k_states ) <EOL> return d_weight_update , d_bias_hidden_update , d_bias_visible_update , d_vis_vis , d_vis_hid </s>
