<s> import numpy as np <EOL> from scipy import linalg <EOL> from scipy . io import loadmat <EOL> from functools import reduce <EOL> import numbers <EOL> import random <EOL> import theano <EOL> import zipfile <EOL> import gzip <EOL> import os <EOL> import glob <EOL> import sys <EOL> import subprocess <EOL> try : <EOL> import cPickle as pickle <EOL> except ImportError : <EOL> import pickle <EOL> from theano import tensor <EOL> from theano . compat . python2x import OrderedDict <EOL> from theano . sandbox . rng_mrg import MRG_RandomStreams <EOL> from collections import defaultdict <EOL> class sgd ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , params ) : <EOL> pass <EOL> def updates ( self , params , grads , learning_rate ) : <EOL> updates = [ ] <EOL> for n , ( param , grad ) in enumerate ( zip ( params , grads ) ) : <EOL> updates . append ( ( param , param - learning_rate * grad ) ) <EOL> return updates <EOL> class sgd_nesterov ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , params ) : <EOL> self . memory_ = [ theano . shared ( np . zeros_like ( p . get_value ( ) ) ) <EOL> for p in params ] <EOL> def updates ( self , params , grads , learning_rate , momentum ) : <EOL> updates = [ ] <EOL> for n , ( param , grad ) in enumerate ( zip ( params , grads ) ) : <EOL> memory = self . memory_ [ n ] <EOL> update = momentum * memory - learning_rate * grad <EOL> update2 = momentum * momentum * memory - ( <EOL> <NUM_LIT:1> + momentum ) * learning_rate * grad <EOL> updates . append ( ( memory , update ) ) <EOL> updates . append ( ( param , param + update2 ) ) <EOL> return updates <EOL> class rmsprop ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , params ) : <EOL> self . running_square_ = [ theano . shared ( np . zeros_like ( p . get_value ( ) ) ) <EOL> for p in params ] <EOL> self . running_avg_ = [ theano . shared ( np . zeros_like ( p . get_value ( ) ) ) <EOL> for p in params ] <EOL> self . memory_ = [ theano . shared ( np . zeros_like ( p . get_value ( ) ) ) <EOL> for p in params ] <EOL> def updates ( self , params , grads , learning_rate , momentum , rescale = <NUM_LIT> ) : <EOL> grad_norm = tensor . sqrt ( sum ( map ( lambda x : tensor . sqr ( x ) . sum ( ) , grads ) ) ) <EOL> not_finite = tensor . or_ ( tensor . isnan ( grad_norm ) , <EOL> tensor . isinf ( grad_norm ) ) <EOL> grad_norm = tensor . sqrt ( grad_norm ) <EOL> scaling_num = rescale <EOL> scaling_den = tensor . maximum ( rescale , grad_norm ) <EOL> combination_coeff = <NUM_LIT> <EOL> minimum_grad = <NUM_LIT> <EOL> updates = [ ] <EOL> for n , ( param , grad ) in enumerate ( zip ( params , grads ) ) : <EOL> grad = tensor . switch ( not_finite , <NUM_LIT:0.1> * param , <EOL> grad * ( scaling_num / scaling_den ) ) <EOL> old_square = self . running_square_ [ n ] <EOL> new_square = combination_coeff * old_square + ( <EOL> <NUM_LIT:1.> - combination_coeff ) * tensor . sqr ( grad ) <EOL> old_avg = self . running_avg_ [ n ] <EOL> new_avg = combination_coeff * old_avg + ( <EOL> <NUM_LIT:1.> - combination_coeff ) * grad <EOL> rms_grad = tensor . sqrt ( new_square - new_avg ** <NUM_LIT:2> ) <EOL> rms_grad = tensor . maximum ( rms_grad , minimum_grad ) <EOL> memory = self . memory_ [ n ] <EOL> update = momentum * memory - learning_rate * grad / rms_grad <EOL> update2 = momentum * momentum * memory - ( <EOL> <NUM_LIT:1> + momentum ) * learning_rate * grad / rms_grad <EOL> updates . append ( ( old_square , new_square ) ) <EOL> updates . append ( ( old_avg , new_avg ) ) <EOL> updates . append ( ( memory , update ) ) <EOL> updates . append ( ( param , param + update2 ) ) <EOL> return updates <EOL> class adagrad ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , params ) : <EOL> self . memory_ = [ theano . shared ( np . zeros_like ( p . get_value ( ) ) ) <EOL> for p in params ] <EOL> def updates ( self , params , grads , learning_rate , eps = <NUM_LIT> ) : <EOL> updates = [ ] <EOL> for n , ( param , grad ) in enumerate ( zip ( params , grads ) ) : <EOL> memory = self . memory_ [ n ] <EOL> m_t = memory + grad ** <NUM_LIT:2> <EOL> g_t = grad / ( eps + tensor . sqrt ( m_t ) ) <EOL> p_t = param - learning_rate * g_t <EOL> updates . append ( ( memory , m_t ) ) <EOL> updates . append ( ( param , p_t ) ) <EOL> return updates <EOL> class adam ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , params ) : <EOL> self . memory_ = [ theano . shared ( np . zeros_like ( p . get_value ( ) ) ) <EOL> for p in params ] <EOL> self . velocity_ = [ theano . shared ( np . zeros_like ( p . get_value ( ) ) ) <EOL> for p in params ] <EOL> self . itr_ = theano . shared ( np . array ( <NUM_LIT:0.> ) . astype ( theano . config . floatX ) ) <EOL> def updates ( self , params , grads , learning_rate , b1 = <NUM_LIT:0.1> , b2 = <NUM_LIT> , eps = <NUM_LIT> ) : <EOL> updates = [ ] <EOL> itr = self . itr_ <EOL> i_t = itr + <NUM_LIT:1.> <EOL> fix1 = <NUM_LIT:1.> - ( <NUM_LIT:1.> - b1 ) ** i_t <EOL> fix2 = <NUM_LIT:1.> - ( <NUM_LIT:1.> - b2 ) ** i_t <EOL> lr_t = learning_rate * ( tensor . sqrt ( fix2 ) / fix1 ) <EOL> for n , ( param , grad ) in enumerate ( zip ( params , grads ) ) : <EOL> memory = self . memory_ [ n ] <EOL> velocity = self . velocity_ [ n ] <EOL> m_t = ( b1 * grad ) + ( ( <NUM_LIT:1.> - b1 ) * memory ) <EOL> v_t = ( b2 * tensor . sqr ( grad ) ) + ( ( <NUM_LIT:1.> - b2 ) * velocity ) <EOL> g_t = m_t / ( tensor . sqrt ( v_t ) + eps ) <EOL> p_t = param - ( lr_t * g_t ) <EOL> updates . append ( ( memory , m_t ) ) <EOL> updates . append ( ( velocity , v_t ) ) <EOL> updates . append ( ( param , p_t ) ) <EOL> updates . append ( ( itr , i_t ) ) <EOL> return updates <EOL> def get_dataset_dir ( dataset_name , data_dir = None , folder = None , create_dir = True ) : <EOL> """<STR_LIT>""" <EOL> if not data_dir : <EOL> data_dir = os . getenv ( "<STR_LIT>" , os . path . join ( <EOL> os . path . expanduser ( "<STR_LIT>" ) , "<STR_LIT>" ) ) <EOL> if folder is None : <EOL> data_dir = os . path . join ( data_dir , dataset_name ) <EOL> else : <EOL> data_dir = os . path . join ( data_dir , folder ) <EOL> if not os . path . exists ( data_dir ) and create_dir : <EOL> os . makedirs ( data_dir ) <EOL> return data_dir <EOL> def download ( url , server_fname , local_fname = None , progress_update_percentage = <NUM_LIT:5> ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> import urllib <EOL> urllib . urlretrieve ( '<STR_LIT>' ) <EOL> except AttributeError : <EOL> import urllib . request as urllib <EOL> u = urllib . urlopen ( url ) <EOL> if local_fname is None : <EOL> local_fname = server_fname <EOL> full_path = local_fname <EOL> meta = u . info ( ) <EOL> with open ( full_path , '<STR_LIT:wb>' ) as f : <EOL> try : <EOL> file_size = int ( meta . get ( "<STR_LIT>" ) ) <EOL> except TypeError : <EOL> print ( "<STR_LIT>" ) <EOL> file_size = <NUM_LIT:100> <EOL> print ( "<STR_LIT>" % ( server_fname , file_size ) ) <EOL> file_size_dl = <NUM_LIT:0> <EOL> block_sz = int ( <NUM_LIT> ) <EOL> p = <NUM_LIT:0> <EOL> while True : <EOL> buffer = u . read ( block_sz ) <EOL> if not buffer : <EOL> break <EOL> file_size_dl += len ( buffer ) <EOL> f . write ( buffer ) <EOL> if ( file_size_dl * <NUM_LIT> / file_size ) > p : <EOL> status = r"<STR_LIT>" % ( file_size_dl , file_size_dl * <EOL> <NUM_LIT> / file_size ) <EOL> print ( status ) <EOL> p += progress_update_percentage <EOL> def make_character_level_from_text ( text ) : <EOL> """<STR_LIT>""" <EOL> all_chars = reduce ( lambda x , y : set ( x ) | set ( y ) , text , set ( ) ) <EOL> mapper = { k : n + <NUM_LIT:2> for n , k in enumerate ( list ( all_chars ) ) } <EOL> mapper [ "<STR_LIT>" ] = <NUM_LIT:1> <EOL> mapper [ "<STR_LIT>" ] = <NUM_LIT:0> <EOL> inverse_mapper = { v : k for k , v in mapper . items ( ) } <EOL> def mapper_func ( text_line ) : <EOL> return [ mapper [ c ] for c in text_line ] + [ mapper [ "<STR_LIT>" ] ] <EOL> def inverse_mapper_func ( symbol_line ) : <EOL> return "<STR_LIT>" . join ( [ inverse_mapper [ s ] for s in symbol_line <EOL> if s != mapper [ "<STR_LIT>" ] ] ) <EOL> cleaned = [ mapper_func ( t ) for t in text if t != "<STR_LIT>" ] <EOL> return cleaned , mapper_func , inverse_mapper_func , mapper <EOL> def check_fetch_uci_words ( ) : <EOL> """<STR_LIT>""" <EOL> url = '<STR_LIT>' <EOL> partial_path = get_dataset_dir ( "<STR_LIT>" ) <EOL> full_path = os . path . join ( partial_path , "<STR_LIT>" ) <EOL> if not os . path . exists ( partial_path ) : <EOL> os . makedirs ( partial_path ) <EOL> if not os . path . exists ( full_path ) : <EOL> all_vocabs = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] <EOL> for vocab in all_vocabs : <EOL> dl_url = url + vocab <EOL> download ( dl_url , os . path . join ( partial_path , vocab ) , <EOL> progress_update_percentage = <NUM_LIT:1> ) <EOL> def zipdir ( path , zipf ) : <EOL> for root , dirs , files in os . walk ( path ) : <EOL> for f in files : <EOL> if "<STR_LIT>" in f : <EOL> zipf . write ( os . path . join ( root , f ) ) <EOL> zipf = zipfile . ZipFile ( full_path , '<STR_LIT:w>' ) <EOL> zipdir ( partial_path , zipf ) <EOL> zipf . close ( ) <EOL> return full_path <EOL> def fetch_uci_words ( ) : <EOL> """<STR_LIT>""" <EOL> data_path = check_fetch_uci_words ( ) <EOL> all_data = [ ] <EOL> with zipfile . ZipFile ( data_path , "<STR_LIT:r>" ) as f : <EOL> for name in f . namelist ( ) : <EOL> if "<STR_LIT>" not in name : <EOL> continue <EOL> data = f . read ( name ) <EOL> data = data . split ( "<STR_LIT:\n>" ) <EOL> data = [ l . strip ( ) for l in data if l != "<STR_LIT>" ] <EOL> all_data . extend ( data ) <EOL> return list ( set ( all_data ) ) <EOL> def check_fetch_lovecraft ( ) : <EOL> """<STR_LIT>""" <EOL> url = '<STR_LIT>' <EOL> partial_path = get_dataset_dir ( "<STR_LIT>" ) <EOL> full_path = os . path . join ( partial_path , "<STR_LIT>" ) <EOL> if not os . path . exists ( partial_path ) : <EOL> os . makedirs ( partial_path ) <EOL> if not os . path . exists ( full_path ) : <EOL> download ( url , full_path , progress_update_percentage = <NUM_LIT:1> ) <EOL> return full_path <EOL> def fetch_lovecraft ( ) : <EOL> """<STR_LIT>""" <EOL> data_path = check_fetch_lovecraft ( ) <EOL> all_data = [ ] <EOL> with zipfile . ZipFile ( data_path , "<STR_LIT:r>" ) as f : <EOL> for name in f . namelist ( ) : <EOL> if "<STR_LIT>" not in name : <EOL> continue <EOL> data = f . read ( name ) <EOL> data = data . split ( "<STR_LIT:\n>" ) <EOL> data = [ l . strip ( ) for l in data if l != "<STR_LIT>" ] <EOL> all_data . extend ( data ) <EOL> return all_data <EOL> def check_fetch_tfd ( ) : <EOL> """<STR_LIT>""" <EOL> partial_path = get_dataset_dir ( "<STR_LIT>" ) <EOL> full_path = os . path . join ( partial_path , "<STR_LIT>" ) <EOL> if not os . path . exists ( partial_path ) : <EOL> os . makedirs ( partial_path ) <EOL> if not os . path . exists ( full_path ) : <EOL> raise ValueError ( "<STR_LIT>" % str ( partial_path ) ) <EOL> return full_path <EOL> def fetch_tfd ( ) : <EOL> """<STR_LIT>""" <EOL> data_path = check_fetch_tfd ( ) <EOL> matfile = loadmat ( data_path ) <EOL> all_data = matfile [ '<STR_LIT>' ] . reshape ( len ( matfile [ '<STR_LIT>' ] ) , - <NUM_LIT:1> ) / <NUM_LIT> <EOL> return all_data <EOL> def check_fetch_frey ( ) : <EOL> """<STR_LIT>""" <EOL> url = '<STR_LIT>' <EOL> partial_path = get_dataset_dir ( "<STR_LIT>" ) <EOL> full_path = os . path . join ( partial_path , "<STR_LIT>" ) <EOL> if not os . path . exists ( partial_path ) : <EOL> os . makedirs ( partial_path ) <EOL> if not os . path . exists ( full_path ) : <EOL> download ( url , full_path , progress_update_percentage = <NUM_LIT:1> ) <EOL> return full_path <EOL> def fetch_frey ( ) : <EOL> """<STR_LIT>""" <EOL> data_path = check_fetch_frey ( ) <EOL> matfile = loadmat ( data_path ) <EOL> all_data = ( matfile [ '<STR_LIT>' ] / <NUM_LIT> ) . T <EOL> return all_data <EOL> def check_fetch_mnist ( ) : <EOL> """<STR_LIT>""" <EOL> url = '<STR_LIT>' <EOL> partial_path = get_dataset_dir ( "<STR_LIT>" ) <EOL> full_path = os . path . join ( partial_path , "<STR_LIT>" ) <EOL> if not os . path . exists ( partial_path ) : <EOL> os . makedirs ( partial_path ) <EOL> if not os . path . exists ( full_path ) : <EOL> download ( url , full_path , progress_update_percentage = <NUM_LIT:1> ) <EOL> return full_path <EOL> def fetch_mnist ( ) : <EOL> """<STR_LIT>""" <EOL> data_path = check_fetch_mnist ( ) <EOL> f = gzip . open ( data_path , '<STR_LIT:rb>' ) <EOL> try : <EOL> train_set , valid_set , test_set = pickle . load ( f , encoding = "<STR_LIT>" ) <EOL> except TypeError : <EOL> train_set , valid_set , test_set = pickle . load ( f ) <EOL> f . close ( ) <EOL> return train_set , valid_set , test_set <EOL> def check_fetch_binarized_mnist ( ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> url = "<STR_LIT>" <EOL> partial_path = get_dataset_dir ( "<STR_LIT>" ) <EOL> fname = "<STR_LIT>" <EOL> full_path = os . path . join ( partial_path , fname ) <EOL> if not os . path . exists ( partial_path ) : <EOL> os . makedirs ( partial_path ) <EOL> if not os . path . exists ( full_path ) : <EOL> download ( url , full_path , progress_update_percentage = <NUM_LIT:1> ) <EOL> """<STR_LIT>""" <EOL> return partial_path <EOL> def fetch_binarized_mnist ( ) : <EOL> """<STR_LIT>""" <EOL> train_set , valid_set , test_set = fetch_mnist ( ) <EOL> train_X = train_set [ <NUM_LIT:0> ] <EOL> train_y = train_set [ <NUM_LIT:1> ] <EOL> valid_X = valid_set [ <NUM_LIT:0> ] <EOL> valid_y = valid_set [ <NUM_LIT:1> ] <EOL> test_X = test_set [ <NUM_LIT:0> ] <EOL> test_y = test_set [ <NUM_LIT:1> ] <EOL> random_state = np . random . RandomState ( <NUM_LIT> ) <EOL> def get_sampled ( arr ) : <EOL> return random_state . binomial ( <NUM_LIT:1> , arr * <NUM_LIT:255> / <NUM_LIT> , size = arr . shape ) <EOL> train_X = get_sampled ( train_X ) <EOL> valid_X = get_sampled ( valid_X ) <EOL> test_X = get_sampled ( test_X ) <EOL> train_set = ( train_X , train_y ) <EOL> valid_set = ( valid_X , valid_y ) <EOL> test_set = ( test_X , test_y ) <EOL> """<STR_LIT>""" <EOL> return train_set , valid_set , test_set <EOL> def make_gif ( arr , gif_name , plot_width , plot_height , list_text_per_frame = None , <EOL> list_text_per_frame_color = None , <EOL> delay = <NUM_LIT:1> , grayscale = False , <EOL> loop = False , turn_on_agg = True ) : <EOL> """<STR_LIT>""" <EOL> if turn_on_agg : <EOL> import matplotlib <EOL> matplotlib . use ( '<STR_LIT>' ) <EOL> import matplotlib . pyplot as plt <EOL> random_code = random . randrange ( <NUM_LIT:2> ** <NUM_LIT:32> ) <EOL> pre = str ( random_code ) <EOL> for n , arr_i in enumerate ( arr ) : <EOL> plt . matshow ( arr_i . reshape ( plot_width , plot_height ) , cmap = "<STR_LIT>" ) <EOL> plt . axis ( '<STR_LIT>' ) <EOL> if list_text_per_frame is not None : <EOL> text = list_text_per_frame [ n ] <EOL> if list_text_per_frame_color is not None : <EOL> color = list_text_per_frame_color [ n ] <EOL> else : <EOL> color = "<STR_LIT>" <EOL> plt . text ( <NUM_LIT:0> , plot_height , text , color = color , <EOL> fontsize = <NUM_LIT:2> * plot_height ) <EOL> plotpath = '<STR_LIT>' % ( str ( n ) . zfill ( len ( <EOL> str ( len ( arr ) ) ) + <NUM_LIT:1> ) , pre ) <EOL> plt . savefig ( plotpath ) <EOL> plt . close ( ) <EOL> assert delay >= <NUM_LIT:1> <EOL> gif_delay = int ( delay ) <EOL> basestr = "<STR_LIT>" % ( pre , str ( gif_delay ) ) <EOL> if loop : <EOL> basestr += "<STR_LIT>" <EOL> else : <EOL> basestr += "<STR_LIT>" <EOL> if grayscale : <EOL> basestr += "<STR_LIT>" <EOL> basestr += "<STR_LIT>" % ( str ( int ( <NUM_LIT:5> * plot_width ) ) , <EOL> str ( int ( <NUM_LIT:5> * plot_height ) ) ) <EOL> basestr += gif_name <EOL> print ( "<STR_LIT>" ) <EOL> print ( basestr ) <EOL> subprocess . call ( basestr , shell = True ) <EOL> filelist = glob . glob ( "<STR_LIT>" % pre ) <EOL> for f in filelist : <EOL> os . remove ( f ) <EOL> def concatenate ( tensor_list , name , axis = <NUM_LIT:0> , force_cast_to_float = True ) : <EOL> """<STR_LIT>""" <EOL> if force_cast_to_float : <EOL> tensor_list = cast_to_float ( tensor_list ) <EOL> out = tensor . concatenate ( tensor_list , axis = axis ) <EOL> conc_dim = int ( sum ( [ calc_expected_dim ( inp ) <EOL> for inp in tensor_list ] ) ) <EOL> shape = list ( expression_shape ( tensor_list [ <NUM_LIT:0> ] ) ) <EOL> shape [ axis ] = conc_dim <EOL> new_shape = tuple ( shape ) <EOL> tag_expression ( out , name , new_shape ) <EOL> return out <EOL> def theano_repeat ( arr , n_repeat , stretch = False ) : <EOL> """<STR_LIT>""" <EOL> if arr . dtype not in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> arr = tensor . cast ( arr , "<STR_LIT>" ) <EOL> if stretch : <EOL> arg1 = arr . dimshuffle ( ( <NUM_LIT:0> , '<STR_LIT:x>' , <NUM_LIT:1> ) ) <EOL> arg2 = tensor . alloc ( <NUM_LIT:1.> , <NUM_LIT:1> , n_repeat , arr . shape [ <NUM_LIT:1> ] ) <EOL> arg2 = tensor . cast ( arg2 , arr . dtype ) <EOL> cloned = ( arg1 * arg2 ) . reshape ( ( n_repeat * arr . shape [ <NUM_LIT:0> ] , arr . shape [ <NUM_LIT:1> ] ) ) <EOL> else : <EOL> arg1 = arr . dimshuffle ( ( '<STR_LIT:x>' , <NUM_LIT:0> , <NUM_LIT:1> ) ) <EOL> arg2 = tensor . alloc ( <NUM_LIT:1.> , n_repeat , <NUM_LIT:1> , arr . shape [ <NUM_LIT:1> ] ) <EOL> arg2 = tensor . cast ( arg2 , arr . dtype ) <EOL> cloned = ( arg1 * arg2 ) . reshape ( ( n_repeat * arr . shape [ <NUM_LIT:0> ] , arr . shape [ <NUM_LIT:1> ] ) ) <EOL> shape = expression_shape ( arr ) <EOL> name = expression_name ( arr ) <EOL> tag_expression ( cloned , name + "<STR_LIT>" , ( shape [ <NUM_LIT:0> ] , shape [ <NUM_LIT:1> ] ) ) <EOL> return cloned <EOL> def cast_to_float ( list_of_inputs ) : <EOL> """<STR_LIT>""" <EOL> input_names = [ inp . name for inp in list_of_inputs ] <EOL> cast_inputs = [ tensor . cast ( inp , theano . config . floatX ) <EOL> for inp in list_of_inputs ] <EOL> for n , inp in enumerate ( cast_inputs ) : <EOL> cast_inputs [ n ] . name = input_names [ n ] <EOL> return cast_inputs <EOL> def interpolate_between_points ( arr , n_steps = <NUM_LIT:50> ) : <EOL> """<STR_LIT>""" <EOL> assert len ( arr ) > <NUM_LIT:2> <EOL> assert n_steps > <NUM_LIT:1> <EOL> path = [ path_between_points ( start , stop , n_steps = n_steps ) <EOL> for start , stop in zip ( arr [ : - <NUM_LIT:1> ] , arr [ <NUM_LIT:1> : ] ) ] <EOL> path = np . vstack ( path ) <EOL> return path <EOL> def path_between_points ( start , stop , n_steps = <NUM_LIT:100> , dtype = theano . config . floatX ) : <EOL> """<STR_LIT>""" <EOL> assert n_steps > <NUM_LIT:1> <EOL> step_vector = <NUM_LIT:1.> / ( n_steps - <NUM_LIT:1> ) * ( stop - start ) <EOL> steps = np . arange ( <NUM_LIT:0> , n_steps ) [ : , None ] * np . ones ( ( n_steps , len ( stop ) ) ) <EOL> steps = steps * step_vector + start <EOL> return steps . astype ( dtype ) <EOL> def minibatch_indices ( itr , minibatch_size ) : <EOL> """<STR_LIT>""" <EOL> is_three_d = False <EOL> if type ( itr ) is np . ndarray : <EOL> if len ( itr . shape ) == <NUM_LIT:3> : <EOL> is_three_d = True <EOL> elif not isinstance ( itr [ <NUM_LIT:0> ] , numbers . Real ) : <EOL> is_three_d = True <EOL> if is_three_d : <EOL> if type ( itr ) is np . ndarray : <EOL> minibatch_indices = np . arange ( <NUM_LIT:0> , itr . shape [ <NUM_LIT:1> ] , minibatch_size ) <EOL> else : <EOL> minibatch_indices = np . arange ( <NUM_LIT:0> , len ( itr ) , minibatch_size ) <EOL> minibatch_indices = np . asarray ( list ( minibatch_indices ) + [ len ( itr ) ] ) <EOL> start_indices = minibatch_indices [ : - <NUM_LIT:1> ] <EOL> end_indices = minibatch_indices [ <NUM_LIT:1> : ] <EOL> return zip ( start_indices , end_indices ) <EOL> else : <EOL> minibatch_indices = np . arange ( <NUM_LIT:0> , len ( itr ) , minibatch_size ) <EOL> minibatch_indices = np . asarray ( list ( minibatch_indices ) + [ len ( itr ) ] ) <EOL> start_indices = minibatch_indices [ : - <NUM_LIT:1> ] <EOL> end_indices = minibatch_indices [ <NUM_LIT:1> : ] <EOL> return zip ( start_indices , end_indices ) <EOL> def convert_to_one_hot ( itr , n_classes , dtype = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> is_three_d = False <EOL> if type ( itr ) is np . ndarray : <EOL> if len ( itr . shape ) == <NUM_LIT:3> : <EOL> is_three_d = True <EOL> elif not isinstance ( itr [ <NUM_LIT:0> ] , numbers . Real ) : <EOL> is_three_d = True <EOL> if is_three_d : <EOL> lengths = [ len ( i ) for i in itr ] <EOL> one_hot = np . zeros ( ( max ( lengths ) , len ( itr ) , n_classes ) , dtype = dtype ) <EOL> for n in range ( len ( itr ) ) : <EOL> one_hot [ np . arange ( lengths [ n ] ) , n , itr [ n ] ] = <NUM_LIT:1> <EOL> else : <EOL> one_hot = np . zeros ( ( len ( itr ) , n_classes ) , dtype = dtype ) <EOL> one_hot [ np . arange ( len ( itr ) ) , itr ] = <NUM_LIT:1> <EOL> return one_hot <EOL> def save_checkpoint ( save_path , items_dict ) : <EOL> """<STR_LIT>""" <EOL> old_recursion_limit = sys . getrecursionlimit ( ) <EOL> sys . setrecursionlimit ( <NUM_LIT> ) <EOL> with open ( save_path , mode = "<STR_LIT:wb>" ) as f : <EOL> pickle . dump ( items_dict , f ) <EOL> sys . setrecursionlimit ( old_recursion_limit ) <EOL> def load_checkpoint ( save_path ) : <EOL> """<STR_LIT>""" <EOL> old_recursion_limit = sys . getrecursionlimit ( ) <EOL> sys . setrecursionlimit ( <NUM_LIT> ) <EOL> with open ( save_path , mode = "<STR_LIT:rb>" ) as f : <EOL> items_dict = pickle . load ( f ) <EOL> sys . setrecursionlimit ( old_recursion_limit ) <EOL> return items_dict <EOL> def print_status_func ( epoch_results ) : <EOL> """<STR_LIT>""" <EOL> n_epochs_seen = max ( [ len ( l ) for l in epoch_results . values ( ) ] ) <EOL> last_results = { k : v [ - <NUM_LIT:1> ] for k , v in epoch_results . items ( ) } <EOL> print ( "<STR_LIT>" % ( n_epochs_seen , last_results ) ) <EOL> def checkpoint_status_func ( save_path , checkpoint_dict , epoch_results ) : <EOL> """<STR_LIT>""" <EOL> checkpoint_dict [ "<STR_LIT>" ] = epoch_results <EOL> save_checkpoint ( save_path , checkpoint_dict ) <EOL> print_status_func ( epoch_results ) <EOL> def early_stopping_status_func ( valid_cost , save_path , checkpoint_dict , <EOL> epoch_results ) : <EOL> """<STR_LIT>""" <EOL> old = min ( epoch_results [ "<STR_LIT>" ] + [ np . inf ] ) <EOL> epoch_results [ "<STR_LIT>" ] . append ( valid_cost ) <EOL> new = min ( epoch_results [ "<STR_LIT>" ] ) <EOL> if new < old : <EOL> print ( "<STR_LIT>" ) <EOL> checkpoint_status_func ( save_path , checkpoint_dict , epoch_results ) <EOL> else : <EOL> print_status_func ( epoch_results ) <EOL> def even_slice ( arr , size ) : <EOL> """<STR_LIT>""" <EOL> extent = - ( len ( arr ) % size ) <EOL> if extent == <NUM_LIT:0> : <EOL> extent = None <EOL> return arr [ : extent ] <EOL> def make_minibatch ( arg , start , stop ) : <EOL> """<STR_LIT>""" <EOL> if len ( arg . shape ) == <NUM_LIT:3> : <EOL> return [ arg [ : , start : stop ] ] <EOL> else : <EOL> return [ arg [ start : stop ] ] <EOL> def gen_text_minibatch_func ( one_hot_size ) : <EOL> """<STR_LIT>""" <EOL> def apply ( arg , start , stop ) : <EOL> sli = arg [ start : stop ] <EOL> expanded = convert_to_one_hot ( sli , one_hot_size ) <EOL> lengths = [ len ( s ) for s in sli ] <EOL> mask = np . zeros ( ( max ( lengths ) , len ( sli ) ) , dtype = theano . config . floatX ) <EOL> for n , l in enumerate ( lengths ) : <EOL> mask [ np . arange ( l ) , n ] = <NUM_LIT:1.> <EOL> return expanded , mask <EOL> return apply <EOL> def iterate_function ( func , list_of_minibatch_args , minibatch_size , <EOL> list_of_non_minibatch_args = None , <EOL> list_of_minibatch_functions = [ make_minibatch ] , <EOL> list_of_output_names = None , <EOL> n_epochs = <NUM_LIT:1000> , n_status = <NUM_LIT:50> , status_func = None , <EOL> previous_epoch_results = None , <EOL> shuffle = False , random_state = None ) : <EOL> """<STR_LIT>""" <EOL> if previous_epoch_results is None : <EOL> epoch_results = defaultdict ( list ) <EOL> else : <EOL> epoch_results = previous_epoch_results <EOL> if shuffle : <EOL> assert random_state is not None <EOL> status_points = list ( range ( n_epochs ) ) <EOL> if len ( status_points ) >= n_status : <EOL> intermediate_points = status_points [ : : n_epochs // n_status ] <EOL> status_points = intermediate_points + [ status_points [ - <NUM_LIT:1> ] ] <EOL> else : <EOL> status_points = range ( len ( status_points ) ) <EOL> for arg in list_of_minibatch_args : <EOL> assert len ( arg ) == len ( list_of_minibatch_args [ <NUM_LIT:0> ] ) <EOL> indices = minibatch_indices ( list_of_minibatch_args [ <NUM_LIT:0> ] , minibatch_size ) <EOL> if len ( list_of_minibatch_args [ <NUM_LIT:0> ] ) % minibatch_size != <NUM_LIT:0> : <EOL> print ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if len ( list_of_minibatch_functions ) == <NUM_LIT:1> : <EOL> list_of_minibatch_functions = list_of_minibatch_functions * len ( <EOL> list_of_minibatch_args ) <EOL> else : <EOL> assert len ( list_of_minibatch_functions ) == len ( list_of_minibatch_args ) <EOL> for e in range ( n_epochs ) : <EOL> results = defaultdict ( list ) <EOL> if shuffle : <EOL> random_state . shuffle ( indices ) <EOL> for i , j in indices : <EOL> minibatch_args = [ ] <EOL> for n , arg in enumerate ( list_of_minibatch_args ) : <EOL> minibatch_args += list_of_minibatch_functions [ n ] ( arg , i , j ) <EOL> if list_of_non_minibatch_args is not None : <EOL> all_args = minibatch_args + list_of_non_minibatch_args <EOL> else : <EOL> all_args = minibatch_args <EOL> minibatch_results = func ( * all_args ) <EOL> if type ( minibatch_results ) is not list : <EOL> minibatch_results = [ minibatch_results ] <EOL> for n , k in enumerate ( minibatch_results ) : <EOL> if list_of_output_names is not None : <EOL> assert len ( list_of_output_names ) == len ( minibatch_results ) <EOL> results [ list_of_output_names [ n ] ] . append ( <EOL> minibatch_results [ n ] ) <EOL> else : <EOL> results [ n ] . append ( minibatch_results [ n ] ) <EOL> avg_output = { r : np . mean ( results [ r ] ) for r in results . keys ( ) } <EOL> for k in avg_output . keys ( ) : <EOL> epoch_results [ k ] . append ( avg_output [ k ] ) <EOL> if e in status_points : <EOL> if status_func is not None : <EOL> epoch_number = e <EOL> status_number = np . searchsorted ( status_points , e ) <EOL> status_func ( status_number , epoch_number , epoch_results ) <EOL> return epoch_results <EOL> def as_shared ( arr , name = None ) : <EOL> """<STR_LIT>""" <EOL> if name is not None : <EOL> return theano . shared ( value = arr , borrow = True ) <EOL> else : <EOL> return theano . shared ( value = arr , name = name , borrow = True ) <EOL> def np_zeros ( shape ) : <EOL> """<STR_LIT>""" <EOL> return np . zeros ( shape ) . astype ( theano . config . floatX ) <EOL> def np_rand ( shape , random_state ) : <EOL> return random_state . uniform ( low = - <NUM_LIT> , high = <NUM_LIT> , size = shape ) . astype ( <EOL> theano . config . floatX ) <EOL> def np_randn ( shape , random_state ) : <EOL> """<STR_LIT>""" <EOL> return ( <NUM_LIT> * random_state . randn ( * shape ) ) . astype ( theano . config . floatX ) <EOL> def np_tanh_fan ( shape , random_state ) : <EOL> bound = np . sqrt ( <NUM_LIT> / np . sum ( shape ) ) <EOL> return random_state . uniform ( low = - bound , high = bound , <EOL> size = shape ) . astype ( theano . config . floatX ) <EOL> def np_sigmoid_fan ( shape , random_state ) : <EOL> return <NUM_LIT:4> * np_tanh_fan ( shape , random_state ) <EOL> def np_ortho ( shape , random_state ) : <EOL> """<STR_LIT>""" <EOL> g = random_state . randn ( * shape ) <EOL> o_g = linalg . svd ( g ) [ <NUM_LIT:0> ] <EOL> return o_g . astype ( theano . config . floatX ) <EOL> def names_in_graph ( list_of_names , graph ) : <EOL> """<STR_LIT>""" <EOL> return all ( [ name in graph . keys ( ) for name in list_of_names ] ) <EOL> def add_arrays_to_graph ( list_of_arrays , list_of_names , graph , strict = True ) : <EOL> assert len ( list_of_arrays ) == len ( list_of_names ) <EOL> arrays_added = [ ] <EOL> for array , name in zip ( list_of_arrays , list_of_names ) : <EOL> if name in graph . keys ( ) and strict : <EOL> raise ValueError ( "<STR_LIT>" % name ) <EOL> shared_array = as_shared ( array , name = name ) <EOL> graph [ name ] = shared_array <EOL> arrays_added . append ( shared_array ) <EOL> def make_shapename ( name , shape ) : <EOL> if len ( shape ) == <NUM_LIT:1> : <EOL> return name + "<STR_LIT>" + str ( shape [ <NUM_LIT:0> ] ) + "<STR_LIT:x>" <EOL> else : <EOL> return name + "<STR_LIT>" + "<STR_LIT:x>" . join ( map ( str , list ( shape ) ) ) <EOL> def parse_shapename ( shapename ) : <EOL> try : <EOL> shape = shapename . split ( "<STR_LIT>" ) [ <NUM_LIT:1> ] . split ( "<STR_LIT:[>" ) [ <NUM_LIT:0> ] . split ( "<STR_LIT:x>" ) <EOL> except AttributeError : <EOL> raise AttributeError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % shapename ) <EOL> if "<STR_LIT:[>" in shapename . split ( "<STR_LIT>" ) [ <NUM_LIT:1> ] : <EOL> shape = shape [ <NUM_LIT:1> : ] <EOL> name = shapename . split ( "<STR_LIT>" ) [ <NUM_LIT:0> ] <EOL> shape = tuple ( [ int ( s ) for s in shape if s != '<STR_LIT>' ] ) <EOL> return name , shape <EOL> def add_datasets_to_graph ( list_of_datasets , list_of_names , graph , strict = True , <EOL> list_of_test_values = None ) : <EOL> assert len ( list_of_datasets ) == len ( list_of_names ) <EOL> datasets_added = [ ] <EOL> for n , ( dataset , name ) in enumerate ( zip ( list_of_datasets , list_of_names ) ) : <EOL> if dataset . dtype != "<STR_LIT>" : <EOL> if len ( dataset . shape ) == <NUM_LIT:1> : <EOL> sym = tensor . vector ( ) <EOL> elif len ( dataset . shape ) == <NUM_LIT:2> : <EOL> sym = tensor . matrix ( ) <EOL> elif len ( dataset . shape ) == <NUM_LIT:3> : <EOL> sym = tensor . tensor3 ( ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % name ) <EOL> elif dataset . dtype == "<STR_LIT>" : <EOL> if len ( dataset . shape ) == <NUM_LIT:1> : <EOL> sym = tensor . ivector ( ) <EOL> elif len ( dataset . shape ) == <NUM_LIT:2> : <EOL> sym = tensor . imatrix ( ) <EOL> elif len ( dataset . shape ) == <NUM_LIT:3> : <EOL> sym = tensor . itensor3 ( ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % name ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % ( <EOL> name , dataset . dtype ) ) <EOL> if list_of_test_values is not None : <EOL> sym . tag . test_value = list_of_test_values [ n ] <EOL> tag_expression ( sym , name , dataset . shape ) <EOL> datasets_added . append ( sym ) <EOL> graph [ "<STR_LIT>" ] = datasets_added <EOL> return datasets_added <EOL> def tag_expression ( expression , name , shape ) : <EOL> expression . name = make_shapename ( name , shape ) <EOL> def expression_name ( expression ) : <EOL> return parse_shapename ( expression . name ) [ <NUM_LIT:0> ] <EOL> def expression_shape ( expression ) : <EOL> return parse_shapename ( expression . name ) [ <NUM_LIT:1> ] <EOL> def calc_expected_dim ( expression ) : <EOL> shape = expression_shape ( expression ) <EOL> dim = shape [ - <NUM_LIT:1> ] <EOL> return dim <EOL> def fetch_from_graph ( list_of_names , graph ) : <EOL> """<STR_LIT>""" <EOL> if "<STR_LIT>" not in graph . keys ( ) : <EOL> raise AttributeError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> return [ graph [ name ] for name in list_of_names ] <EOL> def get_params_and_grads ( graph , cost ) : <EOL> grads = [ ] <EOL> params = [ ] <EOL> for k , p in graph . items ( ) : <EOL> if k [ : <NUM_LIT:2> ] == "<STR_LIT>" : <EOL> continue <EOL> print ( "<STR_LIT>" % k ) <EOL> grad = tensor . grad ( cost , p ) <EOL> params . append ( p ) <EOL> grads . append ( grad ) <EOL> return params , grads <EOL> def binary_crossentropy_nll ( predicted_values , true_values ) : <EOL> """<STR_LIT>""" <EOL> return ( - true_values * tensor . log ( predicted_values ) - ( <EOL> <NUM_LIT:1> - true_values ) * tensor . log ( <NUM_LIT:1> - predicted_values ) ) . sum ( axis = - <NUM_LIT:1> ) <EOL> def binary_entropy ( values ) : <EOL> return ( - values * tensor . log ( values ) ) . sum ( axis = - <NUM_LIT:1> ) <EOL> def categorical_crossentropy_nll ( predicted_values , true_values ) : <EOL> """<STR_LIT>""" <EOL> indices = tensor . argmax ( true_values , axis = - <NUM_LIT:1> ) <EOL> rows = tensor . arange ( true_values . shape [ <NUM_LIT:0> ] ) <EOL> if predicted_values . ndim < <NUM_LIT:3> : <EOL> return - tensor . log ( predicted_values ) [ rows , indices ] <EOL> elif predicted_values . ndim == <NUM_LIT:3> : <EOL> d0 = true_values . shape [ <NUM_LIT:0> ] <EOL> d1 = true_values . shape [ <NUM_LIT:1> ] <EOL> pred = predicted_values . reshape ( ( d0 * d1 , - <NUM_LIT:1> ) ) <EOL> ind = indices . reshape ( ( d0 * d1 , ) ) <EOL> s = tensor . arange ( pred . shape [ <NUM_LIT:0> ] ) <EOL> correct = - tensor . log ( pred ) [ s , ind ] <EOL> return correct . reshape ( ( d0 , d1 , ) ) <EOL> else : <EOL> raise AttributeError ( "<STR_LIT>" ) <EOL> def abs_error_nll ( predicted_values , true_values ) : <EOL> return tensor . abs_ ( predicted_values - true_values ) . sum ( axis = - <NUM_LIT:1> ) <EOL> def squared_error_nll ( predicted_values , true_values ) : <EOL> return tensor . sqr ( predicted_values - true_values ) . sum ( axis = - <NUM_LIT:1> ) <EOL> def gaussian_error_nll ( mu_values , sigma_values , true_values ) : <EOL> """<STR_LIT>""" <EOL> nll = <NUM_LIT:0.5> * ( mu_values - true_values ) ** <NUM_LIT:2> / sigma_values ** <NUM_LIT:2> + tensor . log ( <EOL> <NUM_LIT:2> * np . pi * sigma_values ** <NUM_LIT:2> ) <EOL> return nll <EOL> def log_gaussian_error_nll ( mu_values , log_sigma_values , true_values ) : <EOL> """<STR_LIT>""" <EOL> nll = <NUM_LIT:0.5> * ( mu_values - true_values ) ** <NUM_LIT:2> / tensor . exp ( <EOL> log_sigma_values ) ** <NUM_LIT:2> + tensor . log ( <NUM_LIT:2> * np . pi ) + <NUM_LIT:2> * log_sigma_values <EOL> return nll <EOL> def masked_cost ( cost , mask ) : <EOL> return cost * mask <EOL> def softplus ( X ) : <EOL> return tensor . nnet . softplus ( X ) + <NUM_LIT> <EOL> def relu ( X ) : <EOL> return X * ( X > <NUM_LIT:1> ) <EOL> def linear ( X ) : <EOL> return X <EOL> def softmax ( X ) : <EOL> e_X = tensor . exp ( X - X . max ( axis = - <NUM_LIT:1> , keepdims = True ) ) <EOL> out = e_X / e_X . sum ( axis = - <NUM_LIT:1> , keepdims = True ) <EOL> return out <EOL> def dropout ( X , random_state , on_off_switch , p = <NUM_LIT:0.> ) : <EOL> if p > <NUM_LIT:0> : <EOL> theano_seed = random_state . randint ( - <NUM_LIT> , <NUM_LIT> ) <EOL> if theano_seed == <NUM_LIT:0> : <EOL> print ( "<STR_LIT>" ) <EOL> theano_seed = random_state . randint ( - <NUM_LIT:2> ** <NUM_LIT:32> , <NUM_LIT:2> ** <NUM_LIT:32> ) <EOL> theano_rng = MRG_RandomStreams ( seed = theano_seed ) <EOL> retain_prob = <NUM_LIT:1> - p <EOL> if X . ndim == <NUM_LIT:2> : <EOL> X *= theano_rng . binomial ( <EOL> X . shape , p = retain_prob , <EOL> dtype = theano . config . floatX ) ** on_off_switch <EOL> X /= retain_prob <EOL> elif X . ndim == <NUM_LIT:3> : <EOL> X *= theano_rng . binomial ( ( <EOL> X . shape [ <NUM_LIT:1> ] , X . shape [ <NUM_LIT:2> ] ) , p = retain_prob , <EOL> dtype = theano . config . floatX ) ** on_off_switch <EOL> X /= retain_prob <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % str ( X . ndim ) ) <EOL> return X <EOL> def dropout_layer ( list_of_inputs , name , on_off_switch , dropout_prob = <NUM_LIT:0.5> , <EOL> random_state = None ) : <EOL> theano_seed = random_state . randint ( - <NUM_LIT> , <NUM_LIT> ) <EOL> if theano_seed == <NUM_LIT:0> : <EOL> print ( "<STR_LIT>" ) <EOL> theano_seed = random_state . randint ( - <NUM_LIT:2> ** <NUM_LIT:32> , <NUM_LIT:2> ** <NUM_LIT:32> ) <EOL> conc_input = concatenate ( list_of_inputs , name , axis = - <NUM_LIT:1> ) <EOL> shape = expression_shape ( conc_input ) <EOL> dropped = dropout ( conc_input , random_state , on_off_switch , p = dropout_prob ) <EOL> tag_expression ( dropped , name , shape ) <EOL> return dropped <EOL> def projection_layer ( list_of_inputs , graph , name , proj_dim = None , <EOL> random_state = None , strict = True , init_func = np_tanh_fan , <EOL> func = linear ) : <EOL> W_name = name + '<STR_LIT>' <EOL> b_name = name + '<STR_LIT>' <EOL> list_of_names = [ W_name , b_name ] <EOL> if not names_in_graph ( list_of_names , graph ) : <EOL> assert proj_dim is not None <EOL> assert random_state is not None <EOL> conc_input_dim = int ( sum ( [ calc_expected_dim ( inp ) <EOL> for inp in list_of_inputs ] ) ) <EOL> np_W = init_func ( ( conc_input_dim , proj_dim ) , random_state ) <EOL> np_b = np_zeros ( ( proj_dim , ) ) <EOL> add_arrays_to_graph ( [ np_W , np_b ] , list_of_names , graph , <EOL> strict = strict ) <EOL> else : <EOL> if strict : <EOL> raise AttributeError ( <EOL> "<STR_LIT>" % name ) <EOL> W , b = fetch_from_graph ( list_of_names , graph ) <EOL> conc_input = concatenate ( list_of_inputs , name , axis = - <NUM_LIT:1> ) <EOL> output = tensor . dot ( conc_input , W ) + b <EOL> if func is not None : <EOL> final = func ( output ) <EOL> else : <EOL> final = output <EOL> shape = list ( expression_shape ( conc_input ) ) <EOL> shape [ - <NUM_LIT:1> ] = proj_dim <EOL> new_shape = tuple ( shape ) <EOL> tag_expression ( final , name , new_shape ) <EOL> return final <EOL> def linear_layer ( list_of_inputs , graph , name , proj_dim = None , random_state = None , <EOL> strict = True , init_func = np_tanh_fan ) : <EOL> return projection_layer ( <EOL> list_of_inputs = list_of_inputs , graph = graph , name = name , <EOL> proj_dim = proj_dim , random_state = random_state , <EOL> strict = strict , init_func = init_func , func = linear ) <EOL> def sigmoid_layer ( list_of_inputs , graph , name , proj_dim = None , random_state = None , <EOL> strict = True , init_func = np_sigmoid_fan ) : <EOL> return projection_layer ( <EOL> list_of_inputs = list_of_inputs , graph = graph , name = name , <EOL> proj_dim = proj_dim , random_state = random_state , <EOL> strict = strict , init_func = init_func , func = tensor . nnet . sigmoid ) <EOL> def tanh_layer ( list_of_inputs , graph , name , proj_dim = None , random_state = None , <EOL> strict = True , init_func = np_tanh_fan ) : <EOL> return projection_layer ( <EOL> list_of_inputs = list_of_inputs , graph = graph , name = name , <EOL> proj_dim = proj_dim , random_state = random_state , <EOL> strict = strict , init_func = init_func , func = tensor . tanh ) <EOL> def softplus_layer ( list_of_inputs , graph , name , proj_dim = None , <EOL> random_state = None , strict = True , <EOL> init_func = np_tanh_fan ) : <EOL> return projection_layer ( <EOL> list_of_inputs = list_of_inputs , graph = graph , name = name , <EOL> proj_dim = proj_dim , random_state = random_state , <EOL> strict = strict , init_func = init_func , func = softplus ) <EOL> def exp_layer ( list_of_inputs , graph , name , proj_dim = None , random_state = None , <EOL> strict = True , init_func = np_tanh_fan ) : <EOL> return projection_layer ( <EOL> list_of_inputs = list_of_inputs , graph = graph , name = name , <EOL> proj_dim = proj_dim , random_state = random_state , <EOL> strict = strict , init_func = init_func , func = tensor . exp ) <EOL> def relu_layer ( list_of_inputs , graph , name , proj_dim = None , random_state = None , <EOL> strict = True , init_func = np_tanh_fan ) : <EOL> return projection_layer ( <EOL> list_of_inputs = list_of_inputs , graph = graph , name = name , <EOL> proj_dim = proj_dim , random_state = random_state , <EOL> strict = strict , init_func = init_func , func = relu ) <EOL> def softmax_layer ( list_of_inputs , graph , name , proj_dim = None , random_state = None , <EOL> strict = True , init_func = np_tanh_fan ) : <EOL> return projection_layer ( <EOL> list_of_inputs = list_of_inputs , graph = graph , name = name , <EOL> proj_dim = proj_dim , random_state = random_state , <EOL> strict = strict , init_func = init_func , func = softmax ) <EOL> def softmax_sample_layer ( list_of_multinomial_inputs , name , random_state = None ) : <EOL> theano_seed = random_state . randint ( - <NUM_LIT> , <NUM_LIT> ) <EOL> if theano_seed == <NUM_LIT:0> : <EOL> print ( "<STR_LIT>" ) <EOL> theano_seed = random_state . randint ( - <NUM_LIT:2> ** <NUM_LIT:32> , <NUM_LIT:2> ** <NUM_LIT:32> ) <EOL> theano_rng = MRG_RandomStreams ( seed = theano_seed ) <EOL> conc_multinomial = concatenate ( list_of_multinomial_inputs , name , axis = <NUM_LIT:1> ) <EOL> shape = expression_shape ( conc_multinomial ) <EOL> conc_multinomial /= len ( list_of_multinomial_inputs ) <EOL> tag_expression ( conc_multinomial , name , shape ) <EOL> samp = theano_rng . multinomial ( pvals = conc_multinomial , <EOL> dtype = "<STR_LIT>" ) <EOL> tag_expression ( samp , name , ( shape [ <NUM_LIT:0> ] , shape [ <NUM_LIT:1> ] ) ) <EOL> return samp <EOL> def gaussian_sample_layer ( list_of_mu_inputs , list_of_sigma_inputs , <EOL> name , random_state = None ) : <EOL> theano_seed = random_state . randint ( - <NUM_LIT> , <NUM_LIT> ) <EOL> if theano_seed == <NUM_LIT:0> : <EOL> print ( "<STR_LIT>" ) <EOL> theano_seed = random_state . randint ( - <NUM_LIT:2> ** <NUM_LIT:32> , <NUM_LIT:2> ** <NUM_LIT:32> ) <EOL> theano_rng = MRG_RandomStreams ( seed = theano_seed ) <EOL> conc_mu = concatenate ( list_of_mu_inputs , name , axis = <NUM_LIT:1> ) <EOL> conc_sigma = concatenate ( list_of_sigma_inputs , name , axis = <NUM_LIT:1> ) <EOL> e = theano_rng . normal ( size = ( conc_sigma . shape [ <NUM_LIT:0> ] , <EOL> conc_sigma . shape [ <NUM_LIT:1> ] ) , <EOL> dtype = conc_sigma . dtype ) <EOL> samp = conc_mu + conc_sigma * e <EOL> shape = expression_shape ( conc_sigma ) <EOL> tag_expression ( samp , name , shape ) <EOL> return samp <EOL> def gaussian_log_sample_layer ( list_of_mu_inputs , list_of_log_sigma_inputs , <EOL> name , random_state = None ) : <EOL> """<STR_LIT>""" <EOL> theano_seed = random_state . randint ( - <NUM_LIT> , <NUM_LIT> ) <EOL> if theano_seed == <NUM_LIT:0> : <EOL> print ( "<STR_LIT>" ) <EOL> theano_seed = random_state . randint ( - <NUM_LIT:2> ** <NUM_LIT:32> , <NUM_LIT:2> ** <NUM_LIT:32> ) <EOL> theano_rng = MRG_RandomStreams ( seed = theano_seed ) <EOL> conc_mu = concatenate ( list_of_mu_inputs , name , axis = <NUM_LIT:1> ) <EOL> conc_log_sigma = concatenate ( list_of_log_sigma_inputs , name , axis = <NUM_LIT:1> ) <EOL> e = theano_rng . normal ( size = ( conc_log_sigma . shape [ <NUM_LIT:0> ] , <EOL> conc_log_sigma . shape [ <NUM_LIT:1> ] ) , <EOL> dtype = conc_log_sigma . dtype ) <EOL> samp = conc_mu + tensor . exp ( <NUM_LIT:0.5> * conc_log_sigma ) * e <EOL> shape = expression_shape ( conc_log_sigma ) <EOL> tag_expression ( samp , name , shape ) <EOL> return samp <EOL> def gaussian_kl ( list_of_mu_inputs , list_of_sigma_inputs , name ) : <EOL> conc_mu = concatenate ( list_of_mu_inputs , name ) <EOL> conc_sigma = concatenate ( list_of_sigma_inputs , name ) <EOL> kl = <NUM_LIT:0.5> * tensor . sum ( - <NUM_LIT:2> * tensor . log ( conc_sigma ) + conc_mu ** <NUM_LIT:2> <EOL> + conc_sigma ** <NUM_LIT:2> - <NUM_LIT:1> , axis = <NUM_LIT:1> ) <EOL> return kl <EOL> def gaussian_log_kl ( list_of_mu_inputs , list_of_log_sigma_inputs , name ) : <EOL> """<STR_LIT>""" <EOL> conc_mu = concatenate ( list_of_mu_inputs , name ) <EOL> conc_log_sigma = <NUM_LIT:0.5> * concatenate ( list_of_log_sigma_inputs , name ) <EOL> kl = <NUM_LIT:0.5> * tensor . sum ( - <NUM_LIT:2> * conc_log_sigma + conc_mu ** <NUM_LIT:2> <EOL> + tensor . exp ( conc_log_sigma ) ** <NUM_LIT:2> - <NUM_LIT:1> , axis = <NUM_LIT:1> ) <EOL> return kl <EOL> def switch_wrap ( switch_func , if_true_var , if_false_var , name ) : <EOL> switched = tensor . switch ( switch_func , if_true_var , if_false_var ) <EOL> shape = expression_shape ( if_true_var ) <EOL> assert shape == expression_shape ( if_false_var ) <EOL> tag_expression ( switched , name , shape ) <EOL> return switched </s>
