<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import , print_function , division <EOL> import logging , time , sys <EOL> import numpy <EOL> from six . moves import xrange <EOL> import theano <EOL> from theano . compile import shared , pfunc <EOL> from theano import tensor <EOL> from theano . tensor . nnet import softplus <EOL> from theano . tensor . nnet . nnet import softsign <EOL> _logger = logging . getLogger ( '<STR_LIT>' ) <EOL> def _shared_uniform ( rng , low , high , size , dtype , name = None ) : <EOL> return shared ( <EOL> theano . _asarray ( <EOL> rng . uniform ( low = low , high = high , size = size ) , <EOL> dtype = dtype ) , name ) <EOL> class Kouh2008 ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , w_list , x_list , p , q , r , k , params , updates , eps = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> if len ( w_list ) != len ( x_list ) : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> output = ( sum ( w * tensor . pow ( x , p ) for ( w , x ) in zip ( w_list , x_list ) ) ) / ( theano . _asarray ( eps , dtype = k . type . dtype ) + k + tensor . pow ( sum ( tensor . pow ( x , q ) for x in x_list ) , r ) ) <EOL> assert output . type . ndim == <NUM_LIT:2> <EOL> self . __dict__ . update ( locals ( ) ) <EOL> del self . __dict__ [ '<STR_LIT>' ] <EOL> _logger . debug ( '<STR_LIT>' % output . dtype ) <EOL> @ classmethod <EOL> def new_expbounds ( cls , rng , x_list , n_out , dtype = None , params = None , <EOL> updates = None , exponent_range = ( <NUM_LIT:1.0> , <NUM_LIT> ) ) : <EOL> """<STR_LIT:U+0020>""" <EOL> if params is None : <EOL> params = [ ] <EOL> if updates is None : <EOL> updates = [ ] <EOL> if dtype is None : <EOL> dtype = x_list [ <NUM_LIT:0> ] . dtype <EOL> n_terms = len ( x_list ) <EOL> def shared_uniform ( low , high , size , name ) : <EOL> return _shared_uniform ( rng , low , high , size , dtype , name ) <EOL> use_softmax_w = True <EOL> if use_softmax_w : <EOL> w = shared_uniform ( low = - <NUM_LIT> , high = <NUM_LIT> , size = ( n_out , n_terms ) , name = '<STR_LIT>' ) <EOL> w_sm = theano . tensor . nnet . softmax ( w ) <EOL> w_list = [ w_sm [ : , i ] for i in xrange ( n_terms ) ] <EOL> w_l1 = abs ( w ) . sum ( ) <EOL> w_l2_sqr = ( w ** <NUM_LIT:2> ) . sum ( ) <EOL> else : <EOL> w_list = [ shared_uniform ( low = - <NUM_LIT> / n_terms , high = <NUM_LIT> / n_terms , size = ( n_out , ) , name = '<STR_LIT>' % i ) <EOL> for i in xrange ( n_terms ) ] <EOL> w_l1 = sum ( abs ( wi ) . sum ( ) for wi in w_list ) <EOL> w_l2_sqr = sum ( ( wi ** <NUM_LIT:2> ) . sum ( ) for wi in w_list ) <EOL> e_range_low , e_range_high = exponent_range <EOL> e_range_low = theano . _asarray ( e_range_low , dtype = dtype ) <EOL> e_range_high = theano . _asarray ( e_range_high , dtype = dtype ) <EOL> e_range_mag = e_range_high - e_range_low <EOL> if e_range_mag < <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> p_unbounded = shared_uniform ( low = - <NUM_LIT:0.1> , high = <NUM_LIT:0.1> , size = ( n_out , ) , name = '<STR_LIT:p>' ) <EOL> q_unbounded = shared_uniform ( low = - <NUM_LIT:0.1> , high = <NUM_LIT:0.1> , size = ( n_out , ) , name = '<STR_LIT:q>' ) <EOL> r_unbounded = shared_uniform ( low = - <NUM_LIT:0.1> , high = <NUM_LIT:0.1> , size = ( n_out , ) , name = '<STR_LIT:r>' ) <EOL> k_unbounded = shared_uniform ( low = - <NUM_LIT> , high = <NUM_LIT> , size = ( n_out , ) , name = '<STR_LIT:k>' ) <EOL> p = tensor . nnet . sigmoid ( p_unbounded ) * e_range_mag + e_range_low <EOL> q = tensor . nnet . sigmoid ( q_unbounded ) * e_range_mag + e_range_low <EOL> r = tensor . nnet . sigmoid ( r_unbounded ) * theano . _asarray ( <NUM_LIT:1.0> / e_range_low - <NUM_LIT:1.0> / e_range_high , dtype = dtype ) + theano . _asarray ( <NUM_LIT:1.0> / e_range_high , dtype = dtype ) <EOL> k = softsign ( k_unbounded ) <EOL> if use_softmax_w : <EOL> rval = cls ( w_list , x_list , p , q , r , k , <EOL> params = [ p_unbounded , q_unbounded , r_unbounded , k_unbounded , w ] + params , <EOL> updates = updates ) <EOL> else : <EOL> rval = cls ( w_list , x_list , p , q , r , k , <EOL> params = [ p_unbounded , q_unbounded , r_unbounded , k_unbounded ] + w_list + params , <EOL> updates = updates ) <EOL> rval . p_unbounded = p_unbounded <EOL> rval . q_unbounded = q_unbounded <EOL> rval . r_unbounded = r_unbounded <EOL> rval . k_unbounded = k_unbounded <EOL> rval . exp_l1 = abs ( p_unbounded ) . sum ( ) + abs ( q_unbounded ) . sum ( ) + abs ( r_unbounded ) . sum ( ) <EOL> rval . exp_l2_sqr = ( p_unbounded ** <NUM_LIT:2> ) . sum ( ) + ( q_unbounded ** <NUM_LIT:2> ) . sum ( ) + ( r_unbounded ** <NUM_LIT:2> ) . sum ( ) <EOL> rval . w_l1 = w_l1 <EOL> rval . w_l2_sqr = w_l2_sqr <EOL> return rval <EOL> @ classmethod <EOL> def new_filters_expbounds ( cls , rng , input , n_in , n_out , n_terms , dtype = None , eps = <NUM_LIT> , <EOL> exponent_range = ( <NUM_LIT:1.0> , <NUM_LIT> ) , filter_range = <NUM_LIT:1.0> ) : <EOL> """<STR_LIT>""" <EOL> if input . type . ndim != <NUM_LIT:2> : <EOL> raise TypeError ( '<STR_LIT>' ) <EOL> if dtype is None : <EOL> dtype = input . dtype <EOL> _logger . debug ( '<STR_LIT>' % dtype ) <EOL> def shared_uniform ( low , high , size , name ) : <EOL> return _shared_uniform ( rng , low , high , size , dtype , name ) <EOL> f_list = [ shared_uniform ( low = - <NUM_LIT> / numpy . sqrt ( n_in ) , high = <NUM_LIT> / numpy . sqrt ( n_in ) , size = ( n_in , n_out ) , name = '<STR_LIT>' % i ) <EOL> for i in xrange ( n_terms ) ] <EOL> b_list = [ shared_uniform ( low = <NUM_LIT:0> , high = <NUM_LIT> , size = ( n_out , ) , name = '<STR_LIT>' % i ) <EOL> for i in xrange ( n_terms ) ] <EOL> filter_range = theano . _asarray ( filter_range , dtype = dtype ) <EOL> half_filter_range = theano . _asarray ( filter_range / <NUM_LIT:2> , dtype = dtype ) <EOL> x_list = [ theano . _asarray ( filter_range + eps , dtype = dtype ) + half_filter_range * softsign ( tensor . dot ( input , f_list [ i ] ) + <EOL> b_list [ i ] ) for i in xrange ( n_terms ) ] <EOL> rval = cls . new_expbounds ( rng , x_list , n_out , dtype = dtype , params = f_list + b_list , <EOL> exponent_range = exponent_range ) <EOL> rval . f_list = f_list <EOL> rval . input = input <EOL> rval . filter_l1 = sum ( abs ( fi ) . sum ( ) for fi in f_list ) <EOL> rval . filter_l2_sqr = sum ( ( fi ** <NUM_LIT:2> ) . sum ( ) for fi in f_list ) <EOL> return rval <EOL> def img_from_weights ( self , rows = None , cols = None , row_gap = <NUM_LIT:1> , col_gap = <NUM_LIT:1> , eps = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> n_in , n_out = self . f_list [ <NUM_LIT:0> ] . value . shape <EOL> if rows is None and cols is None : <EOL> rows = int ( numpy . sqrt ( n_out ) ) <EOL> if cols is None : <EOL> cols = n_out // rows <EOL> if n_out % rows : cols += <NUM_LIT:1> <EOL> if rows is None : <EOL> rows = n_out // cols <EOL> if n_out % cols : rows += <NUM_LIT:1> <EOL> filter_shape = self . filter_shape <EOL> height = rows * ( row_gap + filter_shape [ <NUM_LIT:0> ] ) - row_gap <EOL> width = cols * ( col_gap + filter_shape [ <NUM_LIT:1> ] ) - col_gap <EOL> out_array = numpy . zeros ( ( height , width , <NUM_LIT:3> ) , dtype = '<STR_LIT>' ) <EOL> w = self . w . value <EOL> w_col = <NUM_LIT:0> <EOL> def pixel_range ( x ) : <EOL> return <NUM_LIT:255> * ( x - x . min ( ) ) / ( x . max ( ) - x . min ( ) + eps ) <EOL> for r in xrange ( rows ) : <EOL> out_r_low = r * ( row_gap + filter_shape [ <NUM_LIT:0> ] ) <EOL> out_r_high = out_r_low + filter_shape [ <NUM_LIT:0> ] <EOL> for c in xrange ( cols ) : <EOL> out_c_low = c * ( col_gap + filter_shape [ <NUM_LIT:1> ] ) <EOL> out_c_high = out_c_low + filter_shape [ <NUM_LIT:1> ] <EOL> out_tile = out_array [ out_r_low : out_r_high , out_c_low : out_c_high , : ] <EOL> if c % <NUM_LIT:3> == <NUM_LIT:0> : <EOL> if w_col < w . shape [ <NUM_LIT:1> ] : <EOL> out_tile [ ... ] = pixel_range ( w [ : , w_col ] ) . reshape ( filter_shape + ( <NUM_LIT:1> , ) ) <EOL> w_col += <NUM_LIT:1> <EOL> if c % <NUM_LIT:3> == <NUM_LIT:1> : <EOL> if w_col < w . shape [ <NUM_LIT:1> ] : <EOL> for i in xrange ( min ( self . n_E_quadratic , <NUM_LIT:3> ) ) : <EOL> out_tile [ : , : , i ] = pixel_range ( w [ : , w_col + i ] ) . reshape ( filter_shape ) <EOL> w_col += self . n_E_quadratic <EOL> if c % <NUM_LIT:3> == <NUM_LIT:2> : <EOL> if w_col < w . shape [ <NUM_LIT:1> ] : <EOL> for i in xrange ( min ( self . n_S_quadratic , <NUM_LIT:3> ) ) : <EOL> out_tile [ : , : , <NUM_LIT:2> - i ] = pixel_range ( w [ : , w_col + i ] ) . reshape ( filter_shape ) <EOL> w_col += self . n_S_quadratic <EOL> return Image . fromarray ( out_array , '<STR_LIT>' ) <EOL> class Config ( object ) : <EOL> use_gpu = True <EOL> dtype = '<STR_LIT>' <EOL> dtype2 = dtype <EOL> if dtype2 == '<STR_LIT>' : <EOL> import theano . config as c <EOL> dtype2 = c . config . get ( '<STR_LIT>' ) <EOL> rng_seed = <NUM_LIT> <EOL> n_hid = <NUM_LIT> <EOL> n_terms = <NUM_LIT:4> <EOL> ft_lr_t0 = <NUM_LIT> <EOL> ft_t_decay = <NUM_LIT:0> <EOL> ft_lr_t_decay = <NUM_LIT> <EOL> ft_cost_classif_l1 = <NUM_LIT:0> <EOL> ft_cost_classif_l2 = <NUM_LIT:0> <EOL> ft_cost_in_l1_filter = <NUM_LIT:0> <EOL> ft_cost_in_l2_filter = <NUM_LIT:0> <EOL> ft_cost_in_l1_exp = <NUM_LIT:0> <EOL> ft_cost_in_l2_exp = <NUM_LIT:0> <EOL> ft_cost_in_l1_w = <NUM_LIT:0> <EOL> ft_cost_in_l2_w = <NUM_LIT:0> <EOL> ft_limit_iters = - <NUM_LIT:1> <EOL> ft_limit_walltime = <NUM_LIT:0> <EOL> ft_batchsize = <NUM_LIT:30> <EOL> ft_epoch_len = <NUM_LIT> <EOL> ft_status_interval = <NUM_LIT:50> <EOL> ft_validation_interval = property ( lambda s : s . ft_epoch_len / s . ft_batchsize ) <EOL> ft_ntrain_limit = <NUM_LIT:0> <EOL> ft_test_lag1 = True <EOL> lr = <NUM_LIT> <EOL> if <NUM_LIT:0> : <EOL> def test_bench_elemwise ( n_iter = <NUM_LIT:1000> , ** kwargs ) : <EOL> conf = Config ( ) <EOL> for k in kwargs : <EOL> setattr ( conf , k , kwargs [ k ] ) <EOL> if conf . use_gpu : <EOL> from nose . plugins . skip import SkipTest <EOL> import theano . sandbox . cuda as cuda_ndarray <EOL> if not cuda_ndarray . cuda_available : <EOL> raise SkipTest ( '<STR_LIT>' ) <EOL> import theano . sandbox . cuda <EOL> theano . sandbox . cuda . use ( ) <EOL> debug = False <EOL> if isinstance ( theano . compile . mode . get_default_mode ( ) , <EOL> theano . compile . debugmode . DebugMode ) : <EOL> debug = True <EOL> s_lr = theano . tensor . fscalar ( ) <EOL> if not debug : <EOL> sshape = ( None , <NUM_LIT> ) <EOL> else : sshape = ( None , <NUM_LIT:3> ) <EOL> x = theano . tensor . TensorType ( dtype = conf . dtype , broadcastable = ( <NUM_LIT:0> , <NUM_LIT:0> ) , shape = sshape ) ( ) <EOL> y = theano . tensor . lvector ( ) <EOL> rng = numpy . random . RandomState ( conf . rng_seed ) <EOL> if not debug : <EOL> layer = Kouh2008 . new_filters_expbounds ( rng , x , x . type . shape [ <NUM_LIT:1> ] , conf . n_hid , conf . n_terms ) <EOL> else : <EOL> layer = Kouh2008 . new_filters_expbounds ( rng , x , x . type . shape [ <NUM_LIT:1> ] , <NUM_LIT:3> , <NUM_LIT:2> ) <EOL> n_iter = <NUM_LIT:3> <EOL> cost = layer . output . mean ( ) <EOL> assert cost . type . ndim == <NUM_LIT:0> <EOL> print ( layer . params ) <EOL> gparams = theano . tensor . grad ( cost , layer . params ) <EOL> updates = [ ( p , p - s_lr * gp ) for p , gp in zip ( layer . params , gparams ) ] <EOL> train_nll = pfunc ( [ x , y , s_lr ] , [ ] , updates = updates ) <EOL> xval = theano . _asarray ( <EOL> rng . uniform ( size = ( conf . ft_batchsize , x . type . shape [ <NUM_LIT:1> ] ) ) , <EOL> dtype = conf . dtype2 , <EOL> ) <EOL> yval = numpy . arange ( conf . ft_batchsize ) <EOL> for i in xrange ( n_iter ) : <EOL> train_nll ( xval , yval , conf . lr ) </s>
