<s> import theano <EOL> import theano . tensor as T <EOL> class Softmax ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , x ) : <EOL> e_x = T . exp ( x - x . max ( axis = <NUM_LIT:1> ) . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' ) ) <EOL> return e_x / e_x . sum ( axis = <NUM_LIT:1> ) . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' ) <EOL> class Maxout ( object ) : <EOL> def __init__ ( self , n_pool = <NUM_LIT:2> ) : <EOL> self . n_pool = n_pool <EOL> def __call__ ( self , x ) : <EOL> if x . ndim == <NUM_LIT:2> : <EOL> x = T . max ( [ x [ : , n : : self . n_pool ] for n in range ( self . n_pool ) ] , axis = <NUM_LIT:0> ) <EOL> elif x . ndim == <NUM_LIT:4> : <EOL> x = T . max ( [ x [ : , n : : self . n_pool , : , : ] for n in range ( self . n_pool ) ] , axis = <NUM_LIT:0> ) <EOL> elif x . ndim == <NUM_LIT:3> : <EOL> print '<STR_LIT>' <EOL> x = T . max ( [ x [ : , : , n : : self . n_pool ] for n in range ( self . n_pool ) ] , axis = <NUM_LIT:0> ) <EOL> return x <EOL> class ConvRMSPool ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , x ) : <EOL> x = x ** <NUM_LIT:2> <EOL> return T . sqrt ( x [ : , : : <NUM_LIT:2> , : , : ] + x [ : , <NUM_LIT:1> : : <NUM_LIT:2> , : , : ] + <NUM_LIT> ) <EOL> class ConvSoftmax ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , x ) : <EOL> e_x = T . exp ( x - x . max ( axis = <NUM_LIT:1> , keepdims = True ) ) <EOL> return e_x / e_x . sum ( axis = <NUM_LIT:1> , keepdims = True ) <EOL> class MaskedConvSoftmax ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , x , m ) : <EOL> x = x * m . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' , <NUM_LIT:1> , '<STR_LIT:x>' ) <EOL> e_x = T . exp ( x - x . max ( axis = <NUM_LIT:2> , keepdims = True ) ) <EOL> e_x = e_x * m . dimshuffle ( <NUM_LIT:0> , '<STR_LIT:x>' , <NUM_LIT:1> , '<STR_LIT:x>' ) <EOL> return e_x / e_x . sum ( axis = <NUM_LIT:2> , keepdims = True ) <EOL> class ELU ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , x ) : <EOL> return T . switch ( T . ge ( x , <NUM_LIT:0> ) , x , T . exp ( x ) - <NUM_LIT:1> ) <EOL> class Rectify ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , x ) : <EOL> return ( x + abs ( x ) ) / <NUM_LIT> <EOL> class ClippedRectify ( object ) : <EOL> def __init__ ( self , clip = <NUM_LIT> ) : <EOL> self . clip = clip <EOL> def __call__ ( self , x ) : <EOL> return T . clip ( ( x + abs ( x ) ) / <NUM_LIT> , <NUM_LIT:0.> , self . clip ) <EOL> class LeakyRectify ( object ) : <EOL> def __init__ ( self , leak = <NUM_LIT> ) : <EOL> self . leak = leak <EOL> def __call__ ( self , x ) : <EOL> f1 = <NUM_LIT:0.5> * ( <NUM_LIT:1> + self . leak ) <EOL> f2 = <NUM_LIT:0.5> * ( <NUM_LIT:1> - self . leak ) <EOL> return f1 * x + f2 * abs ( x ) <EOL> class Prelu ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , x , leak ) : <EOL> f1 = <NUM_LIT:0.5> * ( <NUM_LIT:1> + leak ) <EOL> f2 = <NUM_LIT:0.5> * ( <NUM_LIT:1> - leak ) <EOL> if leak . ndim == <NUM_LIT:1> : <EOL> return T . flatten ( f1 , <NUM_LIT:1> ) [ <NUM_LIT:0> ] * x + T . flatten ( f2 , <NUM_LIT:1> ) [ <NUM_LIT:0> ] * abs ( x ) <EOL> else : <EOL> return f1 * x + f2 * abs ( x ) <EOL> class Tanh ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , x ) : <EOL> return T . tanh ( x ) <EOL> class Sigmoid ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , x ) : <EOL> return T . nnet . sigmoid ( x ) <EOL> class Linear ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , x ) : <EOL> return x <EOL> class SteeperSigmoid ( object ) : <EOL> def __init__ ( self , scale = <NUM_LIT> ) : <EOL> self . scale = scale <EOL> def __call__ ( self , x ) : <EOL> return <NUM_LIT:1.> / ( <NUM_LIT:1.> + T . exp ( - self . scale * x ) ) <EOL> class HardSigmoid ( object ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> def __call__ ( self , X ) : <EOL> return T . clip ( X + <NUM_LIT:0.5> , <NUM_LIT:0.> , <NUM_LIT:1.> ) </s>
