<s> import gnumpy as gp <EOL> import numpy as np <EOL> def relu_hard ( x , computeGrad = False ) : <EOL> if ( not computeGrad ) : <EOL> f = ( <NUM_LIT:1> / <NUM_LIT> ) * ( x + gp . sign ( x ) * x ) <EOL> return f <EOL> g = gp . sign ( x ) <EOL> return g <EOL> def relu ( x , computeGrad = False ) : <EOL> negslope = <NUM_LIT> <EOL> a = ( <NUM_LIT:1> + negslope ) / <NUM_LIT> ; b = ( <NUM_LIT:1> - negslope ) / <NUM_LIT> <EOL> if ( not computeGrad ) : <EOL> f = a * x + b * gp . sign ( x ) * x <EOL> return f <EOL> g = a + b * gp . sign ( x ) <EOL> return g <EOL> def sigmoid ( x , computeGrad = False ) : <EOL> if ( not computeGrad ) : <EOL> f = gp . logistic ( x ) <EOL> return f <EOL> g = x * ( <NUM_LIT:1.> - x ) <EOL> return g <EOL> class NNet : <EOL> def __init__ ( self , inputDim , outputDim , layerSizes , mbSize = <NUM_LIT> , train = True , <EOL> activation = '<STR_LIT:relu>' ) : <EOL> self . outputDim = outputDim <EOL> self . inputDim = inputDim <EOL> self . layerSizes = layerSizes <EOL> self . stack = None <EOL> self . train = train <EOL> self . funcdict = { <EOL> "<STR_LIT>" : relu_hard , <EOL> "<STR_LIT:relu>" : relu , <EOL> "<STR_LIT>" : sigmoid , <EOL> } <EOL> self . activation = self . funcdict [ activation ] <EOL> self . mbSize = mbSize <EOL> def initParams ( self ) : <EOL> """<STR_LIT>""" <EOL> sizes = [ self . inputDim ] + self . layerSizes + [ self . outputDim ] <EOL> scales = [ gp . sqrt ( <NUM_LIT:6> ) / gp . sqrt ( n + m ) for n , m in zip ( sizes [ : - <NUM_LIT:1> ] , sizes [ <NUM_LIT:1> : ] ) ] <EOL> self . stack = [ [ gp . rand ( m , n ) * <NUM_LIT:2> * s - s , gp . zeros ( ( m , <NUM_LIT:1> ) ) ] for n , m , s in zip ( sizes [ : - <NUM_LIT:1> ] , sizes [ <NUM_LIT:1> : ] , scales ) ] <EOL> self . hActs = [ gp . empty ( ( s , self . mbSize ) ) for s in sizes ] <EOL> if self . train : <EOL> self . deltas = [ gp . empty ( ( s , self . mbSize ) ) for s in sizes [ <NUM_LIT:1> : ] ] <EOL> self . grad = [ [ gp . empty ( w . shape ) , gp . empty ( b . shape ) ] for w , b in self . stack ] <EOL> def costAndGrad ( self , data , labels ) : <EOL> self . hActs [ <NUM_LIT:0> ] = data <EOL> i = <NUM_LIT:1> <EOL> for w , b in self . stack : <EOL> self . hActs [ i ] = w . dot ( self . hActs [ i - <NUM_LIT:1> ] ) + b <EOL> if i <= len ( self . layerSizes ) : <EOL> self . hActs [ i ] = self . activation ( self . hActs [ i ] ) <EOL> i += <NUM_LIT:1> <EOL> probs = self . hActs [ - <NUM_LIT:1> ] - gp . max ( self . hActs [ - <NUM_LIT:1> ] , axis = <NUM_LIT:0> ) <EOL> probs = gp . exp ( probs ) <EOL> probs = probs / gp . sum ( probs , axis = <NUM_LIT:0> ) <EOL> labelMat = np . zeros ( probs . shape ) <EOL> labelMat [ labels , range ( self . mbSize ) ] = <NUM_LIT:1> <EOL> labelMat = gp . garray ( labelMat ) <EOL> cost = - ( <NUM_LIT:1.> / self . mbSize ) * np . nansum ( gp . as_numpy_array ( labelMat * gp . log ( probs ) ) ) <EOL> if not self . train : <EOL> return cost , None <EOL> self . deltas [ - <NUM_LIT:1> ] = probs - labelMat <EOL> i = len ( self . layerSizes ) - <NUM_LIT:1> <EOL> for w , b in reversed ( self . stack [ <NUM_LIT:1> : ] ) : <EOL> grad = self . activation ( self . hActs [ i + <NUM_LIT:1> ] , True ) <EOL> self . deltas [ i ] = w . T . dot ( self . deltas [ i + <NUM_LIT:1> ] ) * grad <EOL> i -= <NUM_LIT:1> <EOL> for i in range ( len ( self . grad ) ) : <EOL> self . grad [ i ] [ <NUM_LIT:0> ] = ( <NUM_LIT:1.> / self . mbSize ) * self . deltas [ i ] . dot ( self . hActs [ i ] . T ) <EOL> self . grad [ i ] [ <NUM_LIT:1> ] = ( <NUM_LIT:1.> / self . mbSize ) * gp . sum ( self . deltas [ i ] , axis = <NUM_LIT:1> ) . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> return cost , self . grad <EOL> def updateParams ( self , scale , update ) : <EOL> self . stack = [ [ ws [ <NUM_LIT:0> ] + scale * wsDelta [ <NUM_LIT:0> ] , ws [ <NUM_LIT:1> ] + scale * wsDelta [ <NUM_LIT:1> ] ] <EOL> for ws , wsDelta in zip ( self . stack , update ) ] <EOL> def vecToStack ( self , vec ) : <EOL> start = <NUM_LIT:0> <EOL> sizes = [ self . inputDim ] + self . layerSizes + [ self . outputDim ] <EOL> for n , m , i in zip ( sizes [ : - <NUM_LIT:1> ] , sizes [ <NUM_LIT:1> : ] , range ( len ( sizes ) - <NUM_LIT:1> ) ) : <EOL> self . stack [ i ] = [ gp . garray ( np . reshape ( np . array ( vec [ start : start + m * n ] ) , ( m , n ) ) ) , gp . garray ( np . reshape ( np . array ( vec [ start + m * n : start + m * ( n + <NUM_LIT:1> ) ] ) , ( m , <NUM_LIT:1> ) ) ) ] <EOL> start += m * ( n + <NUM_LIT:1> ) <EOL> def vectorize ( self , x ) : <EOL> return [ v for layer in x for wb in layer for w_or_b in wb for v in w_or_b ] <EOL> def paramVec ( self ) : <EOL> return self . vectorize ( self . stack ) <EOL> def costAndGradVec ( self , params , data , labels ) : <EOL> """<STR_LIT>""" <EOL> self . vecToStack ( params ) <EOL> cost , grad = self . costAndGrad ( data , labels ) <EOL> if ( grad != None ) : <EOL> vecgrad = self . vectorize ( grad ) <EOL> return cost , vecgrad <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> inputDim = <NUM_LIT:5> <EOL> outputDim = <NUM_LIT:10> <EOL> layerSizes = [ <NUM_LIT:100> , <NUM_LIT:100> , <NUM_LIT> ] <EOL> mbSize = <NUM_LIT:5> <EOL> data = gp . rand ( inputDim , mbSize ) <EOL> import random <EOL> labels = [ random . randint ( <NUM_LIT:0> , <NUM_LIT:9> ) ] * mbSize <EOL> nn = NNet ( inputDim , outputDim , layerSizes , mbSize , train = True ) <EOL> nn . initParams ( ) <EOL> cost , grad = nn . costAndGrad ( data , labels ) <EOL> print cost </s>
