<s> import gnumpy as gp <EOL> import numpy as np <EOL> import ctc_fast as ctc <EOL> def relu_hard ( x , computeGrad = False ) : <EOL> if ( not computeGrad ) : <EOL> f = ( <NUM_LIT:1> / <NUM_LIT> ) * ( x + gp . sign ( x ) * x ) <EOL> return f <EOL> g = np . sign ( x ) <EOL> return g <EOL> def relu ( x , computeGrad = False ) : <EOL> negslope = <NUM_LIT> <EOL> a = ( <NUM_LIT:1> + negslope ) / <NUM_LIT> ; b = ( <NUM_LIT:1> - negslope ) / <NUM_LIT> <EOL> if ( not computeGrad ) : <EOL> f = a * x + b * gp . sign ( x ) * x <EOL> return f <EOL> g = a + b * gp . sign ( x ) <EOL> return g <EOL> def sigmoid ( x , computeGrad = False ) : <EOL> if ( not computeGrad ) : <EOL> f = gp . logistic ( x ) <EOL> return f <EOL> g = x * ( <NUM_LIT:1.> - x ) <EOL> return g <EOL> class RNNet : <EOL> def __init__ ( self , inputDim , outputDim , layerSizes , train = True , <EOL> activation = '<STR_LIT:relu>' , temporalLayer = - <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> self . outputDim = outputDim <EOL> self . inputDim = inputDim <EOL> self . layerSizes = layerSizes <EOL> self . temporalLayer = temporalLayer <EOL> self . stack = None <EOL> self . train = train <EOL> self . funcdict = { <EOL> "<STR_LIT>" : relu_hard , <EOL> "<STR_LIT:relu>" : relu , <EOL> "<STR_LIT>" : sigmoid , <EOL> } <EOL> self . activation = self . funcdict [ activation ] <EOL> self . hist = { } <EOL> def initParams ( self ) : <EOL> """<STR_LIT>""" <EOL> sizes = [ self . inputDim ] + self . layerSizes + [ self . outputDim ] <EOL> scales = [ gp . sqrt ( <NUM_LIT:6> ) / gp . sqrt ( n + m ) for n , m in zip ( sizes [ : - <NUM_LIT:1> ] , sizes [ <NUM_LIT:1> : ] ) ] <EOL> self . stack = [ [ gp . rand ( m , n ) * <NUM_LIT:2> * s - s , gp . zeros ( ( m , <NUM_LIT:1> ) ) ] for n , m , s in zip ( sizes [ : - <NUM_LIT:1> ] , sizes [ <NUM_LIT:1> : ] , scales ) ] <EOL> if self . temporalLayer > <NUM_LIT:0> : <EOL> rs = sizes [ self . temporalLayer ] <EOL> s = gp . sqrt ( <NUM_LIT:6> ) / rs <EOL> self . stack . append ( [ gp . rand ( rs , rs ) * <NUM_LIT:2> * s - s , gp . zeros ( ( <NUM_LIT:2> , <NUM_LIT:1> ) ) ] ) <EOL> if self . train : <EOL> self . grad = [ [ gp . empty ( w . shape ) , gp . empty ( b . shape ) ] for w , b in self . stack ] <EOL> def updateParams ( self , scale , update ) : <EOL> """<STR_LIT>""" <EOL> self . stack = [ [ ws [ <NUM_LIT:0> ] + scale * wsDelta [ <NUM_LIT:0> ] , ws [ <NUM_LIT:1> ] + scale * wsDelta [ <NUM_LIT:1> ] ] <EOL> for ws , wsDelta in zip ( self . stack , update ) ] <EOL> def vecToStack ( self , vec ) : <EOL> start = <NUM_LIT:0> <EOL> sizes = [ self . inputDim ] + self . layerSizes + [ self . outputDim ] <EOL> for n , m , i in zip ( sizes [ : - <NUM_LIT:1> ] , sizes [ <NUM_LIT:1> : ] , range ( len ( sizes ) - <NUM_LIT:1> ) ) : <EOL> self . stack [ i ] = [ gp . garray ( vec [ start : start + m * n ] ) . reshape ( m , n ) , gp . garray ( vec [ start + m * n : start + m * ( n + <NUM_LIT:1> ) ] ) . reshape ( m , <NUM_LIT:1> ) ] <EOL> start += m * ( n + <NUM_LIT:1> ) <EOL> if self . temporalLayer > <NUM_LIT:0> : <EOL> rs = self . layerSizes [ self . temporalLayer - <NUM_LIT:1> ] <EOL> self . stack [ - <NUM_LIT:1> ] = [ gp . garray ( vec [ start : start + rs * rs ] ) . reshape ( rs , rs ) , gp . garray ( vec [ start + rs * rs : ] ) ] <EOL> def vectorize ( self , x ) : <EOL> """<STR_LIT>""" <EOL> return [ v for layer in x for wb in layer for w_or_b in wb for v in w_or_b ] <EOL> return r <EOL> def paramVec ( self ) : <EOL> return self . vectorize ( self . stack ) <EOL> def costAndGradVec ( self , params , data , labels ) : <EOL> """<STR_LIT>""" <EOL> self . vecToStack ( params ) <EOL> cost , grad = self . costAndGrad ( data , labels ) <EOL> if ( grad != None ) : <EOL> vecgrad = self . vectorize ( grad ) <EOL> return cost , vecgrad <EOL> def costAndGrad ( self , data , labels , key = None ) : <EOL> """<STR_LIT>""" <EOL> T = data . shape [ <NUM_LIT:1> ] <EOL> sizes = [ self . inputDim ] + self . layerSizes + [ self . outputDim ] <EOL> stackMax = len ( self . stack ) - <NUM_LIT:1> <EOL> if self . temporalLayer > <NUM_LIT:0> : <EOL> stackMax -= <NUM_LIT:1> <EOL> self . hActs = [ gp . empty ( ( s , T ) ) for s in sizes ] <EOL> self . hActs [ <NUM_LIT:0> ] = data <EOL> i = <NUM_LIT:1> <EOL> for l in range ( stackMax + <NUM_LIT:1> ) : <EOL> w , b = self . stack [ l ] <EOL> self . hActs [ i ] = w . dot ( self . hActs [ i - <NUM_LIT:1> ] ) + b <EOL> if ( self . temporalLayer - <NUM_LIT:1> ) == l : <EOL> for t in range ( T ) : <EOL> if t > <NUM_LIT:0> : <EOL> self . hActs [ i ] [ : , t ] += self . stack [ - <NUM_LIT:1> ] [ <NUM_LIT:0> ] . dot ( self . hActs [ i ] [ : , t - <NUM_LIT:1> ] ) <EOL> if i <= stackMax : <EOL> self . hActs [ i ] [ : , t ] = self . activation ( self . hActs [ i ] [ : , t ] ) <EOL> elif i <= stackMax : <EOL> self . hActs [ i ] = self . activation ( self . hActs [ i ] ) <EOL> i += <NUM_LIT:1> <EOL> probs = self . hActs [ - <NUM_LIT:1> ] - gp . max ( self . hActs [ - <NUM_LIT:1> ] , axis = <NUM_LIT:0> ) <EOL> probs = gp . as_numpy_array ( probs ) <EOL> probs = np . exp ( probs ) <EOL> probs = probs / np . sum ( probs , axis = <NUM_LIT:0> ) <EOL> cost , delta_output , skip = ctc . ctc_loss ( probs , labels . squeeze ( ) , blank = <NUM_LIT:0> ) <EOL> if key is not None and key in self . hist : <EOL> self . hist [ key ] . append ( ( probs , delta_output ) ) <EOL> if not self . train : <EOL> return cost , None <EOL> delta_output = gp . garray ( delta_output ) <EOL> self . grad = [ [ gp . zeros ( w . shape ) , gp . zeros ( b . shape ) ] for w , b in self . stack ] <EOL> if self . temporalLayer > <NUM_LIT:0> : <EOL> delta_t = np . zeros ( self . layerSizes [ self . temporalLayer - <NUM_LIT:1> ] ) <EOL> for t in reversed ( range ( T ) ) : <EOL> delta = delta_output [ : , t ] . T <EOL> self . grad [ stackMax ] [ <NUM_LIT:0> ] += delta . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> ) . dot ( self . hActs [ - <NUM_LIT:2> ] [ : , t ] . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> ) . T ) <EOL> self . grad [ stackMax ] [ <NUM_LIT:1> ] += delta . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> delta = self . stack [ stackMax ] [ <NUM_LIT:0> ] . T . dot ( delta ) <EOL> i = len ( self . layerSizes ) - <NUM_LIT:1> <EOL> while i >= <NUM_LIT:0> : <EOL> if ( self . temporalLayer - <NUM_LIT:1> ) == i : <EOL> delta += delta_t <EOL> delta = delta * self . activation ( self . hActs [ i + <NUM_LIT:1> ] [ : , t ] , True ) <EOL> self . grad [ i ] [ <NUM_LIT:0> ] += delta . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> ) . dot ( self . hActs [ i ] [ : , t ] . T . reshape ( <NUM_LIT:1> , - <NUM_LIT:1> ) ) <EOL> self . grad [ i ] [ <NUM_LIT:1> ] += delta . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> if ( self . temporalLayer - <NUM_LIT:1> ) == i and t > <NUM_LIT:0> : <EOL> self . grad [ - <NUM_LIT:1> ] [ <NUM_LIT:0> ] += delta . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> ) . dot ( self . hActs [ i + <NUM_LIT:1> ] [ : , t - <NUM_LIT:1> ] . T . reshape ( <NUM_LIT:1> , - <NUM_LIT:1> ) ) <EOL> delta_t = self . stack [ - <NUM_LIT:1> ] [ <NUM_LIT:0> ] . T . dot ( delta ) <EOL> self . grad [ - <NUM_LIT:1> ] [ <NUM_LIT:1> ] = np . zeros ( ( <NUM_LIT:2> , <NUM_LIT:1> ) ) <EOL> w , b = self . stack [ i ] <EOL> delta = w . T . dot ( delta ) <EOL> i -= <NUM_LIT:1> <EOL> return cost , self . grad , skip <EOL> def toFile ( self , fid ) : <EOL> """<STR_LIT>""" <EOL> import cPickle as pickle <EOL> pickle . dump ( self . stack , fid ) <EOL> def fromFile ( self , fid ) : <EOL> import cPickle as pickle <EOL> self . stack = pickle . load ( fid ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> inputDim = <NUM_LIT:3> <EOL> numPhones = <NUM_LIT:6> <EOL> outputDim = numPhones + <NUM_LIT:1> <EOL> seq_len_out = <NUM_LIT:2> <EOL> seq_len_in = <NUM_LIT:5> <EOL> assert seq_len_in >= seq_len_out <EOL> layerSizes = [ <NUM_LIT:10> , <NUM_LIT:4> ] <EOL> label_seq = np . floor ( np . random . rand ( seq_len_out , <NUM_LIT:1> ) * numPhones ) <EOL> label_seq = <NUM_LIT:1> + label_seq . astype ( np . int32 ) <EOL> data = np . random . randn ( inputDim , seq_len_in ) <EOL> nn = RNNet ( inputDim , outputDim , layerSizes , train = True , temporalLayer = <NUM_LIT:2> ) <EOL> nn . initParams ( ) <EOL> cost , grad = nn . costAndGrad ( data , label_seq ) <EOL> print cost <EOL> if True : <EOL> import timeit <EOL> setup = '''<STR_LIT>''' <EOL> print timeit . timeit ( '<STR_LIT>' , setup = setup , number = <NUM_LIT:1> ) </s>
