<s> import cudamat as cm <EOL> import ctc_fast as ctc <EOL> import numpy as np <EOL> class NNet : <EOL> def __init__ ( self , inputDim , outputDim , layerSize , numLayers , maxBatch , train = True ) : <EOL> cm . cublas_init ( ) <EOL> self . outputDim = outputDim <EOL> self . inputDim = inputDim <EOL> self . layerSizes = [ layerSize ] * numLayers <EOL> self . maxBatch = maxBatch <EOL> self . train = train <EOL> def initParams ( self ) : <EOL> """<STR_LIT>""" <EOL> sizes = [ self . inputDim ] + self . layerSizes + [ self . outputDim ] <EOL> scales = [ np . sqrt ( <NUM_LIT:6> ) / np . sqrt ( n + m ) for n , m in zip ( sizes [ : - <NUM_LIT:1> ] , sizes [ <NUM_LIT:1> : ] ) ] <EOL> self . stack = [ [ np . random . rand ( m , n ) * <NUM_LIT:2> * s - s , np . zeros ( ( m , <NUM_LIT:1> ) ) ] for n , m , s in zip ( sizes [ : - <NUM_LIT:1> ] , sizes [ <NUM_LIT:1> : ] , scales ) ] <EOL> self . hActs_M = [ cm . empty ( ( s , self . maxBatch ) ) for s in sizes ] <EOL> if self . train : <EOL> self . grad = [ [ cm . empty ( w . shape ) , cm . empty ( b . shape ) ] for w , b in self . stack ] <EOL> self . deltasC_M = cm . empty ( ( self . outputDim , self . maxBatch ) ) <EOL> self . deltasOut_M = cm . empty ( ( sizes [ <NUM_LIT:1> ] , self . maxBatch ) ) <EOL> self . deltasIn_M = cm . empty ( ( sizes [ <NUM_LIT:1> ] , self . maxBatch ) ) <EOL> self . tmpGrad_M = cm . empty ( ( self . layerSizes [ <NUM_LIT:0> ] , self . maxBatch ) ) <EOL> self . probs_M = cm . empty ( ( self . outputDim , self . maxBatch ) ) <EOL> self . rowVec_M = cm . empty ( ( <NUM_LIT:1> , self . maxBatch ) ) <EOL> self . stack = [ [ cm . CUDAMatrix ( w ) , cm . CUDAMatrix ( b ) ] <EOL> for w , b in self . stack ] <EOL> def setViews ( self , batchSize ) : <EOL> """<STR_LIT>""" <EOL> assert batchSize <= self . maxBatch , "<STR_LIT>" <EOL> self . hActs = [ H . get_col_slice ( <NUM_LIT:0> , batchSize ) for H in self . hActs_M ] <EOL> self . deltasC = self . deltasC_M . get_col_slice ( <NUM_LIT:0> , batchSize ) <EOL> self . deltasOut = self . deltasOut_M . get_col_slice ( <NUM_LIT:0> , batchSize ) <EOL> self . deltasIn = self . deltasIn_M . get_col_slice ( <NUM_LIT:0> , batchSize ) <EOL> self . probs = self . probs_M . get_col_slice ( <NUM_LIT:0> , batchSize ) <EOL> self . rowVec = self . rowVec_M . get_col_slice ( <NUM_LIT:0> , batchSize ) <EOL> self . tmpGrad = self . tmpGrad_M . get_col_slice ( <NUM_LIT:0> , batchSize ) <EOL> def costAndGrad ( self , data , labels ) : <EOL> batchSize = data . shape [ <NUM_LIT:1> ] <EOL> self . setViews ( batchSize ) <EOL> self . hActs [ <NUM_LIT:0> ] . assign ( cm . CUDAMatrix ( data ) ) <EOL> i = <NUM_LIT:1> <EOL> for w , b in self . stack : <EOL> cm . dot ( w , self . hActs [ i - <NUM_LIT:1> ] , self . hActs [ i ] ) <EOL> self . hActs [ i ] . add_col_vec ( b ) <EOL> if i <= len ( self . layerSizes ) : <EOL> self . hActs [ i ] . maximum ( <NUM_LIT:0.0> ) <EOL> i += <NUM_LIT:1> <EOL> self . hActs [ - <NUM_LIT:1> ] . max ( axis = <NUM_LIT:0> , target = self . rowVec ) <EOL> self . hActs [ - <NUM_LIT:1> ] . add_row_mult ( self . rowVec , - <NUM_LIT:1.0> , target = self . probs ) <EOL> cm . exp ( self . probs ) <EOL> self . probs . sum ( axis = <NUM_LIT:0> , target = self . rowVec ) <EOL> cm . pow ( self . rowVec , - <NUM_LIT:1.0> , target = self . rowVec ) <EOL> self . probs . mult_by_row ( self . rowVec ) <EOL> self . probs . copy_to_host ( ) <EOL> cost , deltas , skip = ctc . ctc_loss ( self . probs . numpy_array . astype ( np . float64 ) , <EOL> labels , blank = <NUM_LIT:0> ) <EOL> self . deltasC . assign ( cm . CUDAMatrix ( deltas ) ) <EOL> if skip : <EOL> return cost , self . grad , skip <EOL> nl = len ( self . layerSizes ) <EOL> i = nl <EOL> deltasIn , deltasOut = self . deltasC , self . deltasOut <EOL> for w , b in reversed ( self . stack ) : <EOL> cm . dot ( deltasIn , self . hActs [ i ] . T , target = self . grad [ i ] [ <NUM_LIT:0> ] ) <EOL> deltasIn . sum ( axis = <NUM_LIT:1> , target = self . grad [ i ] [ <NUM_LIT:1> ] ) <EOL> if i > <NUM_LIT:0> : <EOL> self . hActs [ i ] . sign ( target = self . tmpGrad ) <EOL> cm . dot ( w . T , deltasIn , target = deltasOut ) <EOL> deltasOut . mult ( self . tmpGrad ) <EOL> if i == nl : <EOL> deltasIn = self . deltasIn <EOL> deltasIn , deltasOut = deltasOut , deltasIn <EOL> i -= <NUM_LIT:1> <EOL> return cost , self . grad , skip <EOL> def updateParams ( self , scale , update ) : <EOL> for params , paramsDel in zip ( self . stack , update ) : <EOL> w , b = params <EOL> dw , db = paramsDel <EOL> w . add_mult ( dw , alpha = scale ) <EOL> b . add_mult ( db , alpha = scale ) <EOL> def toFile ( self , fid ) : <EOL> """<STR_LIT>""" <EOL> import cPickle as pickle <EOL> stack = [ ] <EOL> for w , b in self . stack : <EOL> w . copy_to_host ( ) <EOL> b . copy_to_host ( ) <EOL> stack . append ( [ w . numpy_array , b . numpy_array ] ) <EOL> pickle . dump ( stack , fid ) <EOL> def fromFile ( self , fid ) : <EOL> import cPickle as pickle <EOL> stack = pickle . load ( fid ) <EOL> self . stack = [ [ cm . CUDAMatrix ( w ) , cm . CUDAMatrix ( b ) ] <EOL> for w , b in stack ] <EOL> def check_grad ( self , data , labels , epsilon = <NUM_LIT> ) : <EOL> cost , grad , _ = self . costAndGrad ( data , labels ) <EOL> for param , delta in zip ( self . stack , grad ) : <EOL> w , b = param <EOL> dw , db = delta <EOL> dw . copy_to_host ( ) <EOL> w . copy_to_host ( ) <EOL> for i in xrange ( w . shape [ <NUM_LIT:0> ] ) : <EOL> for j in xrange ( w . shape [ <NUM_LIT:1> ] ) : <EOL> w . numpy_array [ i , j ] += epsilon <EOL> w . copy_to_device ( ) <EOL> costP , _ , _ = self . costAndGrad ( data , labels ) <EOL> numGrad = ( costP - cost ) / epsilon <EOL> w . numpy_array [ i , j ] -= epsilon <EOL> print "<STR_LIT>" % ( dw . numpy_array [ i , j ] , numGrad ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> import dataLoader as dl <EOL> np . random . seed ( <NUM_LIT> ) <EOL> layerSize = <NUM_LIT:200> <EOL> numLayers = <NUM_LIT:3> <EOL> dataDir = "<STR_LIT>" <EOL> inputDim = <NUM_LIT> * <NUM_LIT:15> <EOL> rawDim = <NUM_LIT> * <NUM_LIT:15> <EOL> outputDim = <NUM_LIT> <EOL> maxUttLen = <NUM_LIT> <EOL> loader = dl . DataLoader ( dataDir , rawDim , inputDim ) <EOL> data_dict , alis , keys , _ = loader . loadDataFileDict ( <NUM_LIT:1> ) <EOL> data , labels = data_dict [ keys [ <NUM_LIT:3> ] ] , np . array ( alis [ keys [ <NUM_LIT:3> ] ] , dtype = np . int32 ) <EOL> nn = NNet ( inputDim , outputDim , layerSize , numLayers , maxUttLen ) <EOL> nn . initParams ( ) <EOL> cost , grad , _ = nn . costAndGrad ( data , labels ) <EOL> nn . check_grad ( data , labels ) </s>
