<s> from layer import * <EOL> class ReplicatedSoftmaxLayer ( Layer ) : <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> super ( ReplicatedSoftmaxLayer , self ) . __init__ ( * args , ** kwargs ) <EOL> @ classmethod <EOL> def IsLayerType ( cls , proto ) : <EOL> return proto . hyperparams . activation == deepnet_pb2 . Hyperparams . REPLICATED_SOFTMAX <EOL> def ApplyActivation ( self ) : <EOL> state = self . state <EOL> temp = self . batchsize_temp <EOL> state . max ( axis = <NUM_LIT:0> , target = temp ) <EOL> state . add_row_mult ( temp , - <NUM_LIT:1> ) <EOL> cm . exp ( state ) <EOL> state . sum ( axis = <NUM_LIT:0> , target = temp ) <EOL> self . NN . divide ( temp , target = temp ) <EOL> state . mult_by_row ( temp ) <EOL> def Sample ( self ) : <EOL> sample = self . sample <EOL> state = self . state <EOL> use_lightspeed = False <EOL> if use_lightspeed : <EOL> temp = self . expanded_batch <EOL> state . sum ( axis = <NUM_LIT:0> , target = self . temp ) <EOL> state . div_by_row ( self . temp , target = temp ) <EOL> probs_cpu = temp . asarray ( ) . astype ( np . float64 ) <EOL> numsamples = self . NN . asarray ( ) <EOL> samples_cpu = lightspeed . SampleSoftmax ( probs_cpu , numsamples ) <EOL> sample . overwrite ( samples_cpu . astype ( np . float32 ) ) <EOL> else : <EOL> if self . proto . hyperparams . adaptive_prior > <NUM_LIT:0> : <EOL> sample . assign ( <NUM_LIT:0> ) <EOL> temp_sample = self . expanded_batch <EOL> numsamples = int ( self . proto . hyperparams . adaptive_prior ) <EOL> for i in range ( numsamples ) : <EOL> state . perturb_prob_for_softmax_sampling ( target = temp_sample ) <EOL> temp_sample . choose_max_and_accumulate ( sample ) <EOL> else : <EOL> NN = self . NN . asarray ( ) . reshape ( - <NUM_LIT:1> ) <EOL> numdims , batchsize = self . state . shape <EOL> max_samples = self . big_sample_matrix . shape [ <NUM_LIT:1> ] <EOL> for i in range ( batchsize ) : <EOL> nn = NN [ i ] <EOL> factor = <NUM_LIT:1> <EOL> if nn > max_samples : <EOL> nn = max_samples <EOL> factor = float ( nn ) / max_samples <EOL> samples = self . big_sample_matrix . slice ( <NUM_LIT:0> , nn ) <EOL> samples . assign ( <NUM_LIT:0> ) <EOL> samples . add_col_vec ( self . state . slice ( i , i + <NUM_LIT:1> ) ) <EOL> samples . perturb_prob_for_softmax_sampling ( ) <EOL> samples . choose_max ( axis = <NUM_LIT:0> ) <EOL> samples . sum ( axis = <NUM_LIT:1> , target = sample . slice ( i , i + <NUM_LIT:1> ) ) <EOL> if factor > <NUM_LIT:1> : <EOL> sample . slice ( i , i + <NUM_LIT:1> ) . mult ( factor ) <EOL> def ComputeDeriv ( self ) : <EOL> """<STR_LIT>""" <EOL> raise Exception ( '<STR_LIT>' ) <EOL> def AllocateMemory ( self , batchsize ) : <EOL> super ( ReplicatedSoftmaxLayer , self ) . AllocateMemory ( batchsize ) <EOL> self . expansion_matrix = cm . CUDAMatrix ( np . eye ( self . numlabels ) ) <EOL> self . big_sample_matrix = cm . empty ( ( self . numlabels * self . dimensions , <NUM_LIT:1000> ) ) <EOL> def AllocateBatchsizeDependentMemory ( self , batchsize ) : <EOL> super ( ReplicatedSoftmaxLayer , self ) . AllocateBatchsizeDependentMemory ( batchsize ) <EOL> dimensions = self . dimensions <EOL> numlabels = self . numlabels <EOL> self . expanded_batch = cm . CUDAMatrix ( np . zeros ( ( numlabels * dimensions , batchsize ) ) ) <EOL> self . batchsize_temp = cm . CUDAMatrix ( np . zeros ( ( dimensions , batchsize ) ) ) <EOL> if self . is_input or self . is_initialized or self . is_output : <EOL> self . data = cm . CUDAMatrix ( np . zeros ( ( numlabels * dimensions , batchsize ) ) ) <EOL> self . NN = cm . CUDAMatrix ( np . ones ( ( <NUM_LIT:1> , batchsize ) ) ) <EOL> self . counter = cm . empty ( self . NN . shape ) <EOL> self . count_filter = cm . empty ( self . NN . shape ) <EOL> def ResetState ( self , rand = False ) : <EOL> if self . hyperparams . normalize : <EOL> self . NN . assign ( self . hyperparams . normalize_to ) <EOL> else : <EOL> self . NN . assign ( <NUM_LIT:1> ) <EOL> super ( ReplicatedSoftmaxLayer , self ) . ResetState ( rand = rand ) <EOL> def GetData ( self ) : <EOL> self . state . assign ( self . data ) <EOL> h = self . hyperparams <EOL> self . state . sum ( axis = <NUM_LIT:0> , target = self . NN ) <EOL> self . NN . add ( self . tiny ) <EOL> if h . multiplicative_prior > <NUM_LIT:0> : <EOL> self . NN . mult ( <NUM_LIT:1> + h . multiplicative_prior ) <EOL> self . state . mult ( <NUM_LIT:1> + h . multiplicative_prior ) <EOL> if h . additive_prior > <NUM_LIT:0> : <EOL> self . state . div_by_row ( self . NN ) <EOL> self . NN . add ( h . additive_prior ) <EOL> self . state . mult_by_row ( self . NN ) <EOL> if h . adaptive_prior > <NUM_LIT:0> : <EOL> self . state . div_by_row ( self . NN ) <EOL> self . state . mult ( h . adaptive_prior ) <EOL> self . NN . assign ( h . adaptive_prior ) <EOL> def GetLoss ( self , get_deriv = False ) : <EOL> """<STR_LIT>""" <EOL> perf = deepnet_pb2 . Metrics ( ) <EOL> perf . MergeFrom ( self . proto . performance_stats ) <EOL> perf . count = self . batchsize <EOL> tiny = self . tiny <EOL> temp = self . batchsize_temp <EOL> if self . loss_function == deepnet_pb2 . Layer . SQUARED_LOSS : <EOL> if get_deriv : <EOL> target = self . deriv <EOL> else : <EOL> target = self . statesize <EOL> if self . hyperparams . normalize_error : <EOL> self . data . sum ( axis = <NUM_LIT:0> , target = temp ) <EOL> temp . add ( self . tiny ) <EOL> self . data . div_by_row ( temp , target = target ) <EOL> self . state . div_by_row ( self . NN , target = self . expanded_batch ) <EOL> target . subtract ( self . expanded_batch ) <EOL> else : <EOL> self . data . sum ( axis = <NUM_LIT:0> , target = temp ) <EOL> temp . add ( self . tiny ) <EOL> self . state . div_by_row ( temp , target = target ) <EOL> target . subtract ( self . data ) <EOL> error = target . euclid_norm ( ) ** <NUM_LIT:2> <EOL> perf . error = error <EOL> else : <EOL> raise Exception ( '<STR_LIT>' ) <EOL> return perf <EOL> def GetSparsityDivisor ( self ) : <EOL> raise Exception ( '<STR_LIT>' ) <EOL> def CollectSufficientStatistics ( self , neg = False ) : <EOL> """<STR_LIT>""" <EOL> h = self . hyperparams <EOL> self . state . div_by_row ( self . NN ) <EOL> if not neg : <EOL> self . state . sum ( axis = <NUM_LIT:1> , target = self . suff_stats ) <EOL> else : <EOL> self . suff_stats . add_sums ( self . state , axis = <NUM_LIT:1> , mult = - <NUM_LIT:1.0> ) <EOL> self . state . mult_by_row ( self . NN ) </s>
