@scopes.add_arg_scope
def batch_norm(inputs, decay=0.999, center=True, scale=False, epsilon=0.001, moving_vars='moving_vars', activation=None, is_training=True, trainable=True, restore=True, scope=None, reuse=None):
 inputs_shape = inputs.get_shape()
 with tf.variable_scope(scope, 'BatchNorm', [inputs], reuse=reuse):
  axis = list(range((len(inputs_shape) - 1)))
  params_shape = inputs_shape[(-1):]
  (beta, gamma) = (None, None)
  if center:
   beta = variables.variable('beta', params_shape, initializer=tf.zeros_initializer(), trainable=trainable, restore=restore)
  if scale:
   gamma = variables.variable('gamma', params_shape, initializer=tf.ones_initializer(), trainable=trainable, restore=restore)
  moving_collections = [moving_vars, tf.GraphKeys.MOVING_AVERAGE_VARIABLES]
  moving_mean = variables.variable('moving_mean', params_shape, initializer=tf.zeros_initializer(), trainable=False, restore=restore, collections=moving_collections)
  moving_variance = variables.variable('moving_variance', params_shape, initializer=tf.ones_initializer(), trainable=False, restore=restore, collections=moving_collections)
  if is_training:
   (mean, variance) = tf.nn.moments(inputs, axis)
   update_moving_mean = moving_averages.assign_moving_average(moving_mean, mean, decay)
   tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_mean)
   update_moving_variance = moving_averages.assign_moving_average(moving_variance, variance, decay)
   tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_variance)
  else:
   mean = moving_mean
   variance = moving_variance
  outputs = tf.nn.batch_normalization(inputs, mean, variance, beta, gamma, epsilon)
  outputs.set_shape(inputs.get_shape())
  if activation:
   outputs = activation(outputs)
  return outputs

@ scopes . add_arg_scope def batch_norm ( inputs , decay = 0.999 , center = True , scale = False , epsilon = 0.001 , moving_vars = 'moving_vars' , activation = None , is_training = True , trainable = True , restore = True , scope = None , reuse = None ) : inputs_shape = inputs . get_shape ( ) with tf . variable_scope ( scope , 'BatchNorm' , [ inputs ] , reuse = reuse ) : axis = list ( range ( ( len ( inputs_shape ) - 1 ) ) ) params_shape = inputs_shape [ ( - 1 ) : ] ( beta , gamma ) = ( None , None ) if center : beta = variables . variable ( 'beta' , params_shape , initializer = tf . zeros_initializer ( ) , trainable = trainable , restore = restore ) if scale : gamma = variables . variable ( 'gamma' , params_shape , initializer = tf . ones_initializer ( ) , trainable = trainable , restore = restore ) moving_collections = [ moving_vars , tf . GraphKeys . MOVING_AVERAGE_VARIABLES ] moving_mean = variables . variable ( 'moving_mean' , params_shape , initializer = tf . zeros_initializer ( ) , trainable = False , restore = restore , collections = moving_collections ) moving_variance = variables . variable ( 'moving_variance' , params_shape , initializer = tf . ones_initializer ( ) , trainable = False , restore = restore , collections = moving_collections ) if is_training : ( mean , variance ) = tf . nn . moments ( inputs , axis ) update_moving_mean = moving_averages . assign_moving_average ( moving_mean , mean , decay ) tf . add_to_collection ( UPDATE_OPS_COLLECTION , update_moving_mean ) update_moving_variance = moving_averages . assign_moving_average ( moving_variance , variance , decay ) tf . add_to_collection ( UPDATE_OPS_COLLECTION , update_moving_variance ) else : mean = moving_mean variance = moving_variance outputs = tf . nn . batch_normalization ( inputs , mean , variance , beta , gamma , epsilon ) outputs . set_shape ( inputs . get_shape ( ) ) if activation : outputs = activation ( outputs ) return outputs
apply batch normalization to an existing layer .