def _average_gradients(tower_grads):
 average_grads = []
 for grad_and_vars in zip(*tower_grads):
  grads = []
  for (g, _) in grad_and_vars:
   expanded_g = tf.expand_dims(g, 0)
   grads.append(expanded_g)
  grad = tf.concat(0, grads)
  grad = tf.reduce_mean(grad, 0)
  v = grad_and_vars[0][1]
  grad_and_var = (grad, v)
  average_grads.append(grad_and_var)
 return average_grads

def _average_gradients ( tower_grads ) : average_grads = [ ] for grad_and_vars in zip ( * tower_grads ) : grads = [ ] for ( g , _ ) in grad_and_vars : expanded_g = tf . expand_dims ( g , 0 ) grads . append ( expanded_g ) grad = tf . concat ( 0 , grads ) grad = tf . reduce_mean ( grad , 0 ) v = grad_and_vars [ 0 ] [ 1 ] grad_and_var = ( grad , v ) average_grads . append ( grad_and_var ) return average_grads
calculate the average gradient for each shared variable across all towers .