def _apply_gradients_and_copy(self, opt, raw_grad_list, ps_var_grads):
        """
        Apply averaged gradients to ps vars, and then copy the updated
        variables back to each tower.

        Args:
            raw_grad_list: Ngpu x Nvar x 2 gradient list from all towers
            ps_var_grads: Nvar x 2 (grad, ps_var)

        Returns:
            list of copy ops
        """
        # TODO do this for variables together?
        with tf.name_scope('apply_gradients'):
            var_update_ops = []
            for vid, (g, v) in enumerate(ps_var_grads):
                # TODO do we put momentum variables into local or global?
                apply_gradient_op = opt.apply_gradients([(g, v)])
                barrier = self._add_sync_queues_and_barrier(
                    'param_update_barrier_{}'.format(vid), [apply_gradient_op])
                with tf.control_dependencies([barrier]), \
                        tf.device(self.cpu_device):
                    updated_value = v.read_value()
                    for towerid in range(self.nr_gpu):
                        var_update_ops.append(
                            raw_grad_list[towerid][vid][1].assign(updated_value))
            return var_update_ops
def _apply_gradients_and_copy ( self , opt , raw_grad_list , ps_var_grads ) : with tf . name_scope ( 'apply_gradients' ) : var_update_ops = [ ] for vid , ( g , v ) in enumerate ( ps_var_grads ) : apply_gradient_op = opt . apply_gradients ( [ ( g , v ) ] ) barrier = self . _add_sync_queues_and_barrier ( 'param_update_barrier_{}' . format ( vid ) , [ apply_gradient_op ] ) with tf . control_dependencies ( [ barrier ] ) , tf . device ( self . cpu_device ) : updated_value = v . read_value ( ) for towerid in range ( self . nr_gpu ) : var_update_ops . append ( raw_grad_list [ towerid ] [ vid ] [ 1 ] . assign ( updated_value ) ) return var_update_ops
Apply averaged gradients to ps vars, and then copy the updated
        variables back to each tower.

        Args:
            raw_grad_list: Ngpu x Nvar x 2 gradient list from all towers
            ps_var_grads: Nvar x 2 (grad, ps_var)

        Returns:
            list of copy ops