Undo the structure alterations to tower_grads done by pack_small_tensors.

  Args:
    tower_grads: List of List of (grad, var) tuples.
    packing: A dict generated by pack_small_tensors describing the changes
      it made to tower_grads.

  Returns:
    new_tower_grads: identical to tower_grads except that concatentations
      of small tensors have been split apart and returned to their original
      positions, paired with their original variables.

def unpack_small_tensors(tower_grads, packing):
    """Undo the structure alterations to tower_grads done by pack_small_tensors.

  Args:
    tower_grads: List of List of (grad, var) tuples.
    packing: A dict generated by pack_small_tensors describing the changes
      it made to tower_grads.

  Returns:
    new_tower_grads: identical to tower_grads except that concatentations
      of small tensors have been split apart and returned to their original
      positions, paired with their original variables.
  """
    if not packing:
        return tower_grads
    new_tower_grads = []
    num_devices = len(tower_grads)
    num_packed = len(packing.keys()) // num_devices
    for dev_idx, gv_list in enumerate(tower_grads):
        new_gv_list = gv_list[num_packed:]
        for i in xrange(0, num_packed):
            k = "%d:%d" % (dev_idx, i)
            gpt = packing[k]
            gv = unpack_grad_tuple(gv_list[i], gpt)
            for gi, idx in enumerate(gpt.indices):
                assert idx == gpt.indices[gi]
                new_gv_list.insert(idx, gv[gi])
        new_tower_grads.append(new_gv_list)
    return new_tower_grads