<gh_stars>10-100
import numpy as np
import tensorflow as tf


def forward_mode_gradients(ys, xs, grad_xs=None, stop_gradients=None,
                           colocate_gradients_with_ops=True):
    """Compute forward-mode gradients."""
    # See b/37888268.

    # This version of forward-mode autodiff is based on code by <NAME>
    # and handles list arguments and certain special cases such as when the
    # ys doesn't depend on one or more of the xs, and when tf.IndexedSlices are
    # generated by the first tf.gradients call.
    # copy from https://github.com/tensorflow/kfac/blob/master/kfac/python/ops/utils.py
    us = [tf.zeros_like(y) + float("nan") for y in ys]
    # us = [tf.zeros_like(y, dtype=dtype) + float("nan") for y in ys]
    dydxs = tf.gradients(ys, xs, grad_ys=us, stop_gradients=stop_gradients,
                         colocate_gradients_with_ops=colocate_gradients_with_ops)

    # Deal with strange types that tf.gradients returns but can't
    # deal with.
    dydxs = [
        tf.convert_to_tensor(dydx) if isinstance(
            dydx, tf.IndexedSlices) else dydx
        for dydx in dydxs
    ]
    dydxs = [
        tf.zeros_like(x) if dydx is None else dydx for x, dydx in zip(xs, dydxs)
    ]

    dysdx = tf.gradients(dydxs, us, grad_ys=grad_xs,
                         colocate_gradients_with_ops=colocate_gradients_with_ops)

    return dysdx


def column_to_tensors(tensors_template, colvec):
    """Converts a column vector back to the shape of the given template.
  Args:
    tensors_template: A tensor or list of tensors.
    colvec: A 2d column vector with the same shape as the value of
        tensors_to_column(tensors_template).
  Returns:
    X, where X is tensor or list of tensors with the properties:
     1) tensors_to_column(X) = colvec
     2) X (or its elements) have the same shape as tensors_template (or its
        elements)
  """
    # copy from https://github.com/tensorflow/kfac/blob/master/kfac/python/ops/utils.py
    if isinstance(tensors_template, (tuple, list)):
        offset = 0
        tensors = []
        for tensor_template in tensors_template:
            sz = np.prod(tensor_template.shape.as_list(), dtype=np.int32)
            tensor = tf.reshape(colvec[offset:(offset + sz)], tensor_template.shape)
            tensors.append(tensor)
            offset += sz

        tensors = tuple(tensors)
    else:
        tensors = tf.reshape(colvec, tensors_template.shape)

    return tensors


def tensors_to_column(tensors):
    """Converts a tensor or list of tensors to a column vector.
  Args:
    tensors: A tensor or list of tensors.
  Returns:
    The tensors reshaped into vectors and stacked on top of each other.
  """
    # copy from https://github.com/tensorflow/kfac/blob/master/kfac/python/ops/utils.py
    if isinstance(tensors, (tuple, list)):
        return tf.concat(
            tuple(tf.reshape(tensor, [-1, 1]) for tensor in tensors), axis=0)
    else:
        return tf.reshape(tensors, [-1, 1])


def tensors_to_matrix(tensors, first_dim=1):
    """Converts a tensor or list of tensors to a column vector.
    Args:
      tensors: A tensor or list of tensors.
    Returns:
      The tensors reshaped into vectors and stacked on top of each other.
    """
    if isinstance(tensors, (tuple, list)):
        return tf.concat(
            tuple(tf.reshape(tensor, [first_dim, -1]) for tensor in tensors), axis=1)
    else:
        return tf.reshape(tensors, [first_dim, -1])

import numpy as np
import tensorflow as tf


def forward_mode_gradients(ys, xs, grad_xs=None, stop_gradients=None,
                           colocate_gradients_with_ops=True):
    """Compute forward-mode gradients."""
    # See b/37888268.

    # This version of forward-mode autodiff is based on code by <NAME>
    # and handles list arguments and certain special cases such as when the
    # ys doesn't depend on one or more of the xs, and when tf.IndexedSlices are
    # generated by the first tf.gradients call.
    # copy from https://github.com/tensorflow/kfac/blob/master/kfac/python/ops/utils.py
    us = [tf.zeros_like(y) + float("nan") for y in ys]
    # us = [tf.zeros_like(y, dtype=dtype) + float("nan") for y in ys]
    dydxs = tf.gradients(ys, xs, grad_ys=us, stop_gradients=stop_gradients,
                         colocate_gradients_with_ops=colocate_gradients_with_ops)

    # Deal with strange types that tf.gradients returns but can't
    # deal with.
    dydxs = [
        tf.convert_to_tensor(dydx) if isinstance(
            dydx, tf.IndexedSlices) else dydx
        for dydx in dydxs
    ]
    dydxs = [
        tf.zeros_like(x) if dydx is None else dydx for x, dydx in zip(xs, dydxs)
    ]

    dysdx = tf.gradients(dydxs, us, grad_ys=grad_xs,
                         colocate_gradients_with_ops=colocate_gradients_with_ops)

    return dysdx


def column_to_tensors(tensors_template, colvec):
    """Converts a column vector back to the shape of the given template.
  Args:
    tensors_template: A tensor or list of tensors.
    colvec: A 2d column vector with the same shape as the value of
        tensors_to_column(tensors_template).
  Returns:
    X, where X is tensor or list of tensors with the properties:
     1) tensors_to_column(X) = colvec
     2) X (or its elements) have the same shape as tensors_template (or its
        elements)
  """
    # copy from https://github.com/tensorflow/kfac/blob/master/kfac/python/ops/utils.py
    if isinstance(tensors_template, (tuple, list)):
        offset = 0
        tensors = []
        for tensor_template in tensors_template:
            sz = np.prod(tensor_template.shape.as_list(), dtype=np.int32)
            tensor = tf.reshape(colvec[offset:(offset + sz)], tensor_template.shape)
            tensors.append(tensor)
            offset += sz

        tensors = tuple(tensors)
    else:
        tensors = tf.reshape(colvec, tensors_template.shape)

    return tensors


def tensors_to_column(tensors):
    """Converts a tensor or list of tensors to a column vector.
  Args:
    tensors: A tensor or list of tensors.
  Returns:
    The tensors reshaped into vectors and stacked on top of each other.
  """
    # copy from https://github.com/tensorflow/kfac/blob/master/kfac/python/ops/utils.py
    if isinstance(tensors, (tuple, list)):
        return tf.concat(
            tuple(tf.reshape(tensor, [-1, 1]) for tensor in tensors), axis=0)
    else:
        return tf.reshape(tensors, [-1, 1])


def tensors_to_matrix(tensors, first_dim=1):
    """Converts a tensor or list of tensors to a column vector.
    Args:
      tensors: A tensor or list of tensors.
    Returns:
      The tensors reshaped into vectors and stacked on top of each other.
    """
    if isinstance(tensors, (tuple, list)):
        return tf.concat(
            tuple(tf.reshape(tensor, [first_dim, -1]) for tensor in tensors), axis=1)
    else:
        return tf.reshape(tensors, [first_dim, -1])
Compute forward-mode gradients. # See b/37888268. # This version of forward-mode autodiff is based on code by <NAME> # and handles list arguments and certain special cases such as when the # ys doesn't depend on one or more of the xs, and when tf.IndexedSlices are # generated by the first tf.gradients call. # copy from https://github.com/tensorflow/kfac/blob/master/kfac/python/ops/utils.py # us = [tf.zeros_like(y, dtype=dtype) + float("nan") for y in ys] # Deal with strange types that tf.gradients returns but can't # deal with. Converts a column vector back to the shape of the given template.   Args:     tensors_template: A tensor or list of tensors.     colvec: A 2d column vector with the same shape as the value of         tensors_to_column(tensors_template).   Returns:     X, where X is tensor or list of tensors with the properties:      1) tensors_to_column(X) = colvec      2) X (or its elements) have the same shape as tensors_template (or its         elements) # copy from https://github.com/tensorflow/kfac/blob/master/kfac/python/ops/utils.py Converts a tensor or list of tensors to a column vector.   Args:     tensors: A tensor or list of tensors.   Returns:     The tensors reshaped into vectors and stacked on top of each other. # copy from https://github.com/tensorflow/kfac/blob/master/kfac/python/ops/utils.py Converts a tensor or list of tensors to a column vector.     Args:       tensors: A tensor or list of tensors.     Returns:       The tensors reshaped into vectors and stacked on top of each other.