Stochastic Gradient Descent.

    Parameters
    ----------
    fun : callable
        the function to *minimize*, this must have the signature ``[obj,]``
        grad = fun(x, data, ...)`, where the ``eval_obj`` argument tells
        ``sgd`` if an objective function value is going to be returned by
        ``fun``.
    x0 : ndarray
        a sequence/1D array of initial values for the parameters to learn.
    data : ndarray
        a numpy array or sequence of data to input into ``fun``. This will
        be split along the first axis (axis=0), and then input into
        ``fun``.
    args : sequence, optional
        an optional sequence of arguments to give to fun.
    bounds : sequence, optional
        Bounds for variables, (min, max) pairs for each element in x, defining
        the bounds on that parameter.  Use None for one of min or max when
        there is no bound in that direction.
    batch_size : int, optional
        The number of observations in an SGD batch.
    maxiter : int, optional
        Number of mini-batch iterations before optimization terminates.
    updater : SGDUpdater, optional
        The type of gradient update to use, by default this is Adam.
    eval_obj : bool, optional
        This indicates whether or not ``fun`` also evaluates and returns
        the objective function value. If this is true, ``fun`` must return
        ``(obj, grad)`` and then a list of objective function values is
        also returned.
    random_state : int or RandomState, optional
        random seed

    Returns
    -------
    res : OptimizeResult
        x : narray
            the final result
        norms : list
            the list of gradient norms
        message : str
            the convergence condition ('maxiter reached' or error)
        objs : list
            the list of objective function evaluations if ``eval_obj``
            is True.
        fun : float
            the final objective function evaluation if ``eval_obj`` is
            True.

def sgd(fun, x0, data, args=(), bounds=None, batch_size=10, maxiter=5000,
        updater=None, eval_obj=False, random_state=None):
    """
    Stochastic Gradient Descent.

    Parameters
    ----------
    fun : callable
        the function to *minimize*, this must have the signature ``[obj,]``
        grad = fun(x, data, ...)`, where the ``eval_obj`` argument tells
        ``sgd`` if an objective function value is going to be returned by
        ``fun``.
    x0 : ndarray
        a sequence/1D array of initial values for the parameters to learn.
    data : ndarray
        a numpy array or sequence of data to input into ``fun``. This will
        be split along the first axis (axis=0), and then input into
        ``fun``.
    args : sequence, optional
        an optional sequence of arguments to give to fun.
    bounds : sequence, optional
        Bounds for variables, (min, max) pairs for each element in x, defining
        the bounds on that parameter.  Use None for one of min or max when
        there is no bound in that direction.
    batch_size : int, optional
        The number of observations in an SGD batch.
    maxiter : int, optional
        Number of mini-batch iterations before optimization terminates.
    updater : SGDUpdater, optional
        The type of gradient update to use, by default this is Adam.
    eval_obj : bool, optional
        This indicates whether or not ``fun`` also evaluates and returns
        the objective function value. If this is true, ``fun`` must return
        ``(obj, grad)`` and then a list of objective function values is
        also returned.
    random_state : int or RandomState, optional
        random seed

    Returns
    -------
    res : OptimizeResult
        x : narray
            the final result
        norms : list
            the list of gradient norms
        message : str
            the convergence condition ('maxiter reached' or error)
        objs : list
            the list of objective function evaluations if ``eval_obj``
            is True.
        fun : float
            the final objective function evaluation if ``eval_obj`` is
            True.
    """
    if updater is None:
        updater = Adam()

    # Make sure we aren't using a recycled updater
    updater.reset()

    N = _len_data(data)
    x = np.array(x0, copy=True, dtype=float)
    D = x.shape[0]

    # Make sure we have a valid batch size
    batch_size = min(batch_size, N)

    # Process bounds
    if bounds is not None:
        if len(bounds) != D:
            raise ValueError("The dimension of the bounds does not match x0!")

        lower, upper = zip(*map(normalize_bound, bounds))
        lower = np.array(lower)
        upper = np.array(upper)

    # Learning Records
    obj = None
    objs = []
    norms = []

    for batch in gen_batch(data, batch_size, maxiter, random_state):

        if not eval_obj:
            grad = fun(x, *chain(batch, args))
        else:
            obj, grad = fun(x, *chain(batch, args))
            objs.append(obj)

        norms.append(np.linalg.norm(grad))

        # Truncate gradients if bounded
        if bounds is not None:
            xlower = x <= lower
            grad[xlower] = np.minimum(grad[xlower], 0)
            xupper = x >= upper
            grad[xupper] = np.maximum(grad[xupper], 0)

        # perform update
        x = updater(x, grad)

        # Trucate steps if bounded
        if bounds is not None:
            x = np.clip(x, lower, upper)

    # Format results
    res = OptimizeResult(
        x=x,
        norms=norms,
        message='maxiter reached',
        fun=obj,
        objs=objs
    )

    return res