src/core/optimization/AMSGrad.java
public void optimize(Matrix matrix, Matrix matrixGradient) throws MatrixException, DynamicParamException {
        if (m == null) m = new HashMap<>();
        if (v == null) v = new HashMap<>();

        Matrix mM;
        if (m.containsKey(matrix)) mM = m.get(matrix);
        else m.put(matrix, mM = new DMatrix(matrix.getRows(), matrix.getColumns()));

        Matrix vM;
        if (v.containsKey(matrix)) vM = v.get(matrix);
        else v.put(matrix, vM = new DMatrix(matrix.getRows(), matrix.getColumns()));

        // mt = β1*mt − 1 + (1 − β1)*gt
        mM.multiply(beta1).add(matrixGradient.multiply(1 - beta1), mM);

        // vt = β2*vt − 1 + (1 − β2)*g2t
        Matrix vM_temp = vM.multiply(beta2).add(matrixGradient.power(2).multiply(1 - beta2));

        // vt = max(vt, vt-1)
        vM_temp.max(v.get(matrix), vM);

        // θt+1 = θt − η / (√^vt + ϵ) * mt
        double epsilon = 10E-8;
        matrix.subtract(mM.divide(vM.add(epsilon).apply(UnaryFunctionType.SQRT)).multiply(learningRate), matrix);
    }
/**
     * Optimizes single matrix (M) using calculated matrix gradient (dM).<br>
     * Matrix can be for example weight or bias matrix with gradient.<br>
     *
     * @param matrix matrix to be optimized.
     * @param matrixGradient matrix gradients for optimization step.
     * @throws MatrixException throws exception if matrix operation fails.
     * @throws DynamicParamException throws exception if parameter (params) setting fails.
     */
Optimizes single matrix (M) using calculated matrix gradient (dM).
Matrix can be for example weight or bias matrix with gradient.

@param matrix matrix to be optimized.
@param matrixGradient matrix gradients for optimization step.
@throws MatrixException throws exception if matrix operation fails.
@throws DynamicParamException throws exception if parameter (params) setting fails.
Optimizes single matrix ( M ) using calculated matrix gradient ( dM ) . Matrix can be for example weight or bias matrix with gradient . @param matrix matrix to be optimized . @param matrixGradient matrix gradients for optimization step . @throws MatrixException throws exception if matrix operation fails . @throws DynamicParamException throws exception if parameter ( params ) setting fails .
public void optimize(Matrix matrix, Matrix matrixGradient) throws MatrixException, DynamicParamException {
        if (m == null) m = new HashMap<>();
        if (v == null) v = new HashMap<>();
        Matrix mM;
        if (m.containsKey(matrix)) mM = m.get(matrix);
        else m.put(matrix, mM = new DMatrix(matrix.getRows(), matrix.getColumns()));
        Matrix vM;
        if (v.containsKey(matrix)) vM = v.get(matrix);
        else v.put(matrix, vM = new DMatrix(matrix.getRows(), matrix.getColumns()));
        mM.multiply(beta1).add(matrixGradient.multiply(1 - beta1), mM);
        Matrix vM_temp = vM.multiply(beta2).add(matrixGradient.power(2).multiply(1 - beta2));
        vM_temp.max(v.get(matrix), vM);
        double epsilon = 10E-8;
        matrix.subtract(mM.divide(vM.add(epsilon).apply(UnaryFunctionType.SQRT)).multiply(learningRate), matrix);
    }
public void optimize ( Matrix matrix , Matrix matrixGradient ) throws MatrixException , DynamicParamException { if ( m == null ) m = new HashMap < > ( ) ; if ( v == null ) v = new HashMap < > ( ) ; Matrix mM ; if ( m . containsKey ( matrix ) ) mM = m . get ( matrix ) ; else m . put ( matrix , mM = new DMatrix ( matrix . getRows ( ) , matrix . getColumns ( ) ) ) ; Matrix vM ; if ( v . containsKey ( matrix ) ) vM = v . get ( matrix ) ; else v . put ( matrix , vM = new DMatrix ( matrix . getRows ( ) , matrix . getColumns ( ) ) ) ; mM . multiply ( beta1 ) . add ( matrixGradient . multiply ( 1 - beta1 ) , mM ) ; Matrix vM_temp = vM . multiply ( beta2 ) . add ( matrixGradient . power ( 2 ) . multiply ( 1 - beta2 ) ) ; vM_temp . max ( v . get ( matrix ) , vM ) ; double epsilon = 10E-8 ; matrix . subtract ( mM . divide ( vM . add ( epsilon ) . apply ( UnaryFunctionType . SQRT ) ) . multiply ( learningRate ) , matrix ) ; }
Optimizes single matrix (M) using calculated matrix gradient (dM).<br>
 Matrix can be for example weight or bias matrix with gradient.<br>
Optimizes single matrix ( M ) using calculated matrix gradient ( dM ) . <br > Matrix can be for example weight or bias matrix with gradient . <br >
// mt = β1*mt − 1 + (1 − β1)*gt // vt = β2*vt − 1 + (1 − β2)*g2t // vt = max(vt, vt-1) // θt+1 = θt − η / (√^vt + ϵ) * mt