src/core/optimization/RAdam.java
public void optimize(Matrix matrix, Matrix matrixGradient) throws MatrixException, DynamicParamException {
        if (iterations == null) iterations = new HashMap<>();
        if (m == null) m = new HashMap<>();
        if (v == null) v = new HashMap<>();

        int iteration;
        if (iterations.containsKey(matrix)) iterations.put(matrix, iteration = iterations.get(matrix) + 1);
        else iterations.put(matrix, iteration = 1);

        Matrix vM = null;
        if (v.containsKey(matrix)) vM = v.get(matrix);

        Matrix mM = null;
        if (m.containsKey(matrix)) mM = m.get(matrix);

        vM = vM == null ? matrixGradient.power(2).multiply(1 - beta2) : vM.multiply(beta2).add(matrixGradient.power(2).multiply(1 - beta2));
        v.put(matrix, vM);

        mM = mM == null ? matrixGradient.multiply(1 - beta1) : mM.multiply(beta1).add(matrixGradient.multiply(1 - beta1));
        m.put(matrix, mM);

        double beta1Iteration = Math.pow(beta1, iteration);
        double beta2Iteration = Math.pow(beta2, iteration);

        Matrix mMhat = mM.divide(1 - beta1Iteration);

        double stepSize = learningRate;
        double pt = pinf - 2 * iteration * beta2Iteration / (1 - beta2Iteration);
        if (pt > 4) {
            stepSize *=  Math.sqrt((1 - beta2Iteration) * ((pt - 4) * (pt - 2) * pinf) / ((pinf - 4) * (pinf - 2) * pt));
            double epsilon = 10E-8;
            matrix.subtract(mMhat.divide(vM.apply(UnaryFunctionType.SQRT).add(epsilon)).multiply(stepSize), matrix);
        }
        else {
            matrix.subtract(mMhat.multiply(stepSize), matrix);
        }
    }
/**
     * Optimizes single matrix (M) using calculated matrix gradient (dM).<br>
     * Matrix can be for example weight or bias matrix with gradient.<br>
     *
     * @param matrix matrix to be optimized.
     * @param matrixGradient matrix gradients for optimization step.
     * @throws MatrixException throws exception if matrix operation fails.
     * @throws DynamicParamException throws exception if parameter (params) setting fails.
     */
Optimizes single matrix (M) using calculated matrix gradient (dM).
Matrix can be for example weight or bias matrix with gradient.

@param matrix matrix to be optimized.
@param matrixGradient matrix gradients for optimization step.
@throws MatrixException throws exception if matrix operation fails.
@throws DynamicParamException throws exception if parameter (params) setting fails.
Optimizes single matrix ( M ) using calculated matrix gradient ( dM ) . Matrix can be for example weight or bias matrix with gradient . @param matrix matrix to be optimized . @param matrixGradient matrix gradients for optimization step . @throws MatrixException throws exception if matrix operation fails . @throws DynamicParamException throws exception if parameter ( params ) setting fails .
public void optimize(Matrix matrix, Matrix matrixGradient) throws MatrixException, DynamicParamException {
        if (iterations == null) iterations = new HashMap<>();
        if (m == null) m = new HashMap<>();
        if (v == null) v = new HashMap<>();
        int iteration;
        if (iterations.containsKey(matrix)) iterations.put(matrix, iteration = iterations.get(matrix) + 1);
        else iterations.put(matrix, iteration = 1);
        Matrix vM = null;
        if (v.containsKey(matrix)) vM = v.get(matrix);
        Matrix mM = null;
        if (m.containsKey(matrix)) mM = m.get(matrix);
        vM = vM == null ? matrixGradient.power(2).multiply(1 - beta2) : vM.multiply(beta2).add(matrixGradient.power(2).multiply(1 - beta2));
        v.put(matrix, vM);
        mM = mM == null ? matrixGradient.multiply(1 - beta1) : mM.multiply(beta1).add(matrixGradient.multiply(1 - beta1));
        m.put(matrix, mM);
        double beta1Iteration = Math.pow(beta1, iteration);
        double beta2Iteration = Math.pow(beta2, iteration);
        Matrix mMhat = mM.divide(1 - beta1Iteration);
        double stepSize = learningRate;
        double pt = pinf - 2 * iteration * beta2Iteration / (1 - beta2Iteration);
        if (pt > 4) {
            stepSize *=  Math.sqrt((1 - beta2Iteration) * ((pt - 4) * (pt - 2) * pinf) / ((pinf - 4) * (pinf - 2) * pt));
            double epsilon = 10E-8;
            matrix.subtract(mMhat.divide(vM.apply(UnaryFunctionType.SQRT).add(epsilon)).multiply(stepSize), matrix);
        }
        else {
            matrix.subtract(mMhat.multiply(stepSize), matrix);
        }
    }
public void optimize ( Matrix matrix , Matrix matrixGradient ) throws MatrixException , DynamicParamException { if ( iterations == null ) iterations = new HashMap < > ( ) ; if ( m == null ) m = new HashMap < > ( ) ; if ( v == null ) v = new HashMap < > ( ) ; int iteration ; if ( iterations . containsKey ( matrix ) ) iterations . put ( matrix , iteration = iterations . get ( matrix ) + 1 ) ; else iterations . put ( matrix , iteration = 1 ) ; Matrix vM = null ; if ( v . containsKey ( matrix ) ) vM = v . get ( matrix ) ; Matrix mM = null ; if ( m . containsKey ( matrix ) ) mM = m . get ( matrix ) ; vM = vM == null ? matrixGradient . power ( 2 ) . multiply ( 1 - beta2 ) : vM . multiply ( beta2 ) . add ( matrixGradient . power ( 2 ) . multiply ( 1 - beta2 ) ) ; v . put ( matrix , vM ) ; mM = mM == null ? matrixGradient . multiply ( 1 - beta1 ) : mM . multiply ( beta1 ) . add ( matrixGradient . multiply ( 1 - beta1 ) ) ; m . put ( matrix , mM ) ; double beta1Iteration = Math . pow ( beta1 , iteration ) ; double beta2Iteration = Math . pow ( beta2 , iteration ) ; Matrix mMhat = mM . divide ( 1 - beta1Iteration ) ; double stepSize = learningRate ; double pt = pinf - 2 * iteration * beta2Iteration / ( 1 - beta2Iteration ) ; if ( pt > 4 ) { stepSize *= Math . sqrt ( ( 1 - beta2Iteration ) * ( ( pt - 4 ) * ( pt - 2 ) * pinf ) / ( ( pinf - 4 ) * ( pinf - 2 ) * pt ) ) ; double epsilon = 10E-8 ; matrix . subtract ( mMhat . divide ( vM . apply ( UnaryFunctionType . SQRT ) . add ( epsilon ) ) . multiply ( stepSize ) , matrix ) ; } else { matrix . subtract ( mMhat . multiply ( stepSize ) , matrix ) ; } }
Optimizes single matrix (M) using calculated matrix gradient (dM).<br>
 Matrix can be for example weight or bias matrix with gradient.<br>
Optimizes single matrix ( M ) using calculated matrix gradient ( dM ) . <br > Matrix can be for example weight or bias matrix with gradient . <br >