src/core/optimization/Adamax.java
public void optimize(Matrix matrix, Matrix matrixGradient) throws MatrixException, DynamicParamException {
        if (iterations == null) iterations = new HashMap<>();
        if (m == null) m = new HashMap<>();
        if (v == null) v = new HashMap<>();

        int iteration;
        if (iterations.containsKey(matrix)) iterations.put(matrix, iteration = iterations.get(matrix) + 1);
        else iterations.put(matrix, iteration = 1);

        Matrix mM;
        if (m.containsKey(matrix)) mM = m.get(matrix);
        else m.put(matrix, mM = new DMatrix(matrix.getRows(), matrix.getColumns()));

        Matrix vM;
        if (v.containsKey(matrix)) vM = v.get(matrix);
        else v.put(matrix, vM = new DMatrix(matrix.getRows(), matrix.getColumns()));

        Matrix matrixGradientAbs = matrixGradient.apply(UnaryFunctionType.ABS);

        // mt = β1*mt−1 + (1 − β1)*gt
        mM.multiply(beta1).add(matrixGradient.multiply(1 - beta1), mM);

        // vt = β2*vt−1 + (1 − β2)*|gt|
        vM.multiply(beta2).add(matrixGradientAbs.multiply(1 - beta2), vM);

        // mt = mt / (1 − βt1)
        Matrix mM_hat = mM.divide(1 - Math.pow(beta1, iteration));

        // ut= max(β2⋅vt−1,|gt|)
        Matrix uM = (vM.multiply(beta2)).max(matrixGradientAbs);

        // θt+1 = θt − η / ut * mt
        matrix.subtract(mM_hat.divide(uM).multiply(learningRate), matrix);
    }
/**
     * Optimizes single matrix (M) using calculated matrix gradient (dM).<br>
     * Matrix can be for example weight or bias matrix with gradient.<br>
     *
     * @param matrix matrix to be optimized.
     * @param matrixGradient matrix gradients for optimization step.
     * @throws MatrixException throws exception if matrix operation fails.
     * @throws DynamicParamException throws exception if parameter (params) setting fails.
     */
Optimizes single matrix (M) using calculated matrix gradient (dM).
Matrix can be for example weight or bias matrix with gradient.

@param matrix matrix to be optimized.
@param matrixGradient matrix gradients for optimization step.
@throws MatrixException throws exception if matrix operation fails.
@throws DynamicParamException throws exception if parameter (params) setting fails.
Optimizes single matrix ( M ) using calculated matrix gradient ( dM ) . Matrix can be for example weight or bias matrix with gradient . @param matrix matrix to be optimized . @param matrixGradient matrix gradients for optimization step . @throws MatrixException throws exception if matrix operation fails . @throws DynamicParamException throws exception if parameter ( params ) setting fails .
public void optimize(Matrix matrix, Matrix matrixGradient) throws MatrixException, DynamicParamException {
        if (iterations == null) iterations = new HashMap<>();
        if (m == null) m = new HashMap<>();
        if (v == null) v = new HashMap<>();
        int iteration;
        if (iterations.containsKey(matrix)) iterations.put(matrix, iteration = iterations.get(matrix) + 1);
        else iterations.put(matrix, iteration = 1);
        Matrix mM;
        if (m.containsKey(matrix)) mM = m.get(matrix);
        else m.put(matrix, mM = new DMatrix(matrix.getRows(), matrix.getColumns()));
        Matrix vM;
        if (v.containsKey(matrix)) vM = v.get(matrix);
        else v.put(matrix, vM = new DMatrix(matrix.getRows(), matrix.getColumns()));
        Matrix matrixGradientAbs = matrixGradient.apply(UnaryFunctionType.ABS);
        mM.multiply(beta1).add(matrixGradient.multiply(1 - beta1), mM);
        vM.multiply(beta2).add(matrixGradientAbs.multiply(1 - beta2), vM);
        Matrix mM_hat = mM.divide(1 - Math.pow(beta1, iteration));
        Matrix uM = (vM.multiply(beta2)).max(matrixGradientAbs);
        matrix.subtract(mM_hat.divide(uM).multiply(learningRate), matrix);
    }
public void optimize ( Matrix matrix , Matrix matrixGradient ) throws MatrixException , DynamicParamException { if ( iterations == null ) iterations = new HashMap < > ( ) ; if ( m == null ) m = new HashMap < > ( ) ; if ( v == null ) v = new HashMap < > ( ) ; int iteration ; if ( iterations . containsKey ( matrix ) ) iterations . put ( matrix , iteration = iterations . get ( matrix ) + 1 ) ; else iterations . put ( matrix , iteration = 1 ) ; Matrix mM ; if ( m . containsKey ( matrix ) ) mM = m . get ( matrix ) ; else m . put ( matrix , mM = new DMatrix ( matrix . getRows ( ) , matrix . getColumns ( ) ) ) ; Matrix vM ; if ( v . containsKey ( matrix ) ) vM = v . get ( matrix ) ; else v . put ( matrix , vM = new DMatrix ( matrix . getRows ( ) , matrix . getColumns ( ) ) ) ; Matrix matrixGradientAbs = matrixGradient . apply ( UnaryFunctionType . ABS ) ; mM . multiply ( beta1 ) . add ( matrixGradient . multiply ( 1 - beta1 ) , mM ) ; vM . multiply ( beta2 ) . add ( matrixGradientAbs . multiply ( 1 - beta2 ) , vM ) ; Matrix mM_hat = mM . divide ( 1 - Math . pow ( beta1 , iteration ) ) ; Matrix uM = ( vM . multiply ( beta2 ) ) . max ( matrixGradientAbs ) ; matrix . subtract ( mM_hat . divide ( uM ) . multiply ( learningRate ) , matrix ) ; }
Optimizes single matrix (M) using calculated matrix gradient (dM).<br>
 Matrix can be for example weight or bias matrix with gradient.<br>
Optimizes single matrix ( M ) using calculated matrix gradient ( dM ) . <br > Matrix can be for example weight or bias matrix with gradient . <br >
// mt = β1*mt−1 + (1 − β1)*gt // vt = β2*vt−1 + (1 − β2)*|gt| // mt = mt / (1 − βt1) // ut= max(β2⋅vt−1,|gt|) // θt+1 = θt − η / ut * mt