public static boolean checkGradientsPretrainLayer(Layer layer, double epsilon, double maxRelError,
                    double minAbsoluteError, boolean print, boolean exitOnFirstError, INDArray input, int rngSeed) {

        LayerWorkspaceMgr mgr = LayerWorkspaceMgr.noWorkspaces();

        //Basic sanity checks on input:
        if (epsilon <= 0.0 || epsilon > 0.1)
            throw new IllegalArgumentException("Invalid epsilon: expect epsilon in range (0,0.1], usually 1e-4 or so");
        if (maxRelError <= 0.0 || maxRelError > 0.25)
            throw new IllegalArgumentException("Invalid maxRelativeError: " + maxRelError);

        DataType dataType = DataTypeUtil.getDtypeFromContext();
        if (dataType != DataType.DOUBLE) {
            throw new IllegalStateException("Cannot perform gradient check: Datatype is not set to double precision ("
                            + "is: " + dataType + "). Double precision must be used for gradient checks. Set "
                            + "DataTypeUtil.setDTypeForContext(DataType.DOUBLE); before using GradientCheckUtil");
        }

        //Check network configuration:
        layer.setInput(input, LayerWorkspaceMgr.noWorkspaces());
        Nd4j.getRandom().setSeed(rngSeed);
        layer.computeGradientAndScore(mgr);
        Pair<Gradient, Double> gradAndScore = layer.gradientAndScore();

        Updater updater = UpdaterCreator.getUpdater(layer);
        updater.update(layer, gradAndScore.getFirst(), 0, 0, layer.batchSize(), LayerWorkspaceMgr.noWorkspaces());

        INDArray gradientToCheck = gradAndScore.getFirst().gradient().dup(); //need dup: gradients are a *view* of the full gradient array (which will change every time backprop is done)
        INDArray originalParams = layer.params().dup(); //need dup: params are a *view* of full parameters

        val nParams = originalParams.length();

        Map<String, INDArray> paramTable = layer.paramTable();
        List<String> paramNames = new ArrayList<>(paramTable.keySet());
        val paramEnds = new long[paramNames.size()];
        paramEnds[0] = paramTable.get(paramNames.get(0)).length();
        for (int i = 1; i < paramEnds.length; i++) {
            paramEnds[i] = paramEnds[i - 1] + paramTable.get(paramNames.get(i)).length();
        }


        int totalNFailures = 0;
        double maxError = 0.0;
        int currParamNameIdx = 0;

        INDArray params = layer.params(); //Assumption here: params is a view that we can modify in-place
        for (int i = 0; i < nParams; i++) {
            //Get param name
            if (i >= paramEnds[currParamNameIdx]) {
                currParamNameIdx++;
            }
            String paramName = paramNames.get(currParamNameIdx);

            //(w+epsilon): Do forward pass and score
            double origValue = params.getDouble(i);
            params.putScalar(i, origValue + epsilon);

            //TODO add a 'score' method that doesn't calculate gradients...
            Nd4j.getRandom().setSeed(rngSeed);
            layer.computeGradientAndScore(mgr);
            double scorePlus = layer.score();

            //(w-epsilon): Do forward pass and score
            params.putScalar(i, origValue - epsilon);
            Nd4j.getRandom().setSeed(rngSeed);
            layer.computeGradientAndScore(mgr);
            double scoreMinus = layer.score();

            //Reset original param value
            params.putScalar(i, origValue);

            //Calculate numerical parameter gradient:
            double scoreDelta = scorePlus - scoreMinus;

            double numericalGradient = scoreDelta / (2 * epsilon);
            if (Double.isNaN(numericalGradient))
                throw new IllegalStateException("Numerical gradient was NaN for parameter " + i + " of " + nParams);

            double backpropGradient = gradientToCheck.getDouble(i);
            //http://cs231n.github.io/neural-networks-3/#gradcheck
            //use mean centered
            double relError = Math.abs(backpropGradient - numericalGradient)
                            / (Math.abs(numericalGradient) + Math.abs(backpropGradient));
            if (backpropGradient == 0.0 && numericalGradient == 0.0)
                relError = 0.0; //Edge case: i.e., RNNs with time series length of 1.0

            if (relError > maxError)
                maxError = relError;
            if (relError > maxRelError || Double.isNaN(relError)) {
                double absError = Math.abs(backpropGradient - numericalGradient);
                if (absError < minAbsoluteError) {
                    log.info("Param " + i + " (" + paramName + ") passed: grad= " + backpropGradient
                                    + ", numericalGrad= " + numericalGradient + ", relError= " + relError
                                    + "; absolute error = " + absError + " < minAbsoluteError = " + minAbsoluteError);
                } else {
                    if (print)
                        log.info("Param " + i + " (" + paramName + ") FAILED: grad= " + backpropGradient
                                        + ", numericalGrad= " + numericalGradient + ", relError= " + relError
                                        + ", scorePlus=" + scorePlus + ", scoreMinus= " + scoreMinus + ", paramValue = " + origValue);
                    if (exitOnFirstError)
                        return false;
                    totalNFailures++;
                }
            } else if (print) {
                log.info("Param " + i + " (" + paramName + ") passed: grad= " + backpropGradient + ", numericalGrad= "
                                + numericalGradient + ", relError= " + relError);
            }
        }

        if (print) {
            val nPass = nParams - totalNFailures;
            log.info("GradientCheckUtil.checkGradients(): " + nParams + " params checked, " + nPass + " passed, "
                            + totalNFailures + " failed. Largest relative error = " + maxError);
        }

        return totalNFailures == 0;
    }
public static boolean checkGradientsPretrainLayer ( Layer layer , double epsilon , double maxRelError , double minAbsoluteError , boolean print , boolean exitOnFirstError , INDArray input , int rngSeed ) { LayerWorkspaceMgr mgr = LayerWorkspaceMgr . noWorkspaces ( ) ; if ( epsilon <= 0.0 || epsilon > 0.1 ) throw new IllegalArgumentException ( "Invalid epsilon: expect epsilon in range (0,0.1], usually 1e-4 or so" ) ; if ( maxRelError <= 0.0 || maxRelError > 0.25 ) throw new IllegalArgumentException ( "Invalid maxRelativeError: " + maxRelError ) ; DataType dataType = DataTypeUtil . getDtypeFromContext ( ) ; if ( dataType != DataType . DOUBLE ) { throw new IllegalStateException ( "Cannot perform gradient check: Datatype is not set to double precision (" + "is: " + dataType + "). Double precision must be used for gradient checks. Set " + "DataTypeUtil.setDTypeForContext(DataType.DOUBLE); before using GradientCheckUtil" ) ; } layer . setInput ( input , LayerWorkspaceMgr . noWorkspaces ( ) ) ; Nd4j . getRandom ( ) . setSeed ( rngSeed ) ; layer . computeGradientAndScore ( mgr ) ; Pair < Gradient , Double > gradAndScore = layer . gradientAndScore ( ) ; Updater updater = UpdaterCreator . getUpdater ( layer ) ; updater . update ( layer , gradAndScore . getFirst ( ) , 0 , 0 , layer . batchSize ( ) , LayerWorkspaceMgr . noWorkspaces ( ) ) ; INDArray gradientToCheck = gradAndScore . getFirst ( ) . gradient ( ) . dup ( ) ; INDArray originalParams = layer . params ( ) . dup ( ) ; val nParams = originalParams . length ( ) ; Map < String , INDArray > paramTable = layer . paramTable ( ) ; List < String > paramNames = new ArrayList < > ( paramTable . keySet ( ) ) ; val paramEnds = new long [ paramNames . size ( ) ] ; paramEnds [ 0 ] = paramTable . get ( paramNames . get ( 0 ) ) . length ( ) ; for ( int i = 1 ; i < paramEnds . length ; i ++ ) { paramEnds [ i ] = paramEnds [ i - 1 ] + paramTable . get ( paramNames . get ( i ) ) . length ( ) ; } int totalNFailures = 0 ; double maxError = 0.0 ; int currParamNameIdx = 0 ; INDArray params = layer . params ( ) ; for ( int i = 0 ; i < nParams ; i ++ ) { if ( i >= paramEnds [ currParamNameIdx ] ) { currParamNameIdx ++ ; } String paramName = paramNames . get ( currParamNameIdx ) ; double origValue = params . getDouble ( i ) ; params . putScalar ( i , origValue + epsilon ) ; Nd4j . getRandom ( ) . setSeed ( rngSeed ) ; layer . computeGradientAndScore ( mgr ) ; double scorePlus = layer . score ( ) ; params . putScalar ( i , origValue - epsilon ) ; Nd4j . getRandom ( ) . setSeed ( rngSeed ) ; layer . computeGradientAndScore ( mgr ) ; double scoreMinus = layer . score ( ) ; params . putScalar ( i , origValue ) ; double scoreDelta = scorePlus - scoreMinus ; double numericalGradient = scoreDelta / ( 2 * epsilon ) ; if ( Double . isNaN ( numericalGradient ) ) throw new IllegalStateException ( "Numerical gradient was NaN for parameter " + i + " of " + nParams ) ; double backpropGradient = gradientToCheck . getDouble ( i ) ; double relError = Math . abs ( backpropGradient - numericalGradient ) / ( Math . abs ( numericalGradient ) + Math . abs ( backpropGradient ) ) ; if ( backpropGradient == 0.0 && numericalGradient == 0.0 ) relError = 0.0 ; if ( relError > maxError ) maxError = relError ; if ( relError > maxRelError || Double . isNaN ( relError ) ) { double absError = Math . abs ( backpropGradient - numericalGradient ) ; if ( absError < minAbsoluteError ) { log . info ( "Param " + i + " (" + paramName + ") passed: grad= " + backpropGradient + ", numericalGrad= " + numericalGradient + ", relError= " + relError + "; absolute error = " + absError + " < minAbsoluteError = " + minAbsoluteError ) ; } else { if ( print ) log . info ( "Param " + i + " (" + paramName + ") FAILED: grad= " + backpropGradient + ", numericalGrad= " + numericalGradient + ", relError= " + relError + ", scorePlus=" + scorePlus + ", scoreMinus= " + scoreMinus + ", paramValue = " + origValue ) ; if ( exitOnFirstError ) return false ; totalNFailures ++ ; } } else if ( print ) { log . info ( "Param " + i + " (" + paramName + ") passed: grad= " + backpropGradient + ", numericalGrad= " + numericalGradient + ", relError= " + relError ) ; } } if ( print ) { val nPass = nParams - totalNFailures ; log . info ( "GradientCheckUtil.checkGradients(): " + nParams + " params checked, " + nPass + " passed, " + totalNFailures + " failed. Largest relative error = " + maxError ) ; } return totalNFailures == 0 ; }
Check backprop gradients for a pretrain layer

NOTE: gradient checking pretrain layers can be difficult...