src/core/reinforcement/value/AbstractValueFunction.java
private void updateValue(TreeSet<StateTransition> stateTransitions) throws MatrixException, NeuralNetworkException {
        if (stateTransitions == null) return;
        for (StateTransition stateTransition : stateTransitions.descendingSet()) {
            updateValue(stateTransition);
            stateTransition.tdTarget = stateTransition.reward + (stateTransition.isFinalState() ? 0 : gamma * ((1 - lambda) * getValue(stateTransition.nextStateTransition) + lambda * getTargetValue(stateTransition.nextStateTransition)));
            stateTransition.tdError = stateTransition.tdTarget - getValue(stateTransition);
            stateTransition.advantage = stateTransition.tdError;
            averageTDError = averageTDError == Double.NEGATIVE_INFINITY ? stateTransition.tdError : 0.99 * averageTDError + 0.01 * stateTransition.tdError;
            if (tdErrorPrintCycle > 0 && ++tdErrorPrintCount % tdErrorPrintCycle == 0) System.out.println("Average TD error: " + averageTDError);
        }

        updateBaseline(stateTransitions);
    }
/**
     * Updates value of state transitions.
     *
     * @param stateTransitions state transitions.
     * @throws MatrixException throws exception if matrix operation fails.
     * @throws NeuralNetworkException throws exception if neural network operation fails.
     */
Updates value of state transitions.
@param stateTransitions state transitions.
@throws MatrixException throws exception if matrix operation fails.
@throws NeuralNetworkException throws exception if neural network operation fails.
Updates value of state transitions . @param stateTransitions state transitions . @throws MatrixException throws exception if matrix operation fails . @throws NeuralNetworkException throws exception if neural network operation fails .
private void updateValue(TreeSet<StateTransition> stateTransitions) throws MatrixException, NeuralNetworkException {
        if (stateTransitions == null) return;
        for (StateTransition stateTransition : stateTransitions.descendingSet()) {
            updateValue(stateTransition);
            stateTransition.tdTarget = stateTransition.reward + (stateTransition.isFinalState() ? 0 : gamma * ((1 - lambda) * getValue(stateTransition.nextStateTransition) + lambda * getTargetValue(stateTransition.nextStateTransition)));
            stateTransition.tdError = stateTransition.tdTarget - getValue(stateTransition);
            stateTransition.advantage = stateTransition.tdError;
            averageTDError = averageTDError == Double.NEGATIVE_INFINITY ? stateTransition.tdError : 0.99 * averageTDError + 0.01 * stateTransition.tdError;
            if (tdErrorPrintCycle > 0 && ++tdErrorPrintCount % tdErrorPrintCycle == 0) System.out.println("Average TD error: " + averageTDError);
        }
        updateBaseline(stateTransitions);
    }
private void updateValue ( TreeSet < StateTransition > stateTransitions ) throws MatrixException , NeuralNetworkException { if ( stateTransitions == null ) return ; for ( StateTransition stateTransition : stateTransitions . descendingSet ( ) ) { updateValue ( stateTransition ) ; stateTransition . tdTarget = stateTransition . reward + ( stateTransition . isFinalState ( ) ? 0 : gamma * ( ( 1 - lambda ) * getValue ( stateTransition . nextStateTransition ) + lambda * getTargetValue ( stateTransition . nextStateTransition ) ) ) ; stateTransition . tdError = stateTransition . tdTarget - getValue ( stateTransition ) ; stateTransition . advantage = stateTransition . tdError ; averageTDError = averageTDError == Double . NEGATIVE_INFINITY ? stateTransition . tdError : 0.99 * averageTDError + 0.01 * stateTransition . tdError ; if ( tdErrorPrintCycle > 0 && ++ tdErrorPrintCount % tdErrorPrintCycle == 0 ) System . out . println ( "Average TD error: " + averageTDError ) ; } updateBaseline ( stateTransitions ) ; }
Updates value of state transitions.
Updates value of state transitions .