SiaNet/Layers/Core/BatchNormalization.cs
/// <summary>
/// Calculate the gradient of this layer function
/// </summary>
/// <param name="outputgrad">The calculated output grad from previous layer.</param>
Calculate the gradient of this layer function
Calculate the gradient of this layer function
public override void Backward(Tensor outputgrad)
        {
            Tensor mm = Params["mm"].Data;
            Tensor mv = Params["mv"].Data;
            var X_mu = Input.Data - mm;
            var var_inv = 1 / K.Sqrt(mv + K.Epsilon());
            var dbeta = K.Sum(outputgrad, 0);
            var dgamma = K.Sum(outputgrad * norm, 0);
            var dnorm = outputgrad * Params["gamma"].Data;
            var dvar = K.Sum(dnorm * mu.Data, 0) * K.Pow(-0.5f * (mv + K.Epsilon()), -3 / 2);
            var dmu = K.Sum(-1 * dnorm * var_inv, 0) + K.Sum(dvar * (-2 * X_mu), 0) / Input.Data.Shape[0];
            var dX = dnorm * var_inv + (dmu / Input.Data.Shape[0]) + (dvar * (2 / Input.Data.Shape[0] * X_mu));
            Input.Grad = dX;
            Params["beta"].Grad = dgamma;
            Params["gamma"].Grad = dgamma;
        }
public override void Backward ( Tensor outputgrad ) { Tensor mm = Params [ " " ] . Data ; Tensor mv = Params [ " " ] . Data ; var X_mu = Input . Data - mm ; var var_inv = 1 / K . Sqrt ( mv + K . Epsilon ( ) ) ; var dbeta = K . Sum ( outputgrad , 0 ) ; var dgamma = K . Sum ( outputgrad * norm , 0 ) ; var dnorm = outputgrad * Params [ " " ] . Data ; var dvar = K . Sum ( dnorm * mu . Data , 0 ) * K . Pow ( - 0.5f * ( mv + K . Epsilon ( ) ) , - 3 / 2 ) ; var dmu = K . Sum ( - 1 * dnorm * var_inv , 0 ) + K . Sum ( dvar * ( - 2 * X_mu ) , 0 ) / Input . Data . Shape [ 0 ] ; var dX = dnorm * var_inv + ( dmu / Input . Data . Shape [ 0 ] ) + ( dvar * ( 2 / Input . Data . Shape [ 0 ] * X_mu ) ) ; Input . Grad = dX ; Params [ " " ] . Grad = dgamma ; Params [ " " ] . Grad = dgamma ; }
Calculate the gradient of this layer function
Calculate the gradient of this layer function
outputgrad
Tensor
The calculated output grad from previous layer.
The calculated output grad from previous layer .