public static bool VerifyParametersGradient(LayerBase layer, int batchSize = 1)
{
Tensor.SetOpMode(Tensor.OpMode.CPU);
var inputs = GenerateInputsForLayer(layer, batchSize);
var output = layer.FeedForward(inputs);
var outputGradient = new Tensor(output.Shape);
outputGradient.FillWithValue(1);
layer.BackProp(outputGradient);
var paramsAndGrads = layer.GetParametersAndGradients();
if (paramsAndGrads.Count == 0)
{
return(true);
}
var result = new Tensor(output.Shape);
var parameters = paramsAndGrads[0].Parameters;
var gradients = paramsAndGrads[0].Gradients;
for (var i = 0; i < parameters.Shape.Length; i++)
{
result.Zero();
float oldValue = parameters.GetFlat(i);
parameters.SetFlat(oldValue + DERIVATIVE_EPSILON, i);
var output1 = layer.FeedForward(inputs).Clone();
parameters.SetFlat(oldValue - DERIVATIVE_EPSILON, i);
var output2 = layer.FeedForward(inputs).Clone();
parameters.SetFlat(oldValue, i);
output1.Sub(output2, result);
var approxGrad = new float[output.Shape.Length];
for (int j = 0; j < output.Shape.Length; j++)
{
approxGrad[j] = result.GetFlat(j) / (2.0f * DERIVATIVE_EPSILON);
}
var approxGradient = approxGrad.Sum();
if (Math.Abs(approxGradient - gradients.GetFlat(i)) > 0.02)
{
Debug.Assert(false, $"Parameter gradient validation failed at parameter {i}, expected {approxGradient} actual {gradients.GetFlat(i)}!");
return(false);
}
}
return(true);
}
public static bool VerifyInputGradient(LayerBase layer, int batchSize = 1)
{
Tensor.SetOpMode(Tensor.OpMode.CPU);
var inputs = GenerateInputsForLayer(layer, batchSize);
var output = layer.FeedForward(inputs);
var outputGradient = new Tensor(output.Shape);
outputGradient.FillWithValue(1);
layer.BackProp(outputGradient);
var result = new Tensor(output.Shape);
for (int n = 0; n < inputs.Length; ++n)
{
var input = inputs[n];
for (int i = 0; i < input.Shape.Length; ++i)
{
result.Zero();
var oldValue = input.GetFlat(i);
input.SetFlat(oldValue - DERIVATIVE_EPSILON, i);
var output1 = layer.FeedForward(inputs).Clone();
input.SetFlat(oldValue + DERIVATIVE_EPSILON, i);
var output2 = layer.FeedForward(inputs).Clone();
input.SetFlat(oldValue, i);
output2.Sub(output1, result);
var approxGrad = new float[output.Shape.Length];
float approxGradient = 0;
for (int j = 0; j < output.Shape.Length; j++)
{
approxGrad[j] = result.GetFlat(j) / (2.0f * DERIVATIVE_EPSILON);
approxGradient += approxGrad[j];
}
if (Math.Abs(approxGradient - layer.InputsGradient[n].GetFlat(i)) > 0.02)
{
Debug.Assert(false, $"Input gradient validation failed at element {i} of input {n}, expected {approxGradient} actual {layer.InputsGradient[n].GetFlat(i)}!");
return(false);
}
}
}
return(true);
}