public static void GradientCheck(LayerBase layer, int inputWidth, int inputHeight, int inputDepth, double epsilon = 1e-4)
{
layer.Init(inputWidth, inputHeight, inputDepth);
// Forward pass
var input = new Volume(inputWidth, inputHeight, inputDepth);
var output = layer.Forward(input, true);
// Set output gradients to 1
for (var n = 0; n < output.Length; n++)
{
output.SetGradient(n, 1.0);
}
// Backward pass to retrieve gradients
layer.Backward();
var computedGradients = input;
// Now let's approximate gradient using derivate definition
for (var d = 0; d < inputDepth; d++)
{
for (var y = 0; y < inputHeight; y++)
{
for (var x = 0; x < inputWidth; x++)
{
var oldValue = input.Get(x, y, d);
input.Set(x, y, d, oldValue + epsilon);
var output1 = layer.Forward(input);
input.Set(x, y, d, oldValue - epsilon);
var output2 = layer.Forward(input);
input.Set(x, y, d, oldValue);
output1.AddFromScaled(output2, -1.0); // output1 = output1 - output2
var grad = new double[output.Length];
for (var j = 0; j < output.Length; j++)
{
grad[j] = output1.Get(j) / (2.0 * epsilon);
}
var gradient = grad.Sum(); // approximated gradient
var actual = computedGradients.GetGradient(x, y, d);
Assert.AreEqual(gradient, actual, 1e-4); // compare layer gradient to the approximated gradient
}
}
}
}
public static void GradientCheck(LayerBase layer, int inputWidth, int inputHeight, int inputDepth, double epsilon = 1e-4)
{
layer.Init(inputWidth, inputHeight, inputDepth);
// Forward pass
var input = new Volume(inputWidth, inputHeight, inputDepth);
var output = layer.Forward(input, true);
// Set output gradients to 1
for (var n = 0; n < output.WeightGradients.Length; n++)
{
output.WeightGradients[n] = 1.0;
}
// Backward pass to retrieve gradients
layer.Backward();
var computedGradients = input;
// Now let's approximate gradient using derivate definition
for (var d = 0; d < inputDepth; d++)
{
for (var y = 0; y < inputHeight; y++)
{
for (var x = 0; x < inputWidth; x++)
{
var oldValue = input.Get(x, y, d);
input.Set(x, y, d, oldValue + epsilon);
var output1 = layer.Forward(input);
input.Set(x, y, d, oldValue - epsilon);
var output2 = layer.Forward(input);
input.Set(x, y, d, oldValue);
output1.AddFromScaled(output2, -1.0); // output1 = output1 - output2
var grad = new double[output.WeightGradients.Length];
for (var j = 0; j < output.WeightGradients.Length; j++)
{
grad[j] = output1.Weights[j] / (2.0 * epsilon);
}
var gradient = grad.Sum(); // approximated gradient
var actual = computedGradients.GetGradient(x, y, d);
Assert.AreEqual(gradient, actual, 1e-4); // compare layer gradient to the approximated gradient
}
}
}
}
public static void GradienWrtParameterstCheck(int inputWidth, int inputHeight, int inputDepth, LayerBase layer, double epsilon = 1e-4)
{
layer.Init(inputWidth, inputHeight, inputDepth);
// Forward pass
var input = new Volume(inputWidth, inputHeight, inputDepth, 1.0);
var output = layer.Forward(input);
// Set output gradients to 1
for (var n = 0; n < output.Length; n++)
{
output.SetGradient(n, 1.0);
}
// Backward pass to retrieve gradients
layer.Backward();
List <ParametersAndGradients> paramsAndGrads = layer.GetParametersAndGradients();
foreach (var paramAndGrad in paramsAndGrads)
{
//double[] computedGradients = paramAndGrad.Volume.Gradients;
var vol = paramAndGrad.Volume;
// Now let's approximate gradient
for (var i = 0; i < paramAndGrad.Volume.Length; i++)
{
input = new Volume(inputWidth, inputHeight, inputDepth, 1.0);
var oldValue = vol.Get(i);
vol.Set(i, oldValue + epsilon);
var output1 = layer.Forward(input);
vol.Set(i, oldValue - epsilon);
var output2 = layer.Forward(input);
vol.Set(i, oldValue);
output1.AddFromScaled(output2, -1.0); // output1 = output1 - output2
var grad = new double[output.Length];
for (var j = 0; j < output.Length; j++)
{
grad[j] = output1.Get(j) / (2.0 * epsilon);
}
var gradient = grad.Sum(); // approximated gradient
Assert.AreEqual(gradient, vol.GetGradient(i), 1e-4); // compare layer gradient to the approximated gradient
}
}
}
public static void GradienWrtParameterstCheck(int inputWidth, int inputHeight, int inputDepth, LayerBase layer, double epsilon = 1e-4)
{
layer.Init(inputWidth, inputHeight, inputDepth);
// Forward pass
var input = new Volume(inputWidth, inputHeight, inputDepth, 1.0);
var output = layer.Forward(input);
// Set output gradients to 1
for (var n = 0; n < output.WeightGradients.Length; n++)
{
output.WeightGradients[n] = 1.0;
}
// Backward pass to retrieve gradients
layer.Backward();
List<ParametersAndGradients> paramsAndGrads = layer.GetParametersAndGradients();
foreach (var paramAndGrad in paramsAndGrads)
{
double[] computedGradients = paramAndGrad.Gradients;
// Now let's approximate gradient
for (var i = 0; i < paramAndGrad.Parameters.Length; i++)
{
input = new Volume(inputWidth, inputHeight, inputDepth, 1.0);
var oldValue = paramAndGrad.Parameters[i];
paramAndGrad.Parameters[i] = oldValue + epsilon;
var output1 = layer.Forward(input);
paramAndGrad.Parameters[i] = oldValue - epsilon;
var output2 = layer.Forward(input);
paramAndGrad.Parameters[i] = oldValue;
output1.AddFromScaled(output2, -1.0); // output1 = output1 - output2
var grad = new double[output.WeightGradients.Length];
for (var j = 0; j < output.WeightGradients.Length; j++)
{
grad[j] = output1.Weights[j] / (2.0 * epsilon);
}
var gradient = grad.Sum(); // approximated gradient
Assert.AreEqual(gradient, computedGradients[i], 1e-4); // compare layer gradient to the approximated gradient
}
}
}
