public unsafe void PoolingBackward()
{
// Setup
Tensor.New(400, 58 * 58 * 3, out Tensor x);
KerasWeightsProvider.FillWithHeEtAlUniform(x, 10);
PoolingLayer
cpu = new PoolingLayer(new TensorInfo(58, 58, 3), PoolingInfo.Default, ActivationFunctionType.LeakyReLU),
gpu = new CuDnnPoolingLayer(cpu.InputInfo, PoolingInfo.Default, ActivationFunctionType.LeakyReLU);
gpu.Forward(x, out Tensor z, out Tensor a);
a.Free();
x.Duplicate(out Tensor x1);
x.Duplicate(out Tensor x2);
Tensor.New(z.Entities, z.Length, out Tensor delta);
KerasWeightsProvider.FillWithHeEtAlUniform(delta, 10);
// Backward
cpu.Backpropagate(x, delta, x1, ActivationFunctions.LeakyReLUPrime);
gpu.Backpropagate(x, delta, x2, ActivationFunctions.LeakyReLUPrime);
bool valid = true;
float *px = (float *)x1.Ptr.ToPointer(), px2 = (float *)x2.Ptr.ToPointer();
int count = 0;
for (int i = 0; i < x1.Size; i++)
{
if (px[i].EqualsWithDelta(px2[i], 1e-5f))
{
continue;
}
if (px[i].EqualsWithDelta(px2[i] * 100f, 1e-5f))
{
count++; // The cuDNN pooling backwards method returns a value scaled by 0.01 from time to time for some reason (less than 2% anyways)
}
else
{
valid = false;
break;
}
}
Assert.IsTrue(valid && count * 100f / x1.Size < 2);
x.Free();
x1.Free();
x2.Free();
z.Free();
delta.Free();
}
public unsafe void Inception3x3Pipeline()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(10), 32 * 32 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(10, 32 * 32 * 3);
CuDnnConvolutionalLayer
conv1 = new CuDnnConvolutionalLayer(TensorInfo.Image <Rgb24>(32, 32), ConvolutionInfo.New(ConvolutionMode.CrossCorrelation), (1, 1), 10, ActivationFunctionType.ReLU, BiasInitializationMode.Gaussian),
conv2 = new CuDnnConvolutionalLayer(conv1.OutputInfo, ConvolutionInfo.New(ConvolutionMode.CrossCorrelation, 1, 1), (3, 3), 10, ActivationFunctionType.ReLU, BiasInitializationMode.Gaussian);
CuDnnInceptionLayer inception = new CuDnnInceptionLayer(TensorInfo.Image <Rgb24>(32, 32), InceptionInfo.New(10, 10, 10, 10, 10, PoolingMode.Max, 10));
fixed(float *pw = inception.Weights)
Unsafe.InitBlock(pw, 0, (uint)(sizeof(float) * inception.Weights.Length));
Buffer.BlockCopy(conv1.Weights, 0, inception.Weights, sizeof(float) * 3 * 10, sizeof(float) * conv1.Weights.Length);
Buffer.BlockCopy(conv2.Weights, 0, inception.Weights, sizeof(float) * 3 * 10 + sizeof(float) * conv1.Weights.Length, sizeof(float) * conv2.Weights.Length);
Buffer.BlockCopy(conv1.Biases, 0, inception.Biases, sizeof(float) * 10, sizeof(float) * conv1.Biases.Length);
Buffer.BlockCopy(conv2.Biases, 0, inception.Biases, sizeof(float) * 20, sizeof(float) * conv2.Biases.Length);
fixed(float *px = x)
{
// Forward + Z
Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xTensor);
conv1.Forward(xTensor, out Tensor zTemp, out Tensor aTemp);
conv2.Forward(aTemp, out Tensor zConv, out Tensor aConv);
inception.Forward(xTensor, out Tensor zInc, out Tensor aInc);
Tensor.New(zConv.Entities, zConv.Length, out Tensor reshaped);
float *pzInc = (float *)zInc.Ptr.ToPointer() + 32 * 32 * 10, preshaped = (float *)reshaped.Ptr.ToPointer();
for (int i = 0; i < zConv.Entities; i++)
{
Buffer.MemoryCopy(pzInc + i * zInc.Length, preshaped + i * zConv.Length, sizeof(float) * zConv.Length, sizeof(float) * zConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(zConv));
// A
float *paInc = (float *)aInc.Ptr.ToPointer() + 32 * 32 * 10;
for (int i = 0; i < aConv.Entities; i++)
{
Buffer.MemoryCopy(paInc + i * aInc.Length, preshaped + i * aConv.Length, sizeof(float) * aConv.Length, sizeof(float) * aConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(aConv));
// Backpropagation
Tensor.New(xTensor.Entities, xTensor.Length, out Tensor z1);
KerasWeightsProvider.FillWithHeEtAlUniform(z1, 10);
z1.Duplicate(out Tensor z2);
conv2.Backpropagate(Tensor.Null, aConv, zTemp, conv1.ActivationFunctions.ActivationPrime);
conv1.Backpropagate(Tensor.Null, zTemp, z1, ActivationFunctions.ReLUPrime);
inception.Backpropagate(xTensor, aInc, z2, ActivationFunctions.ReLUPrime);
Assert.IsTrue(z1.ContentEquals(z2));
// Gradient
Tensor.New(xTensor.Entities, xTensor.Length, out Tensor a);
KerasWeightsProvider.FillWithHeEtAlUniform(a, 10);
conv1.ComputeGradient(a, zTemp, out Tensor dJdwConv1, out Tensor dJdbConv1);
conv2.ComputeGradient(aTemp, aConv, out Tensor dJdwConv2, out Tensor dJdbConv2);
inception.ComputeGradient(a, aInc, out Tensor dJdwInc, out Tensor dJdbInc);
Tensor.Reshape((float *)dJdwInc.Ptr.ToPointer() + 30, 1, dJdwConv1.Size, out Tensor dJdwInc0);
Tensor.Reshape((float *)dJdbInc.Ptr.ToPointer() + 10, 1, dJdbConv1.Size, out Tensor dJdbInc0);
Assert.IsTrue(dJdwConv1.ContentEquals(dJdwInc0, 1e-5f));
Assert.IsTrue(dJdbConv1.ContentEquals(dJdbInc0, 1e-5f));
Tensor.Reshape((float *)dJdwInc.Ptr.ToPointer() + 30 + dJdwConv1.Size, 1, dJdwConv2.Size, out Tensor dJdwInc1);
Tensor.Reshape((float *)dJdbInc.Ptr.ToPointer() + 20, 1, dJdbConv2.Size, out Tensor dJdbInc1);
Assert.IsTrue(dJdwConv2.ContentEquals(dJdwInc1, 1e-5f));
Assert.IsTrue(dJdbConv2.ContentEquals(dJdbInc1, 1e-5f));
// Cleanup
z1.Free();
z2.Free();
zTemp.Free();
zConv.Free();
zInc.Free();
aConv.Free();
aInc.Free();
reshaped.Free();
}
}
