public unsafe void PoolingBackward() { // Setup Tensor.New(400, 58 * 58 * 3, out Tensor x); KerasWeightsProvider.FillWithHeEtAlUniform(x, 10); PoolingLayer cpu = new PoolingLayer(new TensorInfo(58, 58, 3), PoolingInfo.Default, ActivationFunctionType.LeakyReLU), gpu = new CuDnnPoolingLayer(cpu.InputInfo, PoolingInfo.Default, ActivationFunctionType.LeakyReLU); gpu.Forward(x, out Tensor z, out Tensor a); a.Free(); x.Duplicate(out Tensor x1); x.Duplicate(out Tensor x2); Tensor.New(z.Entities, z.Length, out Tensor delta); KerasWeightsProvider.FillWithHeEtAlUniform(delta, 10); // Backward cpu.Backpropagate(x, delta, x1, ActivationFunctions.LeakyReLUPrime); gpu.Backpropagate(x, delta, x2, ActivationFunctions.LeakyReLUPrime); bool valid = true; float *px = (float *)x1.Ptr.ToPointer(), px2 = (float *)x2.Ptr.ToPointer(); int count = 0; for (int i = 0; i < x1.Size; i++) { if (px[i].EqualsWithDelta(px2[i], 1e-5f)) { continue; } if (px[i].EqualsWithDelta(px2[i] * 100f, 1e-5f)) { count++; // The cuDNN pooling backwards method returns a value scaled by 0.01 from time to time for some reason (less than 2% anyways) } else { valid = false; break; } } Assert.IsTrue(valid && count * 100f / x1.Size < 2); x.Free(); x1.Free(); x2.Free(); z.Free(); delta.Free(); }
public unsafe void Inception3x3Pipeline() { float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(10), 32 * 32 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(10, 32 * 32 * 3); CuDnnConvolutionalLayer conv1 = new CuDnnConvolutionalLayer(TensorInfo.Image <Rgb24>(32, 32), ConvolutionInfo.New(ConvolutionMode.CrossCorrelation), (1, 1), 10, ActivationFunctionType.ReLU, BiasInitializationMode.Gaussian), conv2 = new CuDnnConvolutionalLayer(conv1.OutputInfo, ConvolutionInfo.New(ConvolutionMode.CrossCorrelation, 1, 1), (3, 3), 10, ActivationFunctionType.ReLU, BiasInitializationMode.Gaussian); CuDnnInceptionLayer inception = new CuDnnInceptionLayer(TensorInfo.Image <Rgb24>(32, 32), InceptionInfo.New(10, 10, 10, 10, 10, PoolingMode.Max, 10)); fixed(float *pw = inception.Weights) Unsafe.InitBlock(pw, 0, (uint)(sizeof(float) * inception.Weights.Length)); Buffer.BlockCopy(conv1.Weights, 0, inception.Weights, sizeof(float) * 3 * 10, sizeof(float) * conv1.Weights.Length); Buffer.BlockCopy(conv2.Weights, 0, inception.Weights, sizeof(float) * 3 * 10 + sizeof(float) * conv1.Weights.Length, sizeof(float) * conv2.Weights.Length); Buffer.BlockCopy(conv1.Biases, 0, inception.Biases, sizeof(float) * 10, sizeof(float) * conv1.Biases.Length); Buffer.BlockCopy(conv2.Biases, 0, inception.Biases, sizeof(float) * 20, sizeof(float) * conv2.Biases.Length); fixed(float *px = x) { // Forward + Z Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xTensor); conv1.Forward(xTensor, out Tensor zTemp, out Tensor aTemp); conv2.Forward(aTemp, out Tensor zConv, out Tensor aConv); inception.Forward(xTensor, out Tensor zInc, out Tensor aInc); Tensor.New(zConv.Entities, zConv.Length, out Tensor reshaped); float *pzInc = (float *)zInc.Ptr.ToPointer() + 32 * 32 * 10, preshaped = (float *)reshaped.Ptr.ToPointer(); for (int i = 0; i < zConv.Entities; i++) { Buffer.MemoryCopy(pzInc + i * zInc.Length, preshaped + i * zConv.Length, sizeof(float) * zConv.Length, sizeof(float) * zConv.Length); } Assert.IsTrue(reshaped.ContentEquals(zConv)); // A float *paInc = (float *)aInc.Ptr.ToPointer() + 32 * 32 * 10; for (int i = 0; i < aConv.Entities; i++) { Buffer.MemoryCopy(paInc + i * aInc.Length, preshaped + i * aConv.Length, sizeof(float) * aConv.Length, sizeof(float) * aConv.Length); } Assert.IsTrue(reshaped.ContentEquals(aConv)); // Backpropagation Tensor.New(xTensor.Entities, xTensor.Length, out Tensor z1); KerasWeightsProvider.FillWithHeEtAlUniform(z1, 10); z1.Duplicate(out Tensor z2); conv2.Backpropagate(Tensor.Null, aConv, zTemp, conv1.ActivationFunctions.ActivationPrime); conv1.Backpropagate(Tensor.Null, zTemp, z1, ActivationFunctions.ReLUPrime); inception.Backpropagate(xTensor, aInc, z2, ActivationFunctions.ReLUPrime); Assert.IsTrue(z1.ContentEquals(z2)); // Gradient Tensor.New(xTensor.Entities, xTensor.Length, out Tensor a); KerasWeightsProvider.FillWithHeEtAlUniform(a, 10); conv1.ComputeGradient(a, zTemp, out Tensor dJdwConv1, out Tensor dJdbConv1); conv2.ComputeGradient(aTemp, aConv, out Tensor dJdwConv2, out Tensor dJdbConv2); inception.ComputeGradient(a, aInc, out Tensor dJdwInc, out Tensor dJdbInc); Tensor.Reshape((float *)dJdwInc.Ptr.ToPointer() + 30, 1, dJdwConv1.Size, out Tensor dJdwInc0); Tensor.Reshape((float *)dJdbInc.Ptr.ToPointer() + 10, 1, dJdbConv1.Size, out Tensor dJdbInc0); Assert.IsTrue(dJdwConv1.ContentEquals(dJdwInc0, 1e-5f)); Assert.IsTrue(dJdbConv1.ContentEquals(dJdbInc0, 1e-5f)); Tensor.Reshape((float *)dJdwInc.Ptr.ToPointer() + 30 + dJdwConv1.Size, 1, dJdwConv2.Size, out Tensor dJdwInc1); Tensor.Reshape((float *)dJdbInc.Ptr.ToPointer() + 20, 1, dJdbConv2.Size, out Tensor dJdbInc1); Assert.IsTrue(dJdwConv2.ContentEquals(dJdwInc1, 1e-5f)); Assert.IsTrue(dJdbConv2.ContentEquals(dJdbInc1, 1e-5f)); // Cleanup z1.Free(); z2.Free(); zTemp.Free(); zConv.Free(); zInc.Free(); aConv.Free(); aInc.Free(); reshaped.Free(); } }