public unsafe void SoftmaxBackwardOutput()
{
float[,]
x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(400), 250, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(400, 250),
y = new float[400, 127];
for (int i = 0; i < 400; i++)
{
y[i, ThreadSafeRandom.NextInt(max: 127)] = 1;
}
OutputLayerBase
cpu = new SoftmaxLayer(TensorInfo.Linear(250), 127, WeightsInitializationMode.GlorotNormal, BiasInitializationMode.Gaussian),
gpu = new CuDnnSoftmaxLayer(cpu.InputInfo, cpu.OutputInfo.Size, cpu.Weights, cpu.Biases);
fixed(float *px = x, py = y)
{
Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xt);
cpu.Forward(xt, out Tensor z, out Tensor a);
a.Duplicate(out Tensor a2);
Tensor.Reshape(py, y.GetLength(0), y.GetLength(1), out Tensor yt);
cpu.Backpropagate(a, yt, z);
gpu.Backpropagate(a2, yt, z);
Assert.IsTrue(a.ContentEquals(a2));
a.Free();
a2.Free();
z.Free();
}
}
public void FullyConnectedForward()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(400), 250, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(400, 250);
FullyConnectedLayer
cpu = new FullyConnectedLayer(TensorInfo.Linear(250), 127, ActivationFunctionType.LeCunTanh, WeightsInitializationMode.GlorotNormal, BiasInitializationMode.Gaussian),
gpu = new CuDnnFullyConnectedLayer(cpu.InputInfo, cpu.OutputInfo.Size, cpu.Weights, cpu.Biases, cpu.ActivationFunctionType);
TestForward(cpu, gpu, x);
}
public void PoolingForward()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(400), 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(400, 58 * 58 * 3);
PoolingLayer
cpu = new PoolingLayer(new TensorInfo(58, 58, 3), PoolingInfo.Default, ActivationFunctionType.LeakyReLU),
gpu = new CuDnnPoolingLayer(cpu.InputInfo, PoolingInfo.Default, ActivationFunctionType.LeakyReLU);
TestForward(cpu, gpu, x);
}
public void ConvolutionForward()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(127), 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(127, 58 * 58 * 3);
ConvolutionalLayer
cpu = new ConvolutionalLayer(new TensorInfo(58, 58, 3), ConvolutionInfo.Default, (5, 5), 20, ActivationFunctionType.LeakyReLU, BiasInitializationMode.Gaussian),
gpu = new CuDnnConvolutionalLayer(cpu.InputInfo, ConvolutionInfo.Default, cpu.KernelInfo, cpu.OutputInfo, cpu.Weights, cpu.Biases, cpu.ActivationFunctionType);
TestForward(cpu, gpu, x);
}
public void SoftmaxForward()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(400), 250, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(400, 250);
OutputLayerBase
cpu = new SoftmaxLayer(TensorInfo.Linear(250), 127, WeightsInitializationMode.GlorotNormal, BiasInitializationMode.Gaussian),
gpu = new CuDnnSoftmaxLayer(cpu.InputInfo, cpu.OutputInfo.Size, cpu.Weights, cpu.Biases);
TestForward(cpu, gpu, x);
}
public unsafe void InceptionPoolPipeline()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(10), 12 * 12 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(10, 12 * 12 * 3);
CuDnnPoolingLayer pool = new CuDnnPoolingLayer(TensorInfo.Image <Rgb24>(12, 12), PoolingInfo.New(PoolingMode.Max, 3, 3, 1, 1, 1, 1), ActivationType.ReLU);
CuDnnConvolutionalLayer conv = new CuDnnConvolutionalLayer(pool.OutputInfo, ConvolutionInfo.New(ConvolutionMode.CrossCorrelation), (1, 1), 10, ActivationType.ReLU, BiasInitializationMode.Gaussian);
CuDnnInceptionLayer inception = new CuDnnInceptionLayer(TensorInfo.Image <Rgb24>(12, 12), InceptionInfo.New(3, 2, 2, 2, 2, PoolingMode.Max, 10));
fixed(float *pw = inception.Weights)
Unsafe.InitBlock(pw, 0, (uint)(sizeof(float) * inception.Weights.Length));
Buffer.BlockCopy(conv.Weights, 0, inception.Weights, sizeof(float) * (3 * 3 + 3 * 2 + 3 * 3 * 2 * 2 + 3 * 2 + 5 * 5 * 2 * 2), sizeof(float) * conv.Weights.Length);
Buffer.BlockCopy(conv.Biases, 0, inception.Biases, sizeof(float) * (3 + 2 + 2 + 2 + 2), sizeof(float) * conv.Biases.Length);
fixed(float *px = x)
{
// Forward + Z
Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xTensor);
pool.Forward(xTensor, out Tensor zTemp, out Tensor aTemp);
conv.Forward(aTemp, out Tensor zConv, out Tensor aConv);
inception.Forward(xTensor, out Tensor zInc, out Tensor aInc);
Tensor.New(zConv.Entities, zConv.Length, out Tensor reshaped);
float *pzInc = (float *)zInc.Ptr.ToPointer() + 12 * 12 * (3 + 2 + 2), preshaped = (float *)reshaped.Ptr.ToPointer();
for (int i = 0; i < zConv.Entities; i++)
{
Buffer.MemoryCopy(pzInc + i * zInc.Length, preshaped + i * zConv.Length, sizeof(float) * zConv.Length, sizeof(float) * zConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(zConv));
// A
float *paInc = (float *)aInc.Ptr.ToPointer() + 12 * 12 * (3 + 2 + 2);
for (int i = 0; i < aConv.Entities; i++)
{
Buffer.MemoryCopy(paInc + i * aInc.Length, preshaped + i * aConv.Length, sizeof(float) * aConv.Length, sizeof(float) * aConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(aConv));
// Backpropagation
Tensor.Like(aTemp, out Tensor convdx);
Tensor.Like(xTensor, out Tensor pooldx);
Tensor.Like(xTensor, out Tensor incdx);
conv.Backpropagate(aTemp, zConv, aConv, convdx, out Tensor convdJdw, out Tensor convdJdb);
pool.Backpropagate(xTensor, zTemp, convdx, pooldx);
inception.Backpropagate(xTensor, zInc, aInc, incdx, out Tensor incdJdw, out Tensor incdJdb);
Assert.IsTrue(incdx.ContentEquals(pooldx));
// Gradient
Tensor.Reshape((float *)incdJdw.Ptr.ToPointer() + (3 * 3 + 3 * 2 + 3 * 3 * 2 * 2 + 3 * 2 + 5 * 5 * 2 * 2), 1, convdJdw.Size, out Tensor dJdwInc0);
Tensor.Reshape((float *)incdJdb.Ptr.ToPointer() + 11, 1, convdJdb.Size, out Tensor dJdbInc0);
Assert.IsTrue(convdJdw.ContentEquals(dJdwInc0, 1e-5f));
Assert.IsTrue(convdJdb.ContentEquals(dJdbInc0, 1e-5f));
// Cleanup
Tensor.Free(zTemp, aTemp, zConv, aConv, zInc, aInc, reshaped, convdx, pooldx, incdx, convdJdw, convdJdb, incdJdw, incdJdb);
}
}
private static unsafe Tensor CreateRandomTensor(int entities, int length)
{
float[] v = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(entities), length, WeightsInitializationMode.GlorotNormal);
Tensor.New(entities, length, out Tensor tensor);
fixed(float *pv = v)
{
Tensor.Reshape(pv, entities, length, out Tensor source);
tensor.Overwrite(source);
return(tensor);
}
}
public void StreamSerialize()
{
using (MemoryStream stream = new MemoryStream())
{
float[] w = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(784), 30, WeightsInitializationMode.GlorotNormal);
stream.WriteShuffled(w);
Assert.IsTrue(stream.Position == sizeof(float) * w.Length);
stream.Seek(0, SeekOrigin.Begin);
float[] t = stream.ReadUnshuffled(w.Length);
Assert.IsTrue(w.ContentEquals(t));
}
}
public unsafe void Inception1x1()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(10), 32 * 32 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(10, 32 * 32 * 3);
CuDnnConvolutionalLayer conv = new CuDnnConvolutionalLayer(TensorInfo.Image <Rgb24>(32, 32), ConvolutionInfo.New(ConvolutionMode.CrossCorrelation), (1, 1), 10, ActivationType.ReLU, BiasInitializationMode.Gaussian);
CuDnnInceptionLayer inception = new CuDnnInceptionLayer(conv.InputInfo, InceptionInfo.New(10, 10, 10, 10, 10, PoolingMode.Max, 10));
fixed(float *pw = inception.Weights)
Unsafe.InitBlock(pw, 0, (uint)(sizeof(float) * inception.Weights.Length));
Buffer.BlockCopy(conv.Weights, 0, inception.Weights, 0, sizeof(float) * conv.Weights.Length);
Buffer.BlockCopy(conv.Biases, 0, inception.Biases, 0, sizeof(float) * conv.Biases.Length);
fixed(float *px = x)
{
// Forward + Z
Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xTensor);
conv.Forward(xTensor, out Tensor zConv, out Tensor aConv);
inception.Forward(xTensor, out Tensor zInc, out Tensor aInc);
Tensor.New(zConv.Entities, zConv.Length, out Tensor reshaped);
float *pzInc = (float *)zInc.Ptr.ToPointer(), preshaped = (float *)reshaped.Ptr.ToPointer();
for (int i = 0; i < zConv.Entities; i++)
{
Buffer.MemoryCopy(pzInc + i * zInc.Length, preshaped + i * zConv.Length, sizeof(float) * zConv.Length, sizeof(float) * zConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(zConv));
// A
float *paInc = (float *)aInc.Ptr.ToPointer();
for (int i = 0; i < aConv.Entities; i++)
{
Buffer.MemoryCopy(paInc + i * aInc.Length, preshaped + i * aConv.Length, sizeof(float) * aConv.Length, sizeof(float) * aConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(aConv));
// Backpropagate
Tensor.Like(xTensor, out Tensor dx1);
Tensor.Like(xTensor, out Tensor dx2);
conv.Backpropagate(xTensor, zConv, aConv, dx1, out Tensor dJdw1, out Tensor dJdb1);
inception.Backpropagate(xTensor, zInc, aInc, dx2, out Tensor dJdw2, out Tensor dJdb2);
Assert.IsTrue(dx1.ContentEquals(dx2));
Tensor.Reshape((float *)dJdw2.Ptr.ToPointer(), 1, dJdw1.Size, out dJdw2);
Tensor.Reshape((float *)dJdb2.Ptr.ToPointer(), 1, dJdb1.Size, out dJdb2);
Assert.IsTrue(dJdw1.ContentEquals(dJdw2, 1e-5f));
Assert.IsTrue(dJdb1.ContentEquals(dJdb2, 1e-5f));
// Cleanup
Tensor.Free(zConv, aConv, zInc, aInc, reshaped, dx1, dx2, dJdw1, dJdw2, dJdb1, dJdb2);
}
}
public unsafe void ConvolutionGradient()
{
float[,]
x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(127), 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(127, 58 * 58 * 3),
delta = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(127), 54 * 54 * 5, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(127, 54 * 54 * 5);
ConvolutionalLayer
cpu = new ConvolutionalLayer(new TensorInfo(58, 58, 3), ConvolutionInfo.Default, (5, 5), 5, ActivationFunctionType.LeCunTanh, BiasInitializationMode.Gaussian),
gpu = new CuDnnConvolutionalLayer(cpu.InputInfo, ConvolutionInfo.Default, cpu.KernelInfo, cpu.OutputInfo, cpu.Weights, cpu.Biases, ActivationFunctionType.LeCunTanh);
fixed(float *px = x)
{
Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xTensor);
gpu.Forward(xTensor, out Tensor z_gpu, out Tensor a_gpu);
z_gpu.Free();
a_gpu.Free();
}
TestGradient(cpu, gpu, x, delta);
}
public unsafe void Inception3x3Pipeline()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(10), 32 * 32 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(10, 32 * 32 * 3);
CuDnnConvolutionalLayer
conv1 = new CuDnnConvolutionalLayer(TensorInfo.Image <Rgb24>(32, 32), ConvolutionInfo.New(ConvolutionMode.CrossCorrelation), (1, 1), 10, ActivationFunctionType.ReLU, BiasInitializationMode.Gaussian),
conv2 = new CuDnnConvolutionalLayer(conv1.OutputInfo, ConvolutionInfo.New(ConvolutionMode.CrossCorrelation, 1, 1), (3, 3), 10, ActivationFunctionType.ReLU, BiasInitializationMode.Gaussian);
CuDnnInceptionLayer inception = new CuDnnInceptionLayer(TensorInfo.Image <Rgb24>(32, 32), InceptionInfo.New(10, 10, 10, 10, 10, PoolingMode.Max, 10));
fixed(float *pw = inception.Weights)
Unsafe.InitBlock(pw, 0, (uint)(sizeof(float) * inception.Weights.Length));
Buffer.BlockCopy(conv1.Weights, 0, inception.Weights, sizeof(float) * 3 * 10, sizeof(float) * conv1.Weights.Length);
Buffer.BlockCopy(conv2.Weights, 0, inception.Weights, sizeof(float) * 3 * 10 + sizeof(float) * conv1.Weights.Length, sizeof(float) * conv2.Weights.Length);
Buffer.BlockCopy(conv1.Biases, 0, inception.Biases, sizeof(float) * 10, sizeof(float) * conv1.Biases.Length);
Buffer.BlockCopy(conv2.Biases, 0, inception.Biases, sizeof(float) * 20, sizeof(float) * conv2.Biases.Length);
fixed(float *px = x)
{
// Forward + Z
Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xTensor);
conv1.Forward(xTensor, out Tensor zTemp, out Tensor aTemp);
conv2.Forward(aTemp, out Tensor zConv, out Tensor aConv);
inception.Forward(xTensor, out Tensor zInc, out Tensor aInc);
Tensor.New(zConv.Entities, zConv.Length, out Tensor reshaped);
float *pzInc = (float *)zInc.Ptr.ToPointer() + 32 * 32 * 10, preshaped = (float *)reshaped.Ptr.ToPointer();
for (int i = 0; i < zConv.Entities; i++)
{
Buffer.MemoryCopy(pzInc + i * zInc.Length, preshaped + i * zConv.Length, sizeof(float) * zConv.Length, sizeof(float) * zConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(zConv));
// A
float *paInc = (float *)aInc.Ptr.ToPointer() + 32 * 32 * 10;
for (int i = 0; i < aConv.Entities; i++)
{
Buffer.MemoryCopy(paInc + i * aInc.Length, preshaped + i * aConv.Length, sizeof(float) * aConv.Length, sizeof(float) * aConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(aConv));
// Backpropagation
Tensor.New(xTensor.Entities, xTensor.Length, out Tensor z1);
KerasWeightsProvider.FillWithHeEtAlUniform(z1, 10);
z1.Duplicate(out Tensor z2);
conv2.Backpropagate(Tensor.Null, aConv, zTemp, conv1.ActivationFunctions.ActivationPrime);
conv1.Backpropagate(Tensor.Null, zTemp, z1, ActivationFunctions.ReLUPrime);
inception.Backpropagate(xTensor, aInc, z2, ActivationFunctions.ReLUPrime);
Assert.IsTrue(z1.ContentEquals(z2));
// Gradient
Tensor.New(xTensor.Entities, xTensor.Length, out Tensor a);
KerasWeightsProvider.FillWithHeEtAlUniform(a, 10);
conv1.ComputeGradient(a, zTemp, out Tensor dJdwConv1, out Tensor dJdbConv1);
conv2.ComputeGradient(aTemp, aConv, out Tensor dJdwConv2, out Tensor dJdbConv2);
inception.ComputeGradient(a, aInc, out Tensor dJdwInc, out Tensor dJdbInc);
Tensor.Reshape((float *)dJdwInc.Ptr.ToPointer() + 30, 1, dJdwConv1.Size, out Tensor dJdwInc0);
Tensor.Reshape((float *)dJdbInc.Ptr.ToPointer() + 10, 1, dJdbConv1.Size, out Tensor dJdbInc0);
Assert.IsTrue(dJdwConv1.ContentEquals(dJdwInc0, 1e-5f));
Assert.IsTrue(dJdbConv1.ContentEquals(dJdbInc0, 1e-5f));
Tensor.Reshape((float *)dJdwInc.Ptr.ToPointer() + 30 + dJdwConv1.Size, 1, dJdwConv2.Size, out Tensor dJdwInc1);
Tensor.Reshape((float *)dJdbInc.Ptr.ToPointer() + 20, 1, dJdbConv2.Size, out Tensor dJdbInc1);
Assert.IsTrue(dJdwConv2.ContentEquals(dJdwInc1, 1e-5f));
Assert.IsTrue(dJdbConv2.ContentEquals(dJdbInc1, 1e-5f));
// Cleanup
z1.Free();
z2.Free();
zTemp.Free();
zConv.Free();
zInc.Free();
aConv.Free();
aInc.Free();
reshaped.Free();
}
}
