public void ConvolutionInfoFactory()
{
ConvolutionInfo info = ConvolutionInfo.Same()(TensorInfo.Image <Alpha8>(28, 28), (3, 3));
Assert.IsTrue(info.VerticalPadding == 1 && info.HorizontalPadding == 1);
info = ConvolutionInfo.Same()(TensorInfo.Image <Alpha8>(28, 28), (5, 5));
Assert.IsTrue(info.VerticalPadding == 2 && info.HorizontalPadding == 2);
info = ConvolutionInfo.Same(ConvolutionMode.Convolution, 2, 2)(TensorInfo.Image <Alpha8>(10, 10), (3, 3));
Assert.IsTrue(info.VerticalPadding == 6 && info.HorizontalPadding == 6);
}
public void NetworkSerialization()
{
INeuralNetwork network = NetworkManager.NewSequential(TensorInfo.Image <Rgb24>(120, 120),
CuDnnNetworkLayers.Convolutional(ConvolutionInfo.New(ConvolutionMode.CrossCorrelation), (10, 10), 20, ActivationType.AbsoluteReLU),
CuDnnNetworkLayers.Convolutional(ConvolutionInfo.New(ConvolutionMode.Convolution, 2, 2), (5, 5), 20, ActivationType.ELU),
CuDnnNetworkLayers.Convolutional(ConvolutionInfo.Default, (10, 10), 20, ActivationType.Identity),
CuDnnNetworkLayers.Pooling(PoolingInfo.New(PoolingMode.AverageIncludingPadding, 2, 2, 1, 1), ActivationType.ReLU),
CuDnnNetworkLayers.Convolutional(ConvolutionInfo.Default, (10, 10), 20, ActivationType.Identity),
CuDnnNetworkLayers.Pooling(PoolingInfo.Default, ActivationType.ReLU),
CuDnnNetworkLayers.FullyConnected(125, ActivationType.Tanh),
CuDnnNetworkLayers.FullyConnected(27, ActivationType.Tanh),
CuDnnNetworkLayers.Softmax(133));
using (MemoryStream stream = new MemoryStream())
{
network.Save(stream);
stream.Seek(0, SeekOrigin.Begin);
INeuralNetwork copy = NetworkLoader.TryLoad(stream, ExecutionModePreference.Cuda);
Assert.IsTrue(network.Equals(copy));
}
}
public static LayerFactory Convolutional(
ConvolutionInfo info, (int X, int Y) kernel, int kernels, ActivationFunctionType activation,
public unsafe void Inception3x3Pipeline()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(10), 32 * 32 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(10, 32 * 32 * 3);
CuDnnConvolutionalLayer
conv1 = new CuDnnConvolutionalLayer(TensorInfo.Image <Rgb24>(32, 32), ConvolutionInfo.New(ConvolutionMode.CrossCorrelation), (1, 1), 10, ActivationType.ReLU, BiasInitializationMode.Gaussian),
conv2 = new CuDnnConvolutionalLayer(conv1.OutputInfo, ConvolutionInfo.New(ConvolutionMode.CrossCorrelation, 1, 1), (3, 3), 10, ActivationType.ReLU, BiasInitializationMode.Gaussian);
CuDnnInceptionLayer inception = new CuDnnInceptionLayer(TensorInfo.Image <Rgb24>(32, 32), InceptionInfo.New(10, 10, 10, 10, 10, PoolingMode.Max, 10));
fixed(float *pw = inception.Weights)
Unsafe.InitBlock(pw, 0, (uint)(sizeof(float) * inception.Weights.Length));
Buffer.BlockCopy(conv1.Weights, 0, inception.Weights, sizeof(float) * 3 * 10, sizeof(float) * conv1.Weights.Length);
Buffer.BlockCopy(conv2.Weights, 0, inception.Weights, sizeof(float) * 3 * 10 + sizeof(float) * conv1.Weights.Length, sizeof(float) * conv2.Weights.Length);
Buffer.BlockCopy(conv1.Biases, 0, inception.Biases, sizeof(float) * 10, sizeof(float) * conv1.Biases.Length);
Buffer.BlockCopy(conv2.Biases, 0, inception.Biases, sizeof(float) * 20, sizeof(float) * conv2.Biases.Length);
fixed(float *px = x)
{
// Forward + Z
Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xTensor);
conv1.Forward(xTensor, out Tensor zTemp, out Tensor aTemp);
conv2.Forward(aTemp, out Tensor zConv, out Tensor aConv);
inception.Forward(xTensor, out Tensor zInc, out Tensor aInc);
Tensor.New(zConv.Entities, zConv.Length, out Tensor reshaped);
float *pzInc = (float *)zInc.Ptr.ToPointer() + 32 * 32 * 10, preshaped = (float *)reshaped.Ptr.ToPointer();
for (int i = 0; i < zConv.Entities; i++)
{
Buffer.MemoryCopy(pzInc + i * zInc.Length, preshaped + i * zConv.Length, sizeof(float) * zConv.Length, sizeof(float) * zConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(zConv));
// A
float *paInc = (float *)aInc.Ptr.ToPointer() + 32 * 32 * 10;
for (int i = 0; i < aConv.Entities; i++)
{
Buffer.MemoryCopy(paInc + i * aInc.Length, preshaped + i * aConv.Length, sizeof(float) * aConv.Length, sizeof(float) * aConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(aConv));
// Backpropagation
Tensor.Like(aTemp, out Tensor conv2dx);
Tensor.Like(xTensor, out Tensor conv1dx);
Tensor.Like(xTensor, out Tensor incdx);
conv2.Backpropagate(aTemp, zConv, aConv, conv2dx, out Tensor conv2dJdw, out Tensor conv2dJdb);
conv1.Backpropagate(xTensor, zTemp, conv2dx, conv1dx, out Tensor conv1dJdw, out Tensor conv1dJdb);
inception.Backpropagate(xTensor, zInc, aInc, incdx, out Tensor incDjdw, out Tensor incdJdb);
Assert.IsTrue(incdx.ContentEquals(conv1dx));
// Gradient
Tensor.Reshape((float *)incDjdw.Ptr.ToPointer() + 30, 1, conv1dJdw.Size, out Tensor dJdwInc0);
Tensor.Reshape((float *)incdJdb.Ptr.ToPointer() + 10, 1, conv1dJdb.Size, out Tensor dJdbInc0);
Assert.IsTrue(conv1dJdw.ContentEquals(dJdwInc0, 1e-5f));
Assert.IsTrue(conv1dJdb.ContentEquals(dJdbInc0, 1e-5f));
Tensor.Reshape((float *)incDjdw.Ptr.ToPointer() + 30 + conv1dJdw.Size, 1, conv2dJdw.Size, out Tensor dJdwInc1);
Tensor.Reshape((float *)incdJdb.Ptr.ToPointer() + 20, 1, conv2dJdb.Size, out Tensor dJdbInc1);
Assert.IsTrue(conv2dJdw.ContentEquals(dJdwInc1, 1e-5f));
Assert.IsTrue(conv2dJdb.ContentEquals(dJdbInc1, 1e-5f));
// Cleanup
Tensor.Free(zTemp, aTemp, zConv, aConv, zInc, aInc, reshaped, conv2dx, conv1dx, incdx, conv2dJdw, conv2dJdb, conv1dJdw, conv1dJdb, incDjdw, incdJdb);
}
}
public unsafe void Inception1x1()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(10), 32 * 32 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(10, 32 * 32 * 3);
CuDnnConvolutionalLayer conv = new CuDnnConvolutionalLayer(TensorInfo.Image <Rgb24>(32, 32), ConvolutionInfo.New(ConvolutionMode.CrossCorrelation), (1, 1), 10, ActivationType.ReLU, BiasInitializationMode.Gaussian);
CuDnnInceptionLayer inception = new CuDnnInceptionLayer(conv.InputInfo, InceptionInfo.New(10, 10, 10, 10, 10, PoolingMode.Max, 10));
fixed(float *pw = inception.Weights)
Unsafe.InitBlock(pw, 0, (uint)(sizeof(float) * inception.Weights.Length));
Buffer.BlockCopy(conv.Weights, 0, inception.Weights, 0, sizeof(float) * conv.Weights.Length);
Buffer.BlockCopy(conv.Biases, 0, inception.Biases, 0, sizeof(float) * conv.Biases.Length);
fixed(float *px = x)
{
// Forward + Z
Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xTensor);
conv.Forward(xTensor, out Tensor zConv, out Tensor aConv);
inception.Forward(xTensor, out Tensor zInc, out Tensor aInc);
Tensor.New(zConv.Entities, zConv.Length, out Tensor reshaped);
float *pzInc = (float *)zInc.Ptr.ToPointer(), preshaped = (float *)reshaped.Ptr.ToPointer();
for (int i = 0; i < zConv.Entities; i++)
{
Buffer.MemoryCopy(pzInc + i * zInc.Length, preshaped + i * zConv.Length, sizeof(float) * zConv.Length, sizeof(float) * zConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(zConv));
// A
float *paInc = (float *)aInc.Ptr.ToPointer();
for (int i = 0; i < aConv.Entities; i++)
{
Buffer.MemoryCopy(paInc + i * aInc.Length, preshaped + i * aConv.Length, sizeof(float) * aConv.Length, sizeof(float) * aConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(aConv));
// Backpropagate
Tensor.Like(xTensor, out Tensor dx1);
Tensor.Like(xTensor, out Tensor dx2);
conv.Backpropagate(xTensor, zConv, aConv, dx1, out Tensor dJdw1, out Tensor dJdb1);
inception.Backpropagate(xTensor, zInc, aInc, dx2, out Tensor dJdw2, out Tensor dJdb2);
Assert.IsTrue(dx1.ContentEquals(dx2));
Tensor.Reshape((float *)dJdw2.Ptr.ToPointer(), 1, dJdw1.Size, out dJdw2);
Tensor.Reshape((float *)dJdb2.Ptr.ToPointer(), 1, dJdb1.Size, out dJdb2);
Assert.IsTrue(dJdw1.ContentEquals(dJdw2, 1e-5f));
Assert.IsTrue(dJdb1.ContentEquals(dJdb2, 1e-5f));
// Cleanup
Tensor.Free(zConv, aConv, zInc, aInc, reshaped, dx1, dx2, dJdw1, dJdw2, dJdb1, dJdb2);
}
}
public unsafe void Inception3x3Pipeline()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.Linear(10), 32 * 32 * 3, WeightsInitializationMode.GlorotNormal).AsSpan().AsMatrix(10, 32 * 32 * 3);
CuDnnConvolutionalLayer
conv1 = new CuDnnConvolutionalLayer(TensorInfo.Image <Rgb24>(32, 32), ConvolutionInfo.New(ConvolutionMode.CrossCorrelation), (1, 1), 10, ActivationFunctionType.ReLU, BiasInitializationMode.Gaussian),
conv2 = new CuDnnConvolutionalLayer(conv1.OutputInfo, ConvolutionInfo.New(ConvolutionMode.CrossCorrelation, 1, 1), (3, 3), 10, ActivationFunctionType.ReLU, BiasInitializationMode.Gaussian);
CuDnnInceptionLayer inception = new CuDnnInceptionLayer(TensorInfo.Image <Rgb24>(32, 32), InceptionInfo.New(10, 10, 10, 10, 10, PoolingMode.Max, 10));
fixed(float *pw = inception.Weights)
Unsafe.InitBlock(pw, 0, (uint)(sizeof(float) * inception.Weights.Length));
Buffer.BlockCopy(conv1.Weights, 0, inception.Weights, sizeof(float) * 3 * 10, sizeof(float) * conv1.Weights.Length);
Buffer.BlockCopy(conv2.Weights, 0, inception.Weights, sizeof(float) * 3 * 10 + sizeof(float) * conv1.Weights.Length, sizeof(float) * conv2.Weights.Length);
Buffer.BlockCopy(conv1.Biases, 0, inception.Biases, sizeof(float) * 10, sizeof(float) * conv1.Biases.Length);
Buffer.BlockCopy(conv2.Biases, 0, inception.Biases, sizeof(float) * 20, sizeof(float) * conv2.Biases.Length);
fixed(float *px = x)
{
// Forward + Z
Tensor.Reshape(px, x.GetLength(0), x.GetLength(1), out Tensor xTensor);
conv1.Forward(xTensor, out Tensor zTemp, out Tensor aTemp);
conv2.Forward(aTemp, out Tensor zConv, out Tensor aConv);
inception.Forward(xTensor, out Tensor zInc, out Tensor aInc);
Tensor.New(zConv.Entities, zConv.Length, out Tensor reshaped);
float *pzInc = (float *)zInc.Ptr.ToPointer() + 32 * 32 * 10, preshaped = (float *)reshaped.Ptr.ToPointer();
for (int i = 0; i < zConv.Entities; i++)
{
Buffer.MemoryCopy(pzInc + i * zInc.Length, preshaped + i * zConv.Length, sizeof(float) * zConv.Length, sizeof(float) * zConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(zConv));
// A
float *paInc = (float *)aInc.Ptr.ToPointer() + 32 * 32 * 10;
for (int i = 0; i < aConv.Entities; i++)
{
Buffer.MemoryCopy(paInc + i * aInc.Length, preshaped + i * aConv.Length, sizeof(float) * aConv.Length, sizeof(float) * aConv.Length);
}
Assert.IsTrue(reshaped.ContentEquals(aConv));
// Backpropagation
Tensor.New(xTensor.Entities, xTensor.Length, out Tensor z1);
KerasWeightsProvider.FillWithHeEtAlUniform(z1, 10);
z1.Duplicate(out Tensor z2);
conv2.Backpropagate(Tensor.Null, aConv, zTemp, conv1.ActivationFunctions.ActivationPrime);
conv1.Backpropagate(Tensor.Null, zTemp, z1, ActivationFunctions.ReLUPrime);
inception.Backpropagate(xTensor, aInc, z2, ActivationFunctions.ReLUPrime);
Assert.IsTrue(z1.ContentEquals(z2));
// Gradient
Tensor.New(xTensor.Entities, xTensor.Length, out Tensor a);
KerasWeightsProvider.FillWithHeEtAlUniform(a, 10);
conv1.ComputeGradient(a, zTemp, out Tensor dJdwConv1, out Tensor dJdbConv1);
conv2.ComputeGradient(aTemp, aConv, out Tensor dJdwConv2, out Tensor dJdbConv2);
inception.ComputeGradient(a, aInc, out Tensor dJdwInc, out Tensor dJdbInc);
Tensor.Reshape((float *)dJdwInc.Ptr.ToPointer() + 30, 1, dJdwConv1.Size, out Tensor dJdwInc0);
Tensor.Reshape((float *)dJdbInc.Ptr.ToPointer() + 10, 1, dJdbConv1.Size, out Tensor dJdbInc0);
Assert.IsTrue(dJdwConv1.ContentEquals(dJdwInc0, 1e-5f));
Assert.IsTrue(dJdbConv1.ContentEquals(dJdbInc0, 1e-5f));
Tensor.Reshape((float *)dJdwInc.Ptr.ToPointer() + 30 + dJdwConv1.Size, 1, dJdwConv2.Size, out Tensor dJdwInc1);
Tensor.Reshape((float *)dJdbInc.Ptr.ToPointer() + 20, 1, dJdbConv2.Size, out Tensor dJdbInc1);
Assert.IsTrue(dJdwConv2.ContentEquals(dJdwInc1, 1e-5f));
Assert.IsTrue(dJdbConv2.ContentEquals(dJdbInc1, 1e-5f));
// Cleanup
z1.Free();
z2.Free();
zTemp.Free();
zConv.Free();
zInc.Free();
aConv.Free();
aInc.Free();
reshaped.Free();
}
}
