public void Softmax()
{
var input1 = NewVolume(new[] { 0.0, 0.0, 0.0, 10000.0 }, new Shape(1, 1, -1, 1));
var softmax1 = input1.Softmax();
AssertNumber.AreEqual(0.0, softmax1.Get(0, 0, 0, 0));
AssertNumber.AreEqual(0.0, softmax1.Get(0, 0, 1, 0));
AssertNumber.AreEqual(0.0, softmax1.Get(0, 0, 2, 0));
AssertNumber.AreEqual(1.0, softmax1.Get(0, 0, 3, 0));
var input2 = NewVolume(new[] { 10000.0, 0.0, 0.0, 10000.0 }, new Shape(1, 1, -1, 1));
var softmax2 = input2.Softmax();
AssertNumber.AreEqual(0.5, softmax2.Get(0, 0, 0, 0));
AssertNumber.AreEqual(0.5, softmax2.Get(0, 0, 3, 0));
}
public void DoConcat()
{
var left = NewVolume(new[] { 1.0, 2.0, 3.0, 4.0 }, new Shape(2, 2, 1, 1));
var right = NewVolume(new[] { 5.0, 6.0, 7.0 }, new Shape(3, 1, 1, 1));
var result = NewVolume(new double[7], new Shape(1, 1, 7, 1));
left.DoConcat(right, result);
AssertNumber.AreEqual(1.0, result.Get(0, 0, 0, 0));
AssertNumber.AreEqual(2.0, result.Get(0, 0, 1, 0));
AssertNumber.AreEqual(3.0, result.Get(0, 0, 2, 0));
AssertNumber.AreEqual(4.0, result.Get(0, 0, 3, 0));
AssertNumber.AreEqual(5.0, result.Get(0, 0, 4, 0));
AssertNumber.AreEqual(6.0, result.Get(0, 0, 5, 0));
AssertNumber.AreEqual(7.0, result.Get(0, 0, 6, 0));
}
public void Norm12DBatch()
{
var x = NewVolume(
new[]
{
1.0, 2.0,
3.0, 4.0,
7.0, -20.0,
3.0, 4.0
}, new Shape(2, 2, 1, 2));
var result = BuilderInstance <T> .Volume.SameAs(new Shape(1, 1, 1, 2));
x.DoNorm1(result);
AssertNumber.AreEqual(10.0, result.Get(0));
AssertNumber.AreEqual(34.0, result.Get(1));
}
public void DoConcatBroadcast()
{
var left = NewVolume(new[] { 1.0, 2.0, 3.0, 4.0 }, new Shape(1, 1, 1, 4));
var right = NewVolume(new[] { 1.0 }, new Shape(1));
var result = NewVolume(new double[8], new Shape(1, 1, 2, 4));
left.DoConcat(right, result);
AssertNumber.AreEqual(1.0, result.Get(0, 0, 0, 0));
AssertNumber.AreEqual(1.0, result.Get(0, 0, 1, 0));
AssertNumber.AreEqual(2.0, result.Get(0, 0, 0, 1));
AssertNumber.AreEqual(1.0, result.Get(0, 0, 1, 1));
AssertNumber.AreEqual(3.0, result.Get(0, 0, 0, 2));
AssertNumber.AreEqual(1.0, result.Get(0, 0, 1, 2));
AssertNumber.AreEqual(4.0, result.Get(0, 0, 0, 3));
AssertNumber.AreEqual(1.0, result.Get(0, 0, 1, 3));
}
public void SoftmaxBatch()
{
var volume1 = NewVolume(new[]
{
0.0, 0.0, 0.0, 10000.0,
0.0, 0.0, 10000.0, 0.0
}, new Shape(1, 1, -1, 2));
var softmax1 = volume1.Softmax();
AssertNumber.AreEqual(0.0, softmax1.Get(0, 0, 0, 0));
AssertNumber.AreEqual(0.0, softmax1.Get(0, 0, 1, 0));
AssertNumber.AreEqual(0.0, softmax1.Get(0, 0, 2, 0));
AssertNumber.AreEqual(1.0, softmax1.Get(0, 0, 3, 0));
AssertNumber.AreEqual(0.0, softmax1.Get(0, 0, 0, 1));
AssertNumber.AreEqual(0.0, softmax1.Get(0, 0, 1, 1));
AssertNumber.AreEqual(1.0, softmax1.Get(0, 0, 2, 1));
AssertNumber.AreEqual(0.0, softmax1.Get(0, 0, 3, 1));
}
public void BiasBackward()
{
var outputGradient = NewVolume(
new[]
{
1.0, 2.0,
3.0, 1.0,
2.0, 3.0
},
new Shape(2, 1, 3, 1));
var biasGradient = BuilderInstance <T> .Volume.SameAs(new Shape(1, 1, 3, 1));
outputGradient.BiasGradient(biasGradient);
AssertNumber.AreEqual(3.0, biasGradient.Get(0, 0, 0, 0));
AssertNumber.AreEqual(4.0, biasGradient.Get(0, 0, 1, 0));
AssertNumber.AreEqual(5.0, biasGradient.Get(0, 0, 2, 0));
}
public void AddBroadcastScalar()
{
var volume = NewVolume(new[]
{
1.0, 2.0,
3.0, 4.0,
1.0, 2.0,
3.0, 4.0,
1.0, 2.0,
3.0, 4.0
}, new Shape(2, 2, 3));
var bias = NewVolume(new[] { 1.0, }, new Shape(1));
var result = BuilderInstance <T> .Volume.SameAs(volume.Shape);
volume.DoAdd(bias, result);
AssertNumber.AreEqual(2.0, result.Get(0, 0, 0));
AssertNumber.AreEqual(2.0, result.Get(0, 0, 1));
AssertNumber.AreEqual(2.0, result.Get(0, 0, 2));
}
public void DoExtract()
{
var x = NewVolume(new[] { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0 }, new Shape(1, 1, 7, 1));
var result = NewVolume(new double[4], new Shape(1, 1, 4, 1));
x.DoExtract(4, 0, result);
AssertNumber.AreEqual(1.0, result.Get(0, 0, 0, 0));
AssertNumber.AreEqual(2.0, result.Get(0, 0, 1, 0));
AssertNumber.AreEqual(3.0, result.Get(0, 0, 2, 0));
AssertNumber.AreEqual(4.0, result.Get(0, 0, 3, 0));
result = NewVolume(new double[3], new Shape(1, 1, 3, 1));
x.DoExtract(3, 4, result);
AssertNumber.AreEqual(5.0, result.Get(0, 0, 0, 0));
AssertNumber.AreEqual(6.0, result.Get(0, 0, 1, 0));
AssertNumber.AreEqual(7.0, result.Get(0, 0, 2, 0));
}
public void Pool2DGradient()
{
var inputActivation = NewVolume(new[]
{
1.0, 0.0, 1.0, 1.0,
1.0, 0.0, 1.0, 7.0,
2.0, 0.0, 1.0, 1.0,
1.0, 0.0, 4.0, 1.0
}, new Shape(4, 4));
var outputActivation = inputActivation.Pool(2, 2, 0, 2);
var outputActivationGradient = NewVolume(new[] { 1.0, 1.0, 1.0, 1.0 }, new Shape(2, 2));
var result = outputActivation.PoolGradient(inputActivation, outputActivationGradient, 2, 2, 0, 2);
AssertNumber.AreEqual(1.0, result.Get(0, 0));
AssertNumber.AreEqual(1.0, result.Get(3, 1));
AssertNumber.AreEqual(1.0, result.Get(0, 2));
AssertNumber.AreEqual(1.0, result.Get(2, 3));
}
public void FullyCon()
{
// 1x3x1x1
var input = NewVolume(new[] { 1.0, 2.0, 3.0 }, new Shape(1, 1, 3, 1));
// 1x1x3x2
var filter = NewVolume(
new[] { 1.0, 1.0, 1.0, 2.0, 2.0, 2.0 },
new Shape(1, 1, 3, 2));
var result = input.Convolve(filter, 0, 1);
// 1x1x2x1
Assert.AreEqual(1, result.Shape.GetDimension(0));
Assert.AreEqual(1, result.Shape.GetDimension(1));
Assert.AreEqual(2, result.Shape.GetDimension(2));
Assert.AreEqual(1, result.Shape.GetDimension(3));
AssertNumber.AreEqual(6.0, result.Storage.Get(0, 0, 0));
AssertNumber.AreEqual(12.0, result.Storage.Get(0, 0, 1));
}
public void SoftmaxGradient()
{
// input = [1, 0.1, 0.1, 0.1]
var input = NewVolume(new[] { 1.0, 0.1, 0.1, 0.1 }, new Shape(1, 1, -1, 1));
// output = softmax(input)
var output = input.Softmax();
// groundTruth = [0, 1, 0 , 0]
var correctClass = 1;
var groundTruth = NewVolume(new double[4], new Shape(1, 1, -1, 1));
groundTruth.Set(0, 0, correctClass, (T)Convert.ChangeType(1.0, typeof(T)));
var inputGradient = output.SoftmaxGradient(groundTruth);
AssertNumber.AreEqual(-0.08251689706523138, inputGradient.Get(0, 0, 0, 0), 1e-4);
AssertNumber.AreEqual(0.14961463059055374, inputGradient.Get(0, 0, 1, 0), 1e-4);
AssertNumber.AreEqual(-0.033548866762661167, inputGradient.Get(0, 0, 2, 0), 1e-4);
AssertNumber.AreEqual(-0.033548866762661167, inputGradient.Get(0, 0, 3, 0), 1e-4);
}
public void Tile2D()
{
var x = NewVolume(new[] { 1.0, 2.0, 3.0 }, new Shape(3));
var reps = NewVolume(new[] { 2.0, 2.0 }, new Shape(2));
var result = BuilderInstance <T> .Volume.SameAs(new Shape(6, 2));
x.DoTile(reps, result);
AssertNumber.AreEqual(1.0, result.Get(0, 0));
AssertNumber.AreEqual(2.0, result.Get(1, 0));
AssertNumber.AreEqual(3.0, result.Get(2, 0));
AssertNumber.AreEqual(1.0, result.Get(3, 0));
AssertNumber.AreEqual(2.0, result.Get(4, 0));
AssertNumber.AreEqual(3.0, result.Get(5, 0));
AssertNumber.AreEqual(1.0, result.Get(0, 1));
AssertNumber.AreEqual(2.0, result.Get(1, 1));
AssertNumber.AreEqual(3.0, result.Get(2, 1));
AssertNumber.AreEqual(1.0, result.Get(3, 1));
AssertNumber.AreEqual(2.0, result.Get(4, 1));
AssertNumber.AreEqual(3.0, result.Get(5, 1));
}
public void ConvolveGradientBatch()
{
// 3x3x3x2
var input = NewVolume(new double[27 * 2].Populate(1.0), new Shape(3, 3, 3, 2));
// 2x2x3x2
var filter = NewVolume(
new double[12].Populate(1.0).Concat(new double[12].Populate(2.0f)).ToArray(),
new Shape(2, 2, 3, 2));
var outputGradient = NewVolume(new[]
{
2.0, 3.0,
4.0, 5.0
}, new Shape(1, 1, 2, 2));
var inputGradient = BuilderInstance <T> .Volume.SameAs(input.Storage, input.Shape);
var filterGradient = BuilderInstance <T> .Volume.SameAs(filter.Storage, filter.Shape);
input.DoConvolutionGradient(filter, outputGradient, inputGradient, filterGradient, 0, 2);
// input gradient
AssertNumber.AreEqual(8.0, inputGradient.Get(0, 0, 0, 0));
AssertNumber.AreEqual(0.0, inputGradient.Get(2, 2, 2, 0));
AssertNumber.AreEqual(0.0, inputGradient.Get(2, 2, 1, 0));
AssertNumber.AreEqual(14.0, inputGradient.Get(0, 0, 0, 1));
AssertNumber.AreEqual(0.0, inputGradient.Get(2, 2, 2, 1));
AssertNumber.AreEqual(0.0, inputGradient.Get(2, 2, 1, 1));
// filter gradient
AssertNumber.AreEqual(1.0, filter.Get(0, 0, 0, 0));
AssertNumber.AreEqual(1.0, filter.Get(0, 0, 1, 0));
AssertNumber.AreEqual(1.0, filter.Get(0, 0, 2, 0));
AssertNumber.AreEqual(2.0, filter.Get(0, 0, 0, 1));
AssertNumber.AreEqual(2.0, filter.Get(0, 0, 1, 1));
AssertNumber.AreEqual(2.0, filter.Get(0, 0, 2, 1));
}
public void LeakyRelu()
{
var volume = NewVolume(new[]
{
-1.0, 0.0, 3.0, 5.0,
6.0, 4.0, 0.0, -5.0
}, new Shape(1, 1, 4, 2));
var alpha = (T)Convert.ChangeType(0.01, typeof(T));
var result = volume.LeakyRelu(alpha);
//Adding a delta of 0.005 to account for floating point randomness
AssertNumber.AreEqual(-0.01, result.Get(0, 0, 0, 0), 0.005);
AssertNumber.AreEqual(0.0, result.Get(0, 0, 1, 0), 0.005);
AssertNumber.AreEqual(3.0, result.Get(0, 0, 2, 0), 0.005);
AssertNumber.AreEqual(5.0, result.Get(0, 0, 3, 0), 0.005);
AssertNumber.AreEqual(6.0, result.Get(0, 0, 0, 1), 0.005);
AssertNumber.AreEqual(4.0, result.Get(0, 0, 1, 1), 0.005);
AssertNumber.AreEqual(0.0, result.Get(0, 0, 2, 1), 0.005);
AssertNumber.AreEqual(-0.05, result.Get(0, 0, 3, 1), 0.005);
}
public void Convolve()
{
// 3x3x3x1
var input = NewVolume(new double[27].Populate(1.0), new Shape(3, 3, 3, 1));
// 2x2x3x2
var filter = NewVolume(
new double[12].Populate(1.0f).Concat(new double[12].Populate(2.0)).ToArray(),
new Shape(2, 2, 3, 2));
var result = BuilderInstance <T> .Volume.SameAs(new Shape(1, 1, 2, 1));
input.DoConvolution(filter, 0, 2, result);
// 1x1x2x1
Assert.AreEqual(1, result.Shape.GetDimension(0));
Assert.AreEqual(1, result.Shape.GetDimension(1));
Assert.AreEqual(2, result.Shape.GetDimension(2));
Assert.AreEqual(1, result.Shape.GetDimension(3));
AssertNumber.AreEqual(12.0, result.Storage.Get(0, 0, 0));
AssertNumber.AreEqual(24.0, result.Storage.Get(0, 0, 1));
}