public override void ApplySigmoid2(Matrix <Float> matrix1, Matrix <Float> matrix2)
{
using (var ptrs = new MatrixPointersBag <Float>(matrix1, matrix2))
{
ApplySigmoid2(ptrs[0], ptrs[1], matrix1.Length());
}
}
public override void ApplyTanh(Matrix <Float> matrix)
{
using (var ptrs = new MatrixPointersBag <Float>(matrix))
{
ApplyTanh(ptrs[0], matrix.Length());
}
}
public override void AdagradUpdate(Float learningRate, NeuroWeight <Float> weight)
{
using (var ptrs = new MatrixPointersBag <Float>(weight.Weight, weight.Cache2, weight.Gradient))
{
AdagradUpdate(learningRate, ptrs[0], ptrs[1], ptrs[2], weight.Weight.Length());
}
}
public override void AdamUpdate(float learningRate, float b1, float b2, NeuroWeight <Float> weight)
{
using (var ptrs = new MatrixPointersBag <Float>(weight.Weight, weight.Cache1, weight.Cache2, weight.Gradient))
{
AdamUpdate(learningRate, b1, b2, weight.Timestep, ptrs[0], ptrs[1], ptrs[2], ptrs[3], weight.Weight.Length());
}
}
public override void CalculateH(Matrix <Float> H, Matrix <Float> hCandidate, Matrix <Float> z, Matrix <Float> lastH)
{
using (var ptrs = new MatrixPointersBag <Float>(H, hCandidate, z, lastH))
{
CalculateH(ptrs[0], ptrs[1], ptrs[2], ptrs[3], H.Length());
}
}
public override void GravesRmsPropUpdate(float weightDecay, float learningRate, float decayRate, float momentum, NeuroWeight <Float> weight)
{
using (var ptrs = new MatrixPointersBag <Float>(weight.Weight, weight.Cache1, weight.Cache2, weight.CacheM, weight.Gradient))
{
GravesRMSPropUpdate(weightDecay, learningRate, decayRate, momentum, ptrs[0], ptrs[1], ptrs[2], ptrs[3], ptrs[4], weight.Weight.Length());
}
}
public void RMSPropValuesAreEqual(float learningRate, float decayRate, float weightDecay, float momentum)
{
var local = new NeuroWeight <float>(MatrixFactory.RandomMatrix <float>(10, 10, 1e-2f));
var remote = local.Clone();
for (int i = 0; i < 100; i++)
{
var grad = MatrixFactory.RandomMatrix <float>(10, 10, 1.0f);
grad.CopyTo(local.Gradient);
grad.CopyTo(remote.Gradient);
MathProvider.GravesRmsPropUpdate(weightDecay, learningRate, decayRate, momentum, local);
using (var ptrs = new MatrixPointersBag <float>(true, remote.Weight, remote.Gradient, remote.Cache1, remote.Cache2, remote.CacheM))
{
Interface.TestRMSPropUpdate(ptrs.Definitions[0], ptrs.Definitions[1], ptrs.Definitions[2], ptrs.Definitions[3], ptrs.Definitions[4],
learningRate, decayRate, momentum, weightDecay);
}
local.Weight.ShouldMatrixEqualWithinError(remote.Weight);
local.Cache1.ShouldMatrixEqualWithinError(remote.Cache1);
local.Cache2.ShouldMatrixEqualWithinError(remote.Cache2);
local.CacheM.ShouldMatrixEqualWithinError(remote.CacheM);
local.Gradient.ShouldMatrixEqualWithinError(remote.Gradient);
}
}
private unsafe void TestGpuLayer(LayerBase <float> layer, TestDataSet <float> dataSet, int?outSize = null)
{
layer.Initialize(dataSet.BatchSize, dataSet.SampleCount);
var gpuLayer = layer.CreateGpuLayer();
int finalOutSize = outSize.GetValueOrDefault(dataSet.TargetSize);
for (int step = 0; step < 3; step++) // Making 3 runs
{
var seq = dataSet.GetNextSamples(dataSet.SampleCount);
var cpuOut = new List <Matrix <float> >();
layer.InitSequence();
foreach (var input in seq.Inputs)
{
cpuOut.Add(layer.Step(input));
}
var gpuOut = Enumerable.Range(0, dataSet.SampleCount).Select(x => Matrix <float> .Build.Dense(finalOutSize, dataSet.BatchSize)).ToArray();
using (var inPtrs = new MatrixPointersBag <float>(true, seq.Inputs.ToArray()))
using (var outPtrs = new MatrixPointersBag <float>(true, gpuOut))
{
fixed(MatrixDefinition *inDef = &inPtrs.Definitions[0], outDef = &outPtrs.Definitions[0])
{
GpuInterface.LayerForwardSequence(gpuLayer, inDef, seq.Inputs.Count);
GpuInterface.TransferLayerOutputsToHost(gpuLayer, outDef, gpuOut.Length);
}
}
//Console.WriteLine("CPU output:");
//foreach (var matrix in cpuOut)
//{
// Console.WriteLine(matrix.ToMatrixString());
// Console.WriteLine("---------------");
//}
//Console.WriteLine("GPU output:");
//foreach (var matrix in gpuOut)
//{
// Console.WriteLine(matrix.ToMatrixString());
// Console.WriteLine("---------------");
//}
//Console.WriteLine("================================");
for (int sample = 0; sample < dataSet.SampleCount; sample++)
{
cpuOut[sample].ShouldMatrixEqualWithinError(gpuOut[sample]);
}
}
}
public void CanTransferMatrixRowMajor()
{
var local = MatrixFactory.RandomMatrix <float>(2, 3, 5.0f);
var remote = local.CloneMatrix();
MutateMatrixRowMajor(local);
using (var ptrs = new MatrixPointersBag <float>(true, true, remote))
{
Interface.TestMatrixTransferRowMajor(ptrs.Definitions[0]);
}
local.ShouldMatrixEqualWithinError(remote);
}
public unsafe void CantAccesMatrixPointersWhenDisposed()
{
var m1 = MatrixFactory.Create <float>(2, 3);
var ptrs = new MatrixPointersBag <float>(m1);
fixed(void *fp1 = &m1.AsColumnMajorArray()[0])
{
(ptrs[0].ToPointer() == fp1).ShouldBeTrue();
}
ptrs.Dispose();
Trap.Exception(() => ptrs[0]).ShouldBeInstanceOf <ObjectDisposedException>();
}
public void CanClampMatrix()
{
var local = MatrixFactory.ParseString <float>(@"2 5
-6 1");
var remote = local.Clone();
local.Clamp(-4.0f, 4.0f);
local.AsColumnMajorArray().ShouldArrayEqualWithinError(MathProvider.Array(2.0f, -4.0f, 4.0f, 1.0f));
using (var ptrs = new MatrixPointersBag <float>(true, remote))
{
Interface.TestClampMatrix(ptrs.Definitions[0], 4.0f);
remote.AsColumnMajorArray().ShouldArrayEqualWithinError(MathProvider.Array(2.0f, -4.0f, 4.0f, 1.0f));
}
}
public void CrossEntropyBackpropagationsAreEqual()
{
var m1 = MatrixFactory.RandomMatrix <float>(100, 100, 1e-5f, 1.0f);
var m2 = MatrixFactory.RandomMatrix <float>(100, 100, 1e-5f, 1.0f);
var remoteResult = MatrixFactory.Create <float>(100, 100);
Matrix <float> local;
using (var matrixPtrs = new MatrixPointersBag <float>(true, m1.CloneMatrix(), m2.CloneMatrix(), remoteResult))
{
local = MathProvider.BackPropagateCrossEntropyError(m1, m2);
Interface.TestCrossEntropyBackprop(matrixPtrs.Definitions[0], matrixPtrs.Definitions[1], matrixPtrs.Definitions[2]);
}
remoteResult.ShouldMatrixEqualWithinError(local);
}
public void CrossEntropyErrorsAreEqual()
{
var m1 = MatrixFactory.RandomMatrix <float>(100, 100, 1e-5f, 1.0f);
var m2 = MatrixFactory.RandomMatrix <float>(100, 100, 1e-5f, 1.0f);
using (var matrixPtrs = new MatrixPointersBag <float>(true, m1.CloneMatrix(), m2.CloneMatrix()))
{
double local = MathProvider.CrossEntropyError(m1, m2);
double remote = Interface.TestCrossEntropyError(matrixPtrs.Definitions[0], matrixPtrs.Definitions[1]);
double.IsNaN(local).ShouldBeFalse();
double.IsNaN(remote).ShouldBeFalse();
remote.ShouldEqualWithinError(local);
}
}
public unsafe void CanPinMatrixPointers()
{
var m1 = MatrixFactory.Create <float>(2, 3);
var m2 = MatrixFactory.Create <float>(2, 3);
var ptrs = new MatrixPointersBag <float>(m1, m2);
var p1 = ptrs[0];
var p2 = ptrs[1];
GC.Collect();
ptrs[0].ShouldEqual(p1);
ptrs[1].ShouldEqual(p2);
fixed(void *fp1 = &m1.AsColumnMajorArray()[0], fp2 = &m2.AsColumnMajorArray()[0])
{
(p1.ToPointer() == fp1).ShouldBeTrue();
(p2.ToPointer() == fp2).ShouldBeTrue();
}
ptrs.Dispose();
}
public override unsafe double TrainSequence(List <Matrix <T> > inputs, List <Matrix <T> > targets)
{
#if !CPUONLY
if (_gpuNetworkPtr != IntPtr.Zero)
{
if (typeof(T) != typeof(float))
{
throw new InvalidOperationException("GPU is only supported for float data type!");
}
using (var inPtrs = new MatrixPointersBag <T>(true, inputs.ToArray()))
using (var targPtrs = new MatrixPointersBag <T>(true, targets.ToArray()))
{
fixed(MatrixDefinition *inPtr = &inPtrs.Definitions[0], targPtr = &targPtrs.Definitions[0])
{
return(GpuInterface.TrainSequence(_gpuNetworkPtr, inPtr, targPtr, inputs.Count));
}
}
}
#endif
return(base.TrainSequence(inputs, targets));
}
