public void DenseLayer_MultipleBackwardsPasses()
{
const int fanIn = 5;
const int batchSize = 2;
const int neuronCount = 3;
var random = new Random(232);
var sut = new DenseLayer(neuronCount, Activation.Undefined);
sut.Initialize(5, 1, 1, batchSize, Initialization.GlorotUniform, random);
var input = Matrix <float> .Build.Random(batchSize, fanIn, random.Next());
sut.Forward(input);
var delta = Matrix <float> .Build.Dense(batchSize, neuronCount, 1.0f);
var expected = Matrix <float> .Build.Dense(batchSize, fanIn);
sut.Backward(delta).CopyTo(expected);
for (int i = 0; i < 20; i++)
{
var actual = sut.Backward(delta);
Assert.AreEqual(expected, actual);
}
}
public void DenseLayer_Backward()
{
const int fanIn = 5;
const int batchSize = 2;
const int neuronCount = 3;
var random = new Random(232);
var sut = new DenseLayer(neuronCount, Activation.Undefined);
sut.Initialize(5, 1, 1, batchSize, Initialization.GlorotUniform, random);
var input = Matrix <float> .Build.Random(batchSize, fanIn, random.Next());
sut.Forward(input);
var delta = Matrix <float> .Build.Random(batchSize, neuronCount, random.Next());
var actual = sut.Backward(delta);
Trace.WriteLine(string.Join(", ", actual.ToColumnMajorArray()));
var expected = Matrix <float> .Build.Dense(batchSize, fanIn, new float[] { 0.001748383f, -0.2615477f, -0.6422306f, -0.01443626f, 0.4605991f, -0.7384186f, -0.6931117f, 0.1083627f, -0.6230267f, -1.20742f });
MatrixAsserts.AreEqual(expected, actual);
}
IEnumerator TrainNetwork()
{
var inputlayer = new DenseLayer(10, 4, Activation.ReLU());
var hiddenlayer1 = new DenseLayer(inputlayer, Activation.ReLU(), 30, LayerType.Hidden);
var hiddenlayer2 = new DenseLayer(hiddenlayer1, Activation.ReLU(), 500, LayerType.Hidden);
var outputlayer = new DenseLayer(hiddenlayer1, Activation.TangesHyperbolic(), 2, LayerType.Output);
Network = outputlayer;
outputlayer.Initilize();
var trainingdata = GenerateTrainingData();
int epoch = 0;
int epochsize = trainingdata.GetLength(0);
var epocherror = float.MaxValue;
yield return(0);
while (epocherror > 10 && _Training)
{
yield return(0);
epocherror = 0f;
epoch++;
for (var t = 0; t < epochsize; t++)
{
var truth = trainingdata[t, 1];
var input = trainingdata[t, 0];
var output = (Tensor1D)outputlayer.Forward(input);
var dif = output - truth;
var sq = dif * dif;
epocherror += (float)Math.Pow(sq.ElementSum(), 2);
outputlayer.Backward(dif);
}
ErrorText.text = epocherror.ToString();
}
ErrorText.text = ("Finished!");
_Training = false;
}
