public IMatrix Execute(IMatrix error, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
{
var matrixList = error.AsIndexable().Columns.Select(v => v.ToArray()).ToList();
var newMatrixList = new List <IMatrix>();
Tuple <int, int> newIndex;
for (var i = 0; i < matrixList.Count; i++)
{
var matrix = matrixList[i];
var table = _indexPosList[i];
newMatrixList.Add(_lap.Create(_rows, _columns, (x, y) => {
if (table.TryGetValue(Tuple.Create(x, y), out newIndex))
{
var newIndex2 = newIndex.Item1 * _newRows + newIndex.Item2;
return(matrix[newIndex2]);
}
return(0f);
}));
}
using (var tensor = _lap.CreateTensor(newMatrixList)) {
var ret = tensor.ConvertToMatrix();
foreach (var item in newMatrixList)
{
item.Dispose();
}
return(ret);
}
}
float _DifferenceCost(IMatrix m1, IMatrix m2)
{
return(m1.AsIndexable().Rows
.Zip(m2.AsIndexable().Rows, (r1, r2) => _costFunction.Compute(r1, r2))
.Average()
);
}
public IReadOnlyList <IFeedForwardOutput> Execute(ITrainingDataProvider data)
{
IMatrix curr = null;
var ret = new List <IFeedForwardOutput>();
foreach (var miniBatch in _GetMiniBatches(data, false, DEFAULT_BATCH_SIZE))
{
var garbage = new List <IMatrix>();
garbage.Add(curr = miniBatch.Input);
garbage.Add(miniBatch.ExpectedOutput);
// feed forward
foreach (var layer in _layer)
{
garbage.Add(curr = layer.FeedForward(curr, false));
}
// break the output into rows
ret.AddRange(curr.AsIndexable().Rows.Zip(miniBatch.ExpectedOutput.AsIndexable().Rows, (a, e) => new FeedForwardOutput(a, e)));
// clear memory
garbage.ForEach(m => m.Dispose());
}
return(ret);
}
public IEnumerable <IIndexableVector[]> ExecuteToLayer(ITrainingDataProvider data, int layerDepth)
{
IMatrix curr = null;
foreach (var miniBatch in _GetMiniBatches(data, false, DEFAULT_BATCH_SIZE))
{
var garbage = new List <IMatrix>();
garbage.Add(curr = miniBatch.Input);
garbage.Add(miniBatch.ExpectedOutput);
// feed forward
for (var i = 0; i < layerDepth; i++)
{
var layer = _layer[i];
garbage.Add(curr = layer.FeedForward(curr, false));
}
var ret = curr.AsIndexable().Rows.ToList();
// clear memory
garbage.ForEach(m => m.Dispose());
yield return(ret.ToArray());
}
}
public void Train(ITrainingDataProvider trainingData, int numEpochs, ITrainingContext context)
{
IMatrix curr = null;
var additionalBackpropagation = trainingData as ICanBackpropagate;
for (int i = 0; i < numEpochs && context.ShouldContinue; i++)
{
context.StartEpoch(trainingData.Count);
trainingData.StartEpoch();
var batchErrorList = new List <double>();
// iterate over each mini batch
foreach (var miniBatch in _GetMiniBatches(trainingData, _stochastic, context.MiniBatchSize))
{
var garbage = new List <IMatrix>();
garbage.Add(curr = miniBatch.Input);
_lap.PushLayer();
// set up the layer stack
var layerStack = new Stack <ICanBackpropagate>();
if (additionalBackpropagation != null)
{
layerStack.Push(additionalBackpropagation);
}
// feed forward
foreach (var layer in _layer)
{
garbage.Add(curr = layer.FeedForward(curr, true));
layerStack.Push(layer);
}
// calculate the error against the training examples
using (var expectedOutput = miniBatch.ExpectedOutput) {
garbage.Add(curr = context.ErrorMetric.CalculateDelta(curr, expectedOutput));
// calculate the training error for this mini batch
if (_calculateTrainingError)
{
batchErrorList.Add(curr.AsIndexable().Values.Select(v => Math.Pow(v, 2)).Average() / 2);
}
// backpropagate the error
while (layerStack.Any())
{
var currentLayer = layerStack.Pop();
garbage.Add(curr = currentLayer.Backpropagate(curr, context, layerStack.Any()));
}
}
// clear memory
context.EndBatch();
garbage.ForEach(m => m?.Dispose());
_lap.PopLayer();
}
context.EndEpoch(_calculateTrainingError ? batchErrorList.Average() : 0f);
}
}
double _VerifyDerivatives(IMatrix activationDerivative)
{
const float epsilon = 0.0001f;
var activation = _layerUpdater.Layer.Activation;
return(_layerOutput.AsIndexable().Values.Zip(activationDerivative.AsIndexable().Values, (val, valD) => {
var approximatedDerivative = (activation.Calculate(val + epsilon) - activation.Calculate(val - epsilon)) / (2 * epsilon);
return Math.Abs(valD - approximatedDerivative);
}).Average());
}
void _TestNode(INode node, IMatrix forwardInput, IMatrix expectedForwardOutput, IMatrix backwardInput, IMatrix expectedBackwardOutput)
{
var context = new TestingContext(_cpu);
var matrix = forwardInput.AsIndexable();
context.Data = matrix.AsGraphData();
node.ExecuteForward(context, 0);
var output = context.Forward.First();
var outputMatrix = output.Item1.Data.GetMatrix();
FloatingPointHelper.AssertEqual(outputMatrix.AsIndexable(), expectedForwardOutput.AsIndexable());
output.Item2.Backward(null, backwardInput.Clone().AsGraphData(), context, new[] { node });
var bpOutput = context.Backward.First().Item1.GetMatrix();
FloatingPointHelper.AssertEqual(bpOutput.AsIndexable(), expectedBackwardOutput.AsIndexable());
}
public IReadOnlyList <IReadOnlyList <IIndexableVector> > Cluster(int numIterations, Action <float> callback, float errorThreshold = 0.001f)
{
for (int i = 0; i < numIterations; i++)
{
using (var wh = _weights.Multiply(_features)) {
var cost = _DifferenceCost(_dataMatrix, wh);
callback(cost);
if (cost <= errorThreshold)
{
break;
}
using (var wT = _weights.Transpose())
using (var hn = wT.Multiply(_dataMatrix))
using (var wTw = wT.Multiply(_weights))
using (var hd = wTw.Multiply(_features))
using (var fhn = _features.PointwiseMultiply(hn)) {
_features.Dispose();
_features = fhn.PointwiseDivide(hd);
}
using (var fT = _features.Transpose())
using (var wn = _dataMatrix.Multiply(fT))
using (var wf = _weights.Multiply(_features))
using (var wd = wf.Multiply(fT))
using (var wwn = _weights.PointwiseMultiply(wn)) {
_weights.Dispose();
_weights = wwn.PointwiseDivide(wd);
}
}
}
// weights gives cluster membership
return(_weights.AsIndexable().Rows
.Select((c, i) => Tuple.Create(i, c.MaximumIndex()))
.GroupBy(d => d.Item2)
.Select(g => g.Select(d => _data[d.Item1]).ToArray())
.ToList()
);
}
