private JobHandle EvaluateInternal(NativeSlice2D <float> inputData, NativeSlice2D <float> outputArray, int count) //Evaluate a single instance
{
//Copy to input layer TODO:Just cast to input layer and set the slice on it
//TODO:Validate size ?
//inputData.Data.CopyTo(InputLayer.OutputActivation.Data);
InputLayer.SetInput(inputData);
//inputData.Data.CopyTo(InputLayer.OutputActivation); //TODO:Re-Work this to take slices
//InputLayer.OutputActivation.CopyFrom(inputData);
//var jobHandle = inputData.CopyToJob(InputLayer.OutputActivation, _lastDependentJobHandle);
JobHandle jobHandle = default;
for (int i = 1; i < _layers.Count; i++)
{
//Evaluate layers
jobHandle = EvaluateLayer(i, jobHandle, count);
}
_lastDependentJobHandle = jobHandle;
var copyToOutputJob = new CopyToOutputJob()
{
Source = OutputLayer.OutputActivation.Slice(0, count),
Destination = outputArray
};
return(copyToOutputJob.Schedule(copyToOutputJob.Source.Dimensions.x, 64, jobHandle));
//Copy to output TODO: Before evaluating, set the output array on the output layer
//return OutputLayer.OutputActivation.BackingStore.CopyToJob(outputArray.BackingSlice, jobHandle);
}
private void Swap(NativeSlice2D <float> slice, int a, int b)
{
for (int i = 0; i < slice.Dimensions.y; i++)
{
var tmp = slice[a, i];
slice[a, i] = slice[b, i];
slice[b, i] = tmp;
}
}
public void GetTrainingCase(int trainingCase, int caseCount, out NativeSlice2D <float> trainingInput, out NativeSlice2D <float> trainingResult)
{
if (trainingCase + caseCount >= TrainingSetSize)
{
throw new ArgumentOutOfRangeException();//TODO:DOCS
}
trainingInput = GetInputSlice(trainingCase, caseCount);
trainingResult = GetResultSlice(trainingCase, caseCount);
}
public void GetTestCase(int testCase, out NativeSlice2D <float> testInput, out NativeSlice2D <float> testResult)
{
if (testCase >= TestingSetSize)
{
throw new ArgumentOutOfRangeException();//TODO:DOCS
}
testInput = GetInputSlice(TrainingSetSize + testCase, 1);
testResult = GetResultSlice(TrainingSetSize + testCase, 1);
}
private void ComputeErrorSum(NativeSlice2D <float> expectedOutput, int testCaseCount, out float errorSum)
{
errorSum = 0.0f;
for (int x = 0; x < testCaseCount; x++)
{
for (int y = 0; y < OutputLayer.OutputActivation.Dimensions.y; y++)
{
errorSum += Math.Abs(OutputLayer.OutputActivation[x, y] - expectedOutput[x, y]);
}
}
errorSum /= testCaseCount;
}
public JobHandle Evaluate(NativeSlice2D <float> inputData, NativeSlice2D <float> outputArray, int count)
{
Profiler.BeginSample("NetworkEvaluator::Evaluate");
var inputLayer = _layers[0];
var outputLayer = _layers[_layers.Count - 1];
if (inputData.Dimensions.y != inputLayer.Size)
{
throw new ArgumentException();//TODO:
}
if (outputArray.Dimensions.y != outputLayer.Size)
{
throw new ArgumentException();//TODO:
}
var result = EvaluateInternal(inputData, outputArray, count);
JobHandle.ScheduleBatchedJobs();
Profiler.EndSample();
return(result);
}
private void TestInitialAccuracy(NetworkEvaluator networkEvaluator, NativeSlice2D <float> tempResult)
{
float totalError = 0.0f;
int totalCorrect = 0;
for (int i = 0; i < _dataset.TestingSetSize; i++)
{
_dataset.GetTestCase(i, out var trainingInput, out var trainingResult);
networkEvaluator.Evaluate(trainingInput, tempResult, 1).Complete();
//var error = Math.Abs(tempResult[0] - trainingResult[0, 0]);
var error = CrossEntropyCost(tempResult[0, 0], trainingResult[0, 0]);
bool wasCorrect = error < 0.5f;
totalCorrect += wasCorrect ? 1 : 0;
totalError += error;
}
float averageError = totalError / (float)_dataset.TestingSetSize;
float accuracy = (float)totalCorrect / (float)_dataset.TestingSetSize;
//Forward test
Debug.Log($"Initial: Accuracy:{accuracy:P2} Average Error:{averageError:F4}");
}
//Oh Boy
public JobHandle GradientDescentBackpropigate(NativeSlice2D <float> inputData, NativeSlice2D <float> expectedOutput, int testCaseCount, out float errorSum)
{
Profiler.BeginSample("NetworkEvaluator::GradientDescentBackpropigate");
var resultArray = new NativeArray2D <float>(testCaseCount, OutputLayer.Size);
var feedforwardHandle = Evaluate(inputData, resultArray.Slice(0, testCaseCount), testCaseCount);
//Now we need to use all the jobs created before to evaluate this.
//Compute output error
/* var computeOutputErrorJob = new ErrorEvaluators.QuadraticSigmoidOutputErrorEvaluator()
* {
* Expected = expectedOutput,
* Actuall = resultArray,
* WeightedActivation = OutputLayer.WeightedInput,
* ErrorOut = OutputLayer.Error
* };*/
var computeOutputErrorJob = new ErrorEvaluators.CrossEntropySigmoidOutputErrorEvaluator()
{
Expected = expectedOutput,
Actuall = resultArray.Slice(0, testCaseCount),
WeightedActivation = OutputLayer.SliceWeightedInputs(0, testCaseCount),
ErrorOut = OutputLayer.SliceError(0, testCaseCount)
};
var outputErrorHandle = computeOutputErrorJob.Schedule(OutputLayer.Error.Dimensions.y, 4, feedforwardHandle);
//Convert that output error to the output node gradient
//outputErrorHandle.Complete();
//Perform backpropigation
JobHandle backpropigationHandle = outputErrorHandle;
for (int layerIndex = _layers.Count - 2; layerIndex >= 0; layerIndex--)
{
var targetLayer = _layers[layerIndex];
var nextLayer = _layers[layerIndex + 1];
if (layerIndex != 0)
{
backpropigationHandle = targetLayer.ModelLayer.BackpropigateLayer(targetLayer, nextLayer, testCaseCount, backpropigationHandle);
}
var accumulateGradientOverWeightJob = new ErrorEvaluators.AccumulateGradientOverWeight()
{
PreviousActivation = targetLayer.SliceActivations(0, testCaseCount),
NextError = nextLayer.SliceError(0, testCaseCount),
WeightGradients = nextLayer.WeightGradients
};
backpropigationHandle =
accumulateGradientOverWeightJob.Schedule(nextLayer.WeightGradients.Length, 4,
backpropigationHandle);
}
JobHandle.ScheduleBatchedJobs();
//Update weights (all layers but first)
JobHandle updateNetworkJobHandle = backpropigationHandle;
for (int layerIndex = 1; layerIndex < _layers.Count; layerIndex++)
{
var layer = _layers[layerIndex];
var applyGradientsToWeightsJob = new ErrorEvaluators.ApplyGradientToLayerWeights()
{
TestCount = 1,
LearningRate = LearningRate,
WeightGradients = layer.WeightGradients,
LayerWeights = layer.ModelLayer.Weights
};
updateNetworkJobHandle =
applyGradientsToWeightsJob.Schedule(layer.ModelLayer.Weights.Length, 4, updateNetworkJobHandle);
//updateNetworkJobHandle.Complete();
var applyGradientsToBiasesJob = new ErrorEvaluators.ApplyGradientToLayerBiases()
{
TestCount = 1,
LearningRate = LearningRate,
LayerBiases = layer.ModelLayer.Biases,
LayerErrors = layer.Error.Slice(0, testCaseCount)
};
updateNetworkJobHandle =
applyGradientsToBiasesJob.Schedule(layer.ModelLayer.Biases.Length, 4, updateNetworkJobHandle);
//updateNetworkJobHandle.Complete();
int x = 1;
}
//return error i guess ?
updateNetworkJobHandle.Complete();
//Compute average error sum for test cases
ComputeErrorSum(expectedOutput, testCaseCount, out errorSum);
resultArray.Dispose();
Profiler.EndSample();
return(updateNetworkJobHandle);
}
public void SetResultTarget(NativeSlice2D <float> newInput)
{
_currentResultTarget = newInput;
}
public void SetInput(NativeSlice2D <float> newInput)
{
_currentInput = newInput;
}
