internal void Initialize(NeuralComputationContext context)
{
Contract.Requires(context != null);
context.ErrorVectorStack_Index = -1;
context.ErrorVectorStack_HasVectors = new bool[MaxSize];
}
internal void PushNull(NeuralComputationContext context)
{
Contract.Requires(context != null);
IncIdx(context);
context.ErrorVectorStack_HasVectors[context.ErrorVectorStack_Index] = false;
}
internal int GetSize(NeuralComputationContext context)
{
Contract.Requires(context != null);
lock (context.SyncRoot)
{
return context.ErrorVectorStack_Index + 1;
}
}
internal void Push(NeuralComputationContext context, IntRange errors)
{
Contract.Requires(context != null);
IncIdx(context);
var v = errorVectors[context.ErrorVectorStack_Index];
network.CopyBuffer(context, errors, v);
context.ErrorVectorStack_HasVectors[context.ErrorVectorStack_Index] = true;
}
internal IntRange? Pop(NeuralComputationContext context)
{
Contract.Requires(context != null);
if (context.ErrorVectorStack_Index < 0) throw new InvalidOperationException("Cannot pop from stack, no values present.");
IntRange? result;
if (context.ErrorVectorStack_HasVectors[context.ErrorVectorStack_Index])
{
result = errorVectors[context.ErrorVectorStack_Index];
}
else
{
result = null;
}
context.ErrorVectorStack_Index--;
return result;
}
private void DoBPTT(NeuralComputationContext context)
{
Debug.Assert(GCAlgo == GradientComputingAlgorithm.BPTT);
Debug.Assert(savedErrorVectors != null);
Debug.Assert(savedErrorVectors.GetSize(context) == context.Training_BPTTEllapsedForwardIterationCount);
bool lastErrorsSet = false;
for (int iterationIndex = 0; iterationIndex < context.Training_BPTTEllapsedForwardIterationCount; iterationIndex++)
{
int storedValueIndex = (context.Training_BPTTEllapsedForwardIterationCount - 1) - iterationIndex;
var errors = savedErrorVectors.Pop(context);
if (errors != null)
{
Network.SetError(context, errors.Value);
lastErrorsSet = true;
}
else if (lastErrorsSet)
{
ZeroErrorInterface(context);
lastErrorsSet = false;
}
Network.Backpropagate(context, (iterationIndex == context.Training_BPTTEllapsedForwardIterationCount - 1) ? BackprogrationMode.BPTTLastStep : BackprogrationMode.BPTTInternalStep, storedValueIndex);
}
}
private void ZeroErrorInterface(NeuralComputationContext context)
{
Network.ZeroBuffer(context, LastErrorBuffer);
Network.SetError(context, LastErrorBuffer);
}
private void BP_ErrorBasedComputed_FF(NeuralComputationContext context)
{
Network.SetError(context, LastErrorBuffer);
Network.Backpropagate(context, BackprogrationMode.FeedForward);
}
private bool BP_ErrorBasedComputed_BPTT(NeuralComputationContext context, int entryIndex, int entryCount)
{
savedErrorVectors.Push(context, LastErrorBuffer);
if (entryIndex == entryCount - 1)
{
DoBPTT(context);
return true;
}
return false;
}
protected override unsafe void DoCopyBuffer(NeuralComputationContext context, IntRange source, IntRange target)
{
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
int size = source.Size;
for (int i = 0; i < size; i++)
{
pValueBuffer[target.MinValue + i] = pValueBuffer[source.MinValue + i];
}
}
}
protected override unsafe void DoComputeError(NeuralComputationContext context, BufferedVector<float> desiredOutputVector, IntRange errorBuffer, IntRange accumulationBuffer)
{
var valueBuffer = GetValueBuff(context);
var mvb = desiredOutputVector.Owner as ManagedVectorBuffer<float>;
var desiredOutputValues = mvb.GetArray(desiredOutputVector);
fixed (float* pValueBuffer = valueBuffer, pDesiredOutputValues = desiredOutputValues)
{
int outputBegin = outputBuffer.MinValue;
int errorBegin = errorBuffer.MinValue;
int accBegin = accumulationBuffer.MinValue;
int accCount = accumulationBuffer.MaxValue;
for (int idx = 0; idx < desiredOutputVector.Length; idx++)
{
float desiredValue = pDesiredOutputValues[idx];
float currentOutputValue = pValueBuffer[outputBegin + idx];
float error = desiredValue - currentOutputValue;
pValueBuffer[errorBegin + idx] = error;
pValueBuffer[accBegin + idx] += (float)Math.Pow(error * 0.5, 2.0);
}
pValueBuffer[accCount]++;
}
}
private static float[] GetValueBuff(NeuralComputationContext context)
{
try
{
return ((ManagedNeuralComputationContext)context).Buff;
}
catch (Exception ex)
{
throw new ArgumentException("Invalid context.", ex);
}
}
private void ResetGradientComputingValues(NeuralComputationContext context)
{
if (GCAlgo == GradientComputingAlgorithm.BPTT)
{
Network.Reset(context, NeuralNetworkResetTarget.Errors | NeuralNetworkResetTarget.Gradients | NeuralNetworkResetTarget.GradientSums);
}
else if (GCAlgo == GradientComputingAlgorithm.RTLR)
{
Network.Reset(context, NeuralNetworkResetTarget.Ps | NeuralNetworkResetTarget.GradientSums);
}
else if (GCAlgo == GradientComputingAlgorithm.BP)
{
Network.Reset(context, NeuralNetworkResetTarget.GradientSums);
}
}
protected override unsafe void DoCallBeforeIterationLearningAlgorithms(NeuralComputationContext context, bool isNewBatch)
{
if (beforeIterationAlgorithms.Length == 0) return;
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
foreach (var algo in beforeIterationAlgorithms)
algo.ForwardIteration(pValueBuffer, isNewBatch);
}
}
protected override unsafe void DoCallErrorBasedStochasticLearningAlgorithms(NeuralComputationContext context, IntRange errorBuffer)
{
if (errorBasedAlgorithms.Length == 0) return;
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
double averageError = ValueBuffer.Average(pValueBuffer, errorBuffer);
foreach (var algo in errorBasedAlgorithms)
algo.BackwardIteration(pValueBuffer, averageError, false);
}
}
protected override unsafe void ResetBackwardValues(NeuralComputationContext context, NeuralNetworkResetTarget target)
{
var valueBuffer = GetValueBuff(context);
for (int groupIndex = 0; groupIndex < backwardComputeGroups.Length; groupIndex++)
{
var compute = backwardComputeGroups[groupIndex];
for (int layerIndex = 0; layerIndex < compute.Length; layerIndex++)
{
compute[layerIndex].Reset(valueBuffer, target);
}
}
}
protected override unsafe void ResetAlgorithms(NeuralComputationContext context)
{
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer) foreach (var algo in algorithms) algo.InitializeNewRun(pValueBuffer);
}
protected override unsafe void ResetAll(NeuralComputationContext context)
{
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
Rtl.ZeroMemory((IntPtr)pValueBuffer, (IntPtr)(sizeof(float) * valueBuffer.Length));
}
}
private void RTLR_ErrorBasedComputed(NeuralComputationContext context)
{
Network.PropagatePValues(context, LastErrorBuffer);
}
protected override unsafe void DoZeroBuffer(NeuralComputationContext context, IntRange accumulationBuffer)
{
var valueBuffer = GetValueBuff(context);
ValueBuffer.Zero(valueBuffer, accumulationBuffer);
}
private void ResetNetworkValues(NeuralComputationContext context)
{
if (Network.IsRecurrent)
{
Network.Reset(context, NeuralNetworkResetTarget.Outputs);
}
}
protected override unsafe void DoReadError(NeuralComputationContext context, float[] values, IntRange errorBuffer)
{
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer, pValues = values)
{
for (int i = 0; i < values.Length; i++) pValues[i] = pValueBuffer[errorBuffer.MinValue + i];
}
}
private void ResetAll(NeuralComputationContext context)
{
Network.Reset(context, NeuralNetworkResetTarget.All);
}
protected override unsafe void DoCalculateAverageError(NeuralComputationContext context, IntRange accumulationBuffer)
{
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
int accBegin = accumulationBuffer.MinValue;
float accCount = pValueBuffer[accumulationBuffer.MaxValue];
if (accCount != 0.0f)
{
int size = accumulationBuffer.Size - 1;
for (int idx = 0; idx < size; idx++)
{
pValueBuffer[accBegin + idx] = pValueBuffer[accBegin + idx] / accCount;
}
}
}
}
private void EnsureInitialized(NeuralComputationContext context)
{
if (!context.Training_IsInitialized)
{
// Call ago init first:
Network.Reset(context, NeuralNetworkResetTarget.Algorithms);
if (savedErrorVectors != null) savedErrorVectors.Initialize(context);
context.Training_IsInitialized = true;
}
}
protected override unsafe void DoSetError(NeuralComputationContext context, IntRange errorBuffer)
{
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
backwardComputeGroups[0][0].SetErrors(pValueBuffer, errorBuffer);
}
}
private void CallErrorBasedStochasticLearningAlgorithms(NeuralComputationContext context)
{
Network.CallErrorBasedStochasticLearningAlgorithms(context, LastErrorBuffer);
context.Training_NumberOfIterationsInBatch++;
}
private bool BP_ErrorBasedComputed(NeuralComputationContext context, int entryIndex, int entryCount)
{
Debug.Assert(GCAlgo == GradientComputingAlgorithm.BP || GCAlgo == GradientComputingAlgorithm.BPTT);
// Errors registered, backpropagate:
if (savedErrorVectors != null) // aka BPTT
{
return BP_ErrorBasedComputed_BPTT(context, entryIndex, entryCount);
}
else // aka BP
{
BP_ErrorBasedComputed_FF(context);
return true;
}
}
private void IncIdx(NeuralComputationContext context)
{
Contract.Requires(context != null);
if (context.ErrorVectorStack_Index == MaxSize - 1) throw new InvalidOperationException("Cannot push to stack because it is full.");
context.ErrorVectorStack_Index++;
}
unsafe protected override void DoReadOutput(NeuralComputationContext context, float[] values)
{
int len = values.Length;
int begin = outputBuffer.MinValue;
var valueBuffer = GetValueBuff(context);
fixed (float* pValues = values, pValueBuffer = valueBuffer)
{
for (int i = 0; i < len; i++)
{
pValues[i] = pValueBuffer[begin + i];
}
}
}
