unsafe public void ComputeForward(NeuralComputationContext context)
{
var valueBuffer = ManagedBufferOps.GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
for (int outputIndex = 0; outputIndex < forwardCompute.OutputBuffer.Size; outputIndex++)
{
// Calculate sum:
float sum = forwardCompute.BiasValueIndex != null ? pValueBuffer[forwardCompute.BiasValueIndex.Value] : 0.0f;
int outputSize = forwardCompute.OutputBuffer.Size;
foreach (var accessItem in forwardCompute.InputValueAccessItems)
{
for (int inputIndex = 0; inputIndex < accessItem.InputSize; inputIndex++)
{
int inputValueIndex = accessItem.InputBufferBeginIndex + inputIndex;
sum += pValueBuffer[inputValueIndex] * pValueBuffer[accessItem.WeightBufferBeginIndex + WeightAccessor.GetWeightValueIndex(inputIndex, outputIndex, outputSize)];
}
}
// Store Net Value:
pValueBuffer[forwardCompute.NetDerivBuffer.Value.MinValue + outputIndex] = forwardCompute.Function.CalculateDerivate(sum);
// Set output derivate
pValueBuffer[forwardCompute.OutputBuffer.MinValue + outputIndex] = forwardCompute.Function.Calculate(sum);
}
}
}
unsafe public void ZeroAll(NeuralComputationContext context)
{
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
Rtl.ZeroMemory((IntPtr)pValueBuffer, (IntPtr)(sizeof(float) * valueBuffer.Length));
}
}
private static void DoTest(NNInitParameters initPars, bool parallelize)
{
Console.WriteLine("Testing: {0}", initPars.GetType().Name + (parallelize ? " parallel" : " no parallel"));
int count = 10000;
var sw = new Stopwatch();
var arch = CreateArch(initPars);
using (var nn = arch.CreateNetwork())
using (var vectorBuff = nn.CreateVectorBuffer())
{
int cpus = Environment.ProcessorCount * 2;
var input = new float[10];
var output = Enumerable.Repeat(1.1f, 20).ToArray();
if (parallelize)
{
var contexts = new NeuralComputationContext[cpus];
for (int i = 0; i < cpus; i++) contexts[i] = nn.CreateContext();
sw.Start();
for (int i = 0; i < count; i+= cpus)
{
int ccount = cpus;
if (i + ccount > count) ccount = (i + ccount) - count;
System.Threading.Tasks.Parallel.For(0, ccount, x =>
{
var ctx = contexts[x];
nn.WriteInput(ctx, vectorBuff.GetOrCreate(x, 0, () => input));
nn.Reset(ctx);
nn.Iteration(ctx);
nn.Backpropagate(ctx, BackprogrationMode.FeedForward);
nn.ReadOutput(ctx, output);
});
}
sw.Stop();
}
else
{
using (var ctx = nn.CreateContext())
{
sw.Start();
for (int i = 0; i < count; i++)
{
nn.WriteInput(ctx, vectorBuff.GetOrCreate(i % cpus, 0, () => input));
nn.Reset(ctx);
nn.Iteration(ctx);
nn.Backpropagate(ctx, BackprogrationMode.FeedForward);
nn.ReadOutput(ctx, output);
}
sw.Stop();
}
}
}
Console.WriteLine(sw.Elapsed.TotalMilliseconds);
}
unsafe public void ComputeBackward(NeuralComputationContext context)
{
var valueBuffer = ManagedBufferOps.GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
for (int outputIndex = 0; outputIndex < backwardCompute.ForwardCompute.OutputBuffer.Size; outputIndex++)
{
SetGradients(backwardCompute, pValueBuffer, outputIndex, ComputeError(backwardCompute, pValueBuffer, outputIndex));
}
}
}
unsafe public void ReadArray(NeuralComputationContext context, IntRange range, float[] values)
{
int len = values.Length;
int begin = range.MinValue;
var valueBuffer = GetValueBuff(context);
fixed (float* pValues = values, pValueBuffer = valueBuffer)
{
for (int i = 0; i < len; i++)
{
pValues[i] = pValueBuffer[begin + i];
}
}
}
unsafe public void CopyBuffer(NeuralComputationContext context, IntRange source, IntRange target)
{
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
int size = source.Size;
for (int i = 0; i < size; i++)
{
pValueBuffer[target.MinValue + i] = pValueBuffer[source.MinValue + i];
}
}
}
unsafe public void WriteVector(NeuralComputationContext context, IntRange range, BufferedVector<float> values)
{
int len = values.Length;
int begin = range.MinValue;
var valueBuffer = GetValueBuff(context);
fixed (float* pValues = GetValueArray(values), pValueBuffer = valueBuffer)
{
for (int i = 0; i < len; i++)
{
pValueBuffer[begin + i] = pValues[i];
}
}
}
internal override void Compute(NeuralComputationContext context, BackprogrationMode mode, int? innerIterationIndex)
{
Debug.Assert((mode == BackprogrationMode.FeedForward && innerIterationIndex == null) || (mode != BackprogrationMode.FeedForward && innerIterationIndex != null));
Debug.Assert(ErrorBuffer != null && ErrorBuffer.Value.Size == ForwardCompute.OutputBuffer.Size);
Debug.Assert(LowerErrorValueAccessItems != null);
switch (mode)
{
case BackprogrationMode.FeedForward:
((ActivationLayerForwardCompute)ForwardCompute).FeedForwardOps.ComputeBackward(context);
break;
case BackprogrationMode.BPTTInternalStep:
((ActivationLayerForwardCompute)ForwardCompute).BPTTOps.ComputeBackwardInternalStep(context, innerIterationIndex.Value);
break;
case BackprogrationMode.BPTTLastStep:
((ActivationLayerForwardCompute)ForwardCompute).BPTTOps.ComputeBackwardLastStep(context, innerIterationIndex.Value);
break;
}
}
internal override void Compute(NeuralComputationContext context, bool collectTrainingData, int? innerIterationIndex)
{
if (collectTrainingData)
{
switch (Method)
{
case ForwardComputationMethod.FeedForward:
FeedForwardOps.ComputeForward(context);
break;
case ForwardComputationMethod.BPTT:
BPTTOps.ComputeForward(context, innerIterationIndex.Value);
break;
case ForwardComputationMethod.RTLR:
RTLROps.ComputeForward(context);
break;
}
}
else
{
FeedForwardOps.ComputeForward(context);
}
}
unsafe public void ComputeError(NeuralComputationContext context, IntRange outputBuffer, BufferedVector<float> desiredOutputVector, IntRange errorBuffer, IntRange accumulationBuffer)
{
var valueBuffer = GetValueBuff(context);
var mvb = desiredOutputVector.Owner as ManagedVectorBuffer<float>;
var desiredOutputValues = mvb.GetArray(desiredOutputVector);
fixed (float* pValueBuffer = valueBuffer, pDesiredOutputValues = desiredOutputValues)
{
int outputBegin = outputBuffer.MinValue;
int errorBegin = errorBuffer.MinValue;
int accBegin = accumulationBuffer.MinValue;
int accCount = accumulationBuffer.MaxValue;
for (int idx = 0; idx < desiredOutputVector.Length; idx++)
{
float desiredValue = pDesiredOutputValues[idx];
float currentOutputValue = pValueBuffer[outputBegin + idx];
float error = desiredValue - currentOutputValue;
pValueBuffer[errorBegin + idx] = error;
pValueBuffer[accBegin + idx] += (float)Math.Pow(error * 0.5, 2.0);
}
pValueBuffer[accCount]++;
}
}
internal void Reset(NeuralComputationContext context, NeuralNetworkResetTarget target)
{
if ((target & NeuralNetworkResetTarget.Errors) == NeuralNetworkResetTarget.Errors)
{
ForwardCompute.BufferOps.Zero(context, ErrorBuffer.Value);
}
if ((target & NeuralNetworkResetTarget.Gradients) == NeuralNetworkResetTarget.Gradients)
{
foreach (var gradBuff in GradientBuffers)
{
ForwardCompute.BufferOps.Zero(context, gradBuff);
}
}
if ((target & NeuralNetworkResetTarget.GradientSums) == NeuralNetworkResetTarget.GradientSums)
{
foreach (var gradSumBuff in GradientSumBuffers)
{
ForwardCompute.BufferOps.Zero(context, gradSumBuff);
}
}
}
internal void BackwardIteration(NeuralComputationContext context, int avgValueIndex, bool p)
{
throw new NotImplementedException();
}
internal void InitializeNewRun(NeuralComputationContext context)
{
throw new NotImplementedException();
}
internal void Reset(NeuralComputationContext context, NeuralNetworkResetTarget target)
{
if ((target & NeuralNetworkResetTarget.Outputs) != 0)
{
BufferOps.Zero(context, OutputBuffer);
}
else if ((target & NeuralNetworkResetTarget.Ps) != 0)
{
if (Method == ForwardComputationMethod.RTLR)
{
foreach (var range in PBiasBuffers)
{
BufferOps.Zero(context, range);
}
foreach (var lvl1 in PWeightBuffers)
{
foreach (var lvl2 in lvl1)
{
foreach (var range in lvl2)
{
BufferOps.Zero(context, range);
}
}
}
}
}
}
internal abstract void Compute(NeuralComputationContext context, bool collectTrainingData, int? innerIterationIndex);
internal abstract void SetError(NeuralComputationContext context, IntRange errorBuffer);
unsafe public void CalculateAverageError(NeuralComputationContext context, IntRange accumulationBuffer)
{
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
int accBegin = accumulationBuffer.MinValue;
float accCount = pValueBuffer[accumulationBuffer.MaxValue];
if (accCount != 0.0f)
{
int size = accumulationBuffer.Size - 1;
for (int idx = 0; idx < size; idx++)
{
pValueBuffer[accBegin + idx] = pValueBuffer[accBegin + idx] / accCount;
}
}
}
}
internal abstract void Compute(NeuralComputationContext context, BackprogrationMode mode, int? innerIterationIndex);
unsafe public void AverageDist(NeuralComputationContext context, int outputIndex, IntRange range)
{
var valueBuffer = GetValueBuff(context);
fixed (float* pValueBuffer = valueBuffer)
{
double averageError = ValueBuffer.AverageDist(pValueBuffer, range);
pValueBuffer[outputIndex] = (float)averageError;
}
}
unsafe public void Zero(NeuralComputationContext context, IntRange range)
{
var valueBuffer = GetValueBuff(context);
ValueBuffer.Zero(valueBuffer, range);
}
internal static float[] GetValueBuff(NeuralComputationContext context)
{
Contract.Requires(context != null);
try
{
return ((ManagedNeuralComputationContext)context).Buff;
}
catch (Exception ex)
{
throw new ArgumentException("Invalid context.", ex);
}
}
internal void Propagate(NeuralComputationContext context, IntRange? errorBuffer)
{
throw new NotImplementedException();
}
internal void ForwardIteration(NeuralComputationContext context, bool isNewBatch)
{
throw new NotImplementedException();
}
