unsafe public void ComputeForward(NeuralComputationContext context) { var valueBuffer = ManagedBufferOps.GetValueBuff(context); fixed (float* pValueBuffer = valueBuffer) { for (int outputIndex = 0; outputIndex < forwardCompute.OutputBuffer.Size; outputIndex++) { // Calculate sum: float sum = forwardCompute.BiasValueIndex != null ? pValueBuffer[forwardCompute.BiasValueIndex.Value] : 0.0f; int outputSize = forwardCompute.OutputBuffer.Size; foreach (var accessItem in forwardCompute.InputValueAccessItems) { for (int inputIndex = 0; inputIndex < accessItem.InputSize; inputIndex++) { int inputValueIndex = accessItem.InputBufferBeginIndex + inputIndex; sum += pValueBuffer[inputValueIndex] * pValueBuffer[accessItem.WeightBufferBeginIndex + WeightAccessor.GetWeightValueIndex(inputIndex, outputIndex, outputSize)]; } } // Store Net Value: pValueBuffer[forwardCompute.NetDerivBuffer.Value.MinValue + outputIndex] = forwardCompute.Function.CalculateDerivate(sum); // Set output derivate pValueBuffer[forwardCompute.OutputBuffer.MinValue + outputIndex] = forwardCompute.Function.Calculate(sum); } } }
unsafe public void ZeroAll(NeuralComputationContext context) { var valueBuffer = GetValueBuff(context); fixed (float* pValueBuffer = valueBuffer) { Rtl.ZeroMemory((IntPtr)pValueBuffer, (IntPtr)(sizeof(float) * valueBuffer.Length)); } }
private static void DoTest(NNInitParameters initPars, bool parallelize) { Console.WriteLine("Testing: {0}", initPars.GetType().Name + (parallelize ? " parallel" : " no parallel")); int count = 10000; var sw = new Stopwatch(); var arch = CreateArch(initPars); using (var nn = arch.CreateNetwork()) using (var vectorBuff = nn.CreateVectorBuffer()) { int cpus = Environment.ProcessorCount * 2; var input = new float[10]; var output = Enumerable.Repeat(1.1f, 20).ToArray(); if (parallelize) { var contexts = new NeuralComputationContext[cpus]; for (int i = 0; i < cpus; i++) contexts[i] = nn.CreateContext(); sw.Start(); for (int i = 0; i < count; i+= cpus) { int ccount = cpus; if (i + ccount > count) ccount = (i + ccount) - count; System.Threading.Tasks.Parallel.For(0, ccount, x => { var ctx = contexts[x]; nn.WriteInput(ctx, vectorBuff.GetOrCreate(x, 0, () => input)); nn.Reset(ctx); nn.Iteration(ctx); nn.Backpropagate(ctx, BackprogrationMode.FeedForward); nn.ReadOutput(ctx, output); }); } sw.Stop(); } else { using (var ctx = nn.CreateContext()) { sw.Start(); for (int i = 0; i < count; i++) { nn.WriteInput(ctx, vectorBuff.GetOrCreate(i % cpus, 0, () => input)); nn.Reset(ctx); nn.Iteration(ctx); nn.Backpropagate(ctx, BackprogrationMode.FeedForward); nn.ReadOutput(ctx, output); } sw.Stop(); } } } Console.WriteLine(sw.Elapsed.TotalMilliseconds); }
unsafe public void ComputeBackward(NeuralComputationContext context) { var valueBuffer = ManagedBufferOps.GetValueBuff(context); fixed (float* pValueBuffer = valueBuffer) { for (int outputIndex = 0; outputIndex < backwardCompute.ForwardCompute.OutputBuffer.Size; outputIndex++) { SetGradients(backwardCompute, pValueBuffer, outputIndex, ComputeError(backwardCompute, pValueBuffer, outputIndex)); } } }
unsafe public void ReadArray(NeuralComputationContext context, IntRange range, float[] values) { int len = values.Length; int begin = range.MinValue; var valueBuffer = GetValueBuff(context); fixed (float* pValues = values, pValueBuffer = valueBuffer) { for (int i = 0; i < len; i++) { pValues[i] = pValueBuffer[begin + i]; } } }
unsafe public void CopyBuffer(NeuralComputationContext context, IntRange source, IntRange target) { var valueBuffer = GetValueBuff(context); fixed (float* pValueBuffer = valueBuffer) { int size = source.Size; for (int i = 0; i < size; i++) { pValueBuffer[target.MinValue + i] = pValueBuffer[source.MinValue + i]; } } }
unsafe public void WriteVector(NeuralComputationContext context, IntRange range, BufferedVector<float> values) { int len = values.Length; int begin = range.MinValue; var valueBuffer = GetValueBuff(context); fixed (float* pValues = GetValueArray(values), pValueBuffer = valueBuffer) { for (int i = 0; i < len; i++) { pValueBuffer[begin + i] = pValues[i]; } } }
internal override void Compute(NeuralComputationContext context, BackprogrationMode mode, int? innerIterationIndex) { Debug.Assert((mode == BackprogrationMode.FeedForward && innerIterationIndex == null) || (mode != BackprogrationMode.FeedForward && innerIterationIndex != null)); Debug.Assert(ErrorBuffer != null && ErrorBuffer.Value.Size == ForwardCompute.OutputBuffer.Size); Debug.Assert(LowerErrorValueAccessItems != null); switch (mode) { case BackprogrationMode.FeedForward: ((ActivationLayerForwardCompute)ForwardCompute).FeedForwardOps.ComputeBackward(context); break; case BackprogrationMode.BPTTInternalStep: ((ActivationLayerForwardCompute)ForwardCompute).BPTTOps.ComputeBackwardInternalStep(context, innerIterationIndex.Value); break; case BackprogrationMode.BPTTLastStep: ((ActivationLayerForwardCompute)ForwardCompute).BPTTOps.ComputeBackwardLastStep(context, innerIterationIndex.Value); break; } }
internal override void Compute(NeuralComputationContext context, bool collectTrainingData, int? innerIterationIndex) { if (collectTrainingData) { switch (Method) { case ForwardComputationMethod.FeedForward: FeedForwardOps.ComputeForward(context); break; case ForwardComputationMethod.BPTT: BPTTOps.ComputeForward(context, innerIterationIndex.Value); break; case ForwardComputationMethod.RTLR: RTLROps.ComputeForward(context); break; } } else { FeedForwardOps.ComputeForward(context); } }
unsafe public void ComputeError(NeuralComputationContext context, IntRange outputBuffer, BufferedVector<float> desiredOutputVector, IntRange errorBuffer, IntRange accumulationBuffer) { var valueBuffer = GetValueBuff(context); var mvb = desiredOutputVector.Owner as ManagedVectorBuffer<float>; var desiredOutputValues = mvb.GetArray(desiredOutputVector); fixed (float* pValueBuffer = valueBuffer, pDesiredOutputValues = desiredOutputValues) { int outputBegin = outputBuffer.MinValue; int errorBegin = errorBuffer.MinValue; int accBegin = accumulationBuffer.MinValue; int accCount = accumulationBuffer.MaxValue; for (int idx = 0; idx < desiredOutputVector.Length; idx++) { float desiredValue = pDesiredOutputValues[idx]; float currentOutputValue = pValueBuffer[outputBegin + idx]; float error = desiredValue - currentOutputValue; pValueBuffer[errorBegin + idx] = error; pValueBuffer[accBegin + idx] += (float)Math.Pow(error * 0.5, 2.0); } pValueBuffer[accCount]++; } }
internal void Reset(NeuralComputationContext context, NeuralNetworkResetTarget target) { if ((target & NeuralNetworkResetTarget.Errors) == NeuralNetworkResetTarget.Errors) { ForwardCompute.BufferOps.Zero(context, ErrorBuffer.Value); } if ((target & NeuralNetworkResetTarget.Gradients) == NeuralNetworkResetTarget.Gradients) { foreach (var gradBuff in GradientBuffers) { ForwardCompute.BufferOps.Zero(context, gradBuff); } } if ((target & NeuralNetworkResetTarget.GradientSums) == NeuralNetworkResetTarget.GradientSums) { foreach (var gradSumBuff in GradientSumBuffers) { ForwardCompute.BufferOps.Zero(context, gradSumBuff); } } }
internal void BackwardIteration(NeuralComputationContext context, int avgValueIndex, bool p) { throw new NotImplementedException(); }
internal void InitializeNewRun(NeuralComputationContext context) { throw new NotImplementedException(); }
internal void Reset(NeuralComputationContext context, NeuralNetworkResetTarget target) { if ((target & NeuralNetworkResetTarget.Outputs) != 0) { BufferOps.Zero(context, OutputBuffer); } else if ((target & NeuralNetworkResetTarget.Ps) != 0) { if (Method == ForwardComputationMethod.RTLR) { foreach (var range in PBiasBuffers) { BufferOps.Zero(context, range); } foreach (var lvl1 in PWeightBuffers) { foreach (var lvl2 in lvl1) { foreach (var range in lvl2) { BufferOps.Zero(context, range); } } } } } }
internal abstract void Compute(NeuralComputationContext context, bool collectTrainingData, int? innerIterationIndex);
internal abstract void SetError(NeuralComputationContext context, IntRange errorBuffer);
unsafe public void CalculateAverageError(NeuralComputationContext context, IntRange accumulationBuffer) { var valueBuffer = GetValueBuff(context); fixed (float* pValueBuffer = valueBuffer) { int accBegin = accumulationBuffer.MinValue; float accCount = pValueBuffer[accumulationBuffer.MaxValue]; if (accCount != 0.0f) { int size = accumulationBuffer.Size - 1; for (int idx = 0; idx < size; idx++) { pValueBuffer[accBegin + idx] = pValueBuffer[accBegin + idx] / accCount; } } } }
internal abstract void Compute(NeuralComputationContext context, BackprogrationMode mode, int? innerIterationIndex);
unsafe public void AverageDist(NeuralComputationContext context, int outputIndex, IntRange range) { var valueBuffer = GetValueBuff(context); fixed (float* pValueBuffer = valueBuffer) { double averageError = ValueBuffer.AverageDist(pValueBuffer, range); pValueBuffer[outputIndex] = (float)averageError; } }
unsafe public void Zero(NeuralComputationContext context, IntRange range) { var valueBuffer = GetValueBuff(context); ValueBuffer.Zero(valueBuffer, range); }
internal static float[] GetValueBuff(NeuralComputationContext context) { Contract.Requires(context != null); try { return ((ManagedNeuralComputationContext)context).Buff; } catch (Exception ex) { throw new ArgumentException("Invalid context.", ex); } }
internal void Propagate(NeuralComputationContext context, IntRange? errorBuffer) { throw new NotImplementedException(); }
internal void ForwardIteration(NeuralComputationContext context, bool isNewBatch) { throw new NotImplementedException(); }