/// <summary> /// Back propagates the neuronal network. /// </summary> /// <param name="inputVector">The input vector.</param> /// <param name="inputCount">The input count.</param> /// <param name="targetOutputVector">The target output vector.</param> /// <param name="actualOutputVector">The actual output vector.</param> /// <param name="outputCount">The output count.</param> /// <param name="memorizedNeuronOutputs">The memorized neuronal outputs.</param> /// <param name="distort">The distort.</param> private void BackPropagateNeuronalNetwork( double[] inputVector, int inputCount, double[] targetOutputVector, double[] actualOutputVector, int outputCount, NeuronalNetworkNeuronOutputsList memorizedNeuronOutputs, bool distort) { if (this.NeuronalNetwork is null) { throw new ArgumentNullException(nameof(this.NeuronalNetwork), "The neuronal network wasn't initialized properly."); } // Function to back propagate through the neuronal net. // Determine if it's time to adjust the learning rate this.mutexes[2].WaitOne(); if (this.backProperties % this.AfterEveryNBackProperties == 0 && this.backProperties != 0) { var eta = this.NeuronalNetwork.EtaLearningRate; eta *= this.EtaDecay; if (eta < this.MinimumEta) { eta = this.MinimumEta; } this.NeuronalNetwork.PreviousEtaLearningRate = this.NeuronalNetwork.EtaLearningRate; this.NeuronalNetwork.EtaLearningRate = eta; } // Determine if it's time to adjust the Hessian (currently once per epoch) if (this.needHessian || this.backProperties % this.Preferences.NumberOfItemsTrainingImages == 0) { // Adjust the Hessian. This is a lengthy operation, since it must process approximately 500 labels this.needHessian = false; this.CalculateHessian(); } // Increment counter for tracking number of back properties this.backProperties++; // Determine if it's time to randomize the sequence of training patterns (currently once per epoch) if (this.backProperties % this.Preferences.NumberOfItemsTrainingImages == 0) { this.database.RandomizePatternSequence(); } this.mutexes[2].ReleaseMutex(); // Forward calculate through the neuronal network this.CalculateNeuronalNetwork(inputVector, inputCount, actualOutputVector, outputCount, memorizedNeuronOutputs, distort); this.mutexes[2].WaitOne(); // Calculate error in the output of the neuronal network // note that this code duplicates that found in many other places, and it's probably sensible to // define a (global/static ??) function for it var localDmse = 0.0; for (var ii = 0; ii < 10; ++ii) { localDmse += (actualOutputVector[ii] - targetOutputVector[ii]) * (actualOutputVector[ii] - targetOutputVector[ii]); } localDmse /= 2.0; var worthWhileBackPropagate = !(localDmse <= 0.10 * this.EstimatedCurrentMse); if (worthWhileBackPropagate && memorizedNeuronOutputs == null) { // The caller has not provided a place to store neuron outputs, so we need to // back propagate now, while the neuronal net is still captured. Otherwise, another thread // might come along and call CalculateNeuronalNetwork(), which would entirely change the neuron // outputs and thereby inject errors into back propagation this.NeuronalNetwork.BackPropagate(actualOutputVector, targetOutputVector, outputCount, null); return; } // If we have reached here, then the mutex for the neuronal net has been released for other // threads. The caller must have provided a place to store neuron outputs, which we can // use to back propagate, even if other threads call CalculateNeuronalNetwork() and change the outputs // of the neurons if (worthWhileBackPropagate) { this.NeuronalNetwork.BackPropagate(actualOutputVector, targetOutputVector, outputCount, memorizedNeuronOutputs); } this.mutexes[2].ReleaseMutex(); }
/// <summary> /// Handles the back propagation. /// </summary> public void BackPropagationThread() { // Thread for back propagation training of the neuronal network // Thread is "owned" by the doc, and accepts a pointer to the doc // continuously back propagates until threadAbortFlag is set to TRUE var inputVector = new double[841]; // Note: 29x29, not 28x28 var targetOutputVector = new double[10]; var actualOutputVector = new double[10]; for (var i = 0; i < 841; i++) { inputVector[i] = 0.0; } for (var i = 0; i < 10; i++) { targetOutputVector[i] = 0.0; actualOutputVector[i] = 0.0; } var memorizedNeuronOutputs = new NeuronalNetworkNeuronOutputsList(); // Prepare for training while (true) { this.mutexes[3].WaitOne(); if (this.nextPattern == 0) { this.highPerformanceTimer.Start(); this.database.RandomizePatternSequence(); } var grayLevels = new byte[this.Preferences.NumberOfRowImages * this.Preferences.NumberOfColumnImages]; var pattern = this.database.GetNextPatternNumber(this.database.FromRandomizedPatternSequence); this.database.ImagePatterns[pattern].Pattern.CopyTo(grayLevels, 0); var label = this.database.ImagePatterns[pattern].Label; this.nextPattern++; if (label > 9) { label = 9; } // Pad to 29x29, convert to double precision int ii; for (ii = 0; ii < 841; ++ii) { // One is white, -one is black inputVector[ii] = 1.0; } // Top row of inputVector is left as zero, left-most column is left as zero for (ii = 0; ii < SystemGlobals.ImageSize; ++ii) { int jj; for (jj = 0; jj < SystemGlobals.ImageSize; ++jj) { // One is white, -one is black inputVector[1 + jj + (29 * (ii + 1))] = (grayLevels[jj + (SystemGlobals.ImageSize * ii)] / 128.0) - 1.0; } } // Desired output vector for (ii = 0; ii < 10; ++ii) { targetOutputVector[ii] = -1.0; } targetOutputVector[label] = 1.0; // Now back propagate this.mutexes[3].ReleaseMutex(); this.BackPropagateNeuronalNetwork( inputVector, 841, targetOutputVector, actualOutputVector, 10, memorizedNeuronOutputs, this.DistortTrainingPatterns); this.mutexes[3].WaitOne(); // Calculate error for this pattern and post it to the HWND so it can calculate a running estimate of MSE var localDmse = 0.0; for (ii = 0; ii < 10; ++ii) { localDmse += (actualOutputVector[ii] - targetOutputVector[ii]) * (actualOutputVector[ii] - targetOutputVector[ii]); } localDmse /= 2.0; this.dmse += localDmse; this.dmse200 += localDmse; // Determine the neuronal network's answer, and compare it to the actual answer. // Post a message if the answer was incorrect, so the dialog can display mis-recognition statistics this.neuronalNetworks++; var bestIndex = 0; var maxValue = -99.0; for (ii = 0; ii < 10; ++ii) { if (!(actualOutputVector[ii] > maxValue)) { continue; } bestIndex = ii; maxValue = actualOutputVector[ii]; } if (bestIndex != label) { this.recognitions++; } // Make step string s; if (this.neuronalNetworks >= 200) { this.dmse200 /= 200; s = "MSE:" + this.dmse200.ToString(CultureInfo.InvariantCulture); this.mainForm?.Invoke(this.mainForm.DelegateAddObject, 4, s); this.dmse200 = 0; this.neuronalNetworks = 0; } s = $"{Convert.ToString(this.nextPattern)} Miss Number:{this.recognitions}"; // Make synchronous call to main form. // MainForm.AddString function runs in main thread. // To make asynchronous call use BeginInvoke this.mainForm?.Invoke(this.mainForm.DelegateAddObject, 5, s); if (this.nextPattern >= this.database.ImagePatterns.Count - 1) { this.highPerformanceTimer.Stop(); this.dmse /= this.nextPattern; s = $"Completed Epochs:{Convert.ToString(this.epochsCompleted + 1)}, MisPatterns:{Convert.ToString(this.recognitions)}, MSE:{this.dmse.ToString(CultureInfo.InvariantCulture)}, Ex. time: {this.highPerformanceTimer.Duration}, eta:{this.NeuronalNetwork?.EtaLearningRate ?? .001} "; // Make synchronous call to main form. // MainForm.AddString function runs in main thread. // To make asynchronous call use BeginInvoke this.mainForm?.Invoke(this.mainForm.DelegateAddObject, 3, s); this.recognitions = 0; this.epochsCompleted++; this.nextPattern = 0; this.dmse = 0; } // Check if thread is cancelled if (this.eventStop.WaitOne(0, true)) { // clean-up operations may be placed here // Make synchronous call to main form. // MainForm.AddString function runs in main thread. // To make asynchronous call use BeginInvoke this.mainForm?.Invoke(this.mainForm.DelegateAddObject, 3, $"BackPropagation thread: {Thread.CurrentThread.Name} stopped"); // Inform main thread that this thread stopped this.eventStopped.Set(); this.mutexes[3].ReleaseMutex(); return; } this.mutexes[3].ReleaseMutex(); } }