public void PatternsTestingThread(int iPatternNum) { // thread for backpropagation training of NN // // thread is "owned" by the doc, and accepts a pointer to the doc // continuously backpropagates until m_bThreadAbortFlag is set to TRUE double[] inputVector = new double[841]; // note: 29x29, not 28x28 double[] targetOutputVector = new double[10]; double[] actualOutputVector = new double[10]; // for (int i = 0; i < 841; i++) { inputVector[i] = 0.0; } for (int i = 0; i < 10; i++) { targetOutputVector[i] = 0.0; actualOutputVector[i] = 0.0; } // byte label = 0; int ii, jj; var memorizedNeuronOutputs = new NNNeuronOutputsList(); //prepare for training m_HiPerfTime.Start(); while (_iNextPattern < iPatternNum) { m_Mutexs[1].WaitOne(); byte[] grayLevels = new byte[m_Preferences.m_nRowsImages * m_Preferences.m_nColsImages]; //iSequentialNum = m_MnistDataSet.GetCurrentPatternNumber(m_MnistDataSet.m_bFromRandomizedPatternSequence); _MnistDataSet.m_pImagePatterns[(int)_iNextPattern].pPattern.CopyTo(grayLevels, 0); label = _MnistDataSet.m_pImagePatterns[(int)_iNextPattern].nLabel; if (label < 0) { label = 0; } if (label > 9) { label = 9; } // pad to 29x29, convert to double precision for (ii = 0; ii < 841; ++ii) { inputVector[ii] = 1.0; // one is white, -one is black } // top row of inputVector is left as zero, left-most column is left as zero for (ii = 0; ii < MyDefinations.g_cImageSize; ++ii) { for (jj = 0; jj < MyDefinations.g_cImageSize; ++jj) { inputVector[1 + jj + 29 * (ii + 1)] = (double)((int)(byte)grayLevels[jj + MyDefinations.g_cImageSize * ii]) / 128.0 - 1.0; // one is white, -one is black } } // desired output vector for (ii = 0; ii < 10; ++ii) { targetOutputVector[ii] = -1.0; } targetOutputVector[label] = 1.0; // forward calculate through the neural net CalculateNeuralNet(inputVector, 841, actualOutputVector, 10, memorizedNeuronOutputs, false); int iBestIndex = 0; double maxValue = -99.0; for (ii = 0; ii < 10; ++ii) { if (actualOutputVector[ii] > maxValue) { iBestIndex = ii; maxValue = actualOutputVector[ii]; } } string s = ""; if (iBestIndex != label) { _iMisNum++; s = "Pattern No:" + _iNextPattern.ToString() + " Recognized value:" + iBestIndex.ToString() + " Actual value:" + label.ToString(); if (_form != null) { _form.Invoke(_form._DelegateAddObject, new Object[] { 6, s }); } } else { s = _iNextPattern.ToString() + ", Mis Nums:" + _iMisNum.ToString(); if (_form != null) { _form.Invoke(_form._DelegateAddObject, new Object[] { 7, s }); } } // check if thread is cancelled if (m_EventStop.WaitOne(0, true)) { // clean-up operations may be placed here // ... s = String.Format("Mnist Testing thread: {0} stoped", Thread.CurrentThread.Name); // Make synchronous call to main form. // MainForm.AddString function runs in main thread. // To make asynchronous call use BeginInvoke if (_form != null) { _form.Invoke(_form._DelegateAddObject, new Object[] { 8, s }); } // inform main thread that this thread stopped m_EventStopped.Set(); m_Mutexs[1].ReleaseMutex(); return; } _iNextPattern++; m_Mutexs[1].ReleaseMutex(); } { string s = String.Format("Mnist Testing thread: {0} stoped", Thread.CurrentThread.Name); _form.Invoke(_form._DelegateAddObject, new Object[] { 8, s }); } }
/// <summary> /// /// </summary> /// <param name="inputVector"></param> void CalculateHessian() { // controls the Neural network's calculation if the diagonal Hessian for the Neural net // This will be called from a thread, so although the calculation is lengthy, it should not interfere // with the UI // we need the neural net exclusively during this calculation, so grab it now double[] inputVector = new double[841]; // note: 29x29, not 28x28 double[] targetOutputVector = new double[10]; double[] actualOutputVector = new double[10]; m_Mutexs[1].WaitOne(); for (int i = 0; i < 841; i++) { inputVector[i] = 0.0; } for (int j = 0; j < 10; j++) { targetOutputVector[j] = 0.0; actualOutputVector[j] = 0.0; } byte label = 0; int ii, jj; uint kk; // calculate the diagonal Hessian using 500 random patterns, per Yann LeCun 1998 "Gradient-Based Learning // Applied To Document Recognition" string s = "Commencing Caculation of Hessian..."; // Make synchronous call to main form. // MainForm.AddString function runs in main thread. // To make asynchronous call use BeginInvoke if (_form != null) { _form.Invoke(_form._DelegateAddObject, new Object[] { 3, s }); } // some of this code is similar to the BackpropagationThread() code _NN.EraseHessianInformation(); uint numPatternsSampled = m_Preferences.m_nNumHessianPatterns; for (kk = 0; kk < numPatternsSampled; ++kk) { int iRandomPatternNum; iRandomPatternNum = _MnistDataSet.GetRandomPatternNumber(); label = _MnistDataSet.m_pImagePatterns[iRandomPatternNum].nLabel; if (label < 0) { label = 0; } if (label > 9) { label = 9; } byte[] grayLevels = _MnistDataSet.m_pImagePatterns[iRandomPatternNum].pPattern; // pad to 29x29, convert to double precision for (ii = 0; ii < 841; ++ii) { inputVector[ii] = 1.0; // one is white, -one is black } // top row of inputVector is left as zero, left-most column is left as zero for (ii = 0; ii < MyDefinations.g_cImageSize; ++ii) { for (jj = 0; jj < MyDefinations.g_cImageSize; ++jj) { inputVector[1 + jj + 29 * (ii + 1)] = (double)((int)(byte)grayLevels[jj + MyDefinations.g_cImageSize * ii]) / 128.0 - 1.0; // one is white, -one is black } } // desired output vector for (ii = 0; ii < 10; ++ii) { targetOutputVector[ii] = -1.0; } targetOutputVector[label] = 1.0; // apply distortion map to inputVector. It's not certain that this is needed or helpful. // The second derivatives do NOT rely on the output of the neural net (i.e., because the // second derivative of the MSE function is exactly 1 (one), regardless of the actual output // of the net). However, since the backpropagated second derivatives rely on the outputs of // each neuron, distortion of the pattern might reveal previously-unseen information about the // nature of the Hessian. But I am reluctant to give the full distortion, so I set the // severityFactor to only 2/3 approx GenerateDistortionMap(0.65); ApplyDistortionMap(inputVector); // forward calculate the neural network _NN.Calculate(inputVector, 841, actualOutputVector, 10, null); // backpropagate the second derivatives _NN.BackpropagateSecondDervatives(actualOutputVector, targetOutputVector, 10); // // check if thread is cancelled if (m_EventStop.WaitOne(0, true)) { // clean-up operations may be placed here // ... string ss = "BackPropagation stoped"; // Make synchronous call to main form. // MainForm.AddString function runs in main thread. // To make asynchronous call use BeginInvoke if (_form != null) { _form.Invoke(_form._DelegateAddObject, new Object[] { 3, ss }); } // inform main thread that this thread stopped m_EventStopped.Set(); return; } } _NN.DivideHessianInformationBy((double)numPatternsSampled); s = " Caculation of Hessian...completed"; if (_form != null) { _form.Invoke(_form._DelegateAddObject, new Object[] { 3, s }); } m_Mutexs[1].ReleaseMutex(); }