public void PatternsTestingThread(int iPatternNum)
{
// thread for backpropagation training of NN
//
// thread is "owned" by the doc, and accepts a pointer to the doc
// continuously backpropagates until m_bThreadAbortFlag is set to TRUE
double[] inputVector = new double[841]; // note: 29x29, not 28x28
double[] targetOutputVector = new double[10];
double[] actualOutputVector = new double[10];
//
for (int i = 0; i < 841; i++)
{
inputVector[i] = 0.0;
}
for (int i = 0; i < 10; i++)
{
targetOutputVector[i] = 0.0;
actualOutputVector[i] = 0.0;
}
//
byte label = 0;
int ii, jj;
var memorizedNeuronOutputs = new NNNeuronOutputsList();
//prepare for training
m_HiPerfTime.Start();
while (_iNextPattern < iPatternNum)
{
m_Mutexs[1].WaitOne();
byte[] grayLevels = new byte[m_Preferences.m_nRowsImages * m_Preferences.m_nColsImages];
//iSequentialNum = m_MnistDataSet.GetCurrentPatternNumber(m_MnistDataSet.m_bFromRandomizedPatternSequence);
_MnistDataSet.m_pImagePatterns[(int)_iNextPattern].pPattern.CopyTo(grayLevels, 0);
label = _MnistDataSet.m_pImagePatterns[(int)_iNextPattern].nLabel;
if (label < 0)
{
label = 0;
}
if (label > 9)
{
label = 9;
}
// pad to 29x29, convert to double precision
for (ii = 0; ii < 841; ++ii)
{
inputVector[ii] = 1.0; // one is white, -one is black
}
// top row of inputVector is left as zero, left-most column is left as zero
for (ii = 0; ii < MyDefinations.g_cImageSize; ++ii)
{
for (jj = 0; jj < MyDefinations.g_cImageSize; ++jj)
{
inputVector[1 + jj + 29 * (ii + 1)] = (double)((int)(byte)grayLevels[jj + MyDefinations.g_cImageSize * ii]) / 128.0 - 1.0; // one is white, -one is black
}
}
// desired output vector
for (ii = 0; ii < 10; ++ii)
{
targetOutputVector[ii] = -1.0;
}
targetOutputVector[label] = 1.0;
// forward calculate through the neural net
CalculateNeuralNet(inputVector, 841, actualOutputVector, 10, memorizedNeuronOutputs, false);
int iBestIndex = 0;
double maxValue = -99.0;
for (ii = 0; ii < 10; ++ii)
{
if (actualOutputVector[ii] > maxValue)
{
iBestIndex = ii;
maxValue = actualOutputVector[ii];
}
}
string s = "";
if (iBestIndex != label)
{
_iMisNum++;
s = "Pattern No:" + _iNextPattern.ToString() + " Recognized value:" + iBestIndex.ToString() + " Actual value:" + label.ToString();
if (_form != null)
{
_form.Invoke(_form._DelegateAddObject, new Object[] { 6, s });
}
}
else
{
s = _iNextPattern.ToString() + ", Mis Nums:" + _iMisNum.ToString();
if (_form != null)
{
_form.Invoke(_form._DelegateAddObject, new Object[] { 7, s });
}
}
// check if thread is cancelled
if (m_EventStop.WaitOne(0, true))
{
// clean-up operations may be placed here
// ...
s = String.Format("Mnist Testing thread: {0} stoped", Thread.CurrentThread.Name);
// Make synchronous call to main form.
// MainForm.AddString function runs in main thread.
// To make asynchronous call use BeginInvoke
if (_form != null)
{
_form.Invoke(_form._DelegateAddObject, new Object[] { 8, s });
}
// inform main thread that this thread stopped
m_EventStopped.Set();
m_Mutexs[1].ReleaseMutex();
return;
}
_iNextPattern++;
m_Mutexs[1].ReleaseMutex();
}
{
string s = String.Format("Mnist Testing thread: {0} stoped", Thread.CurrentThread.Name);
_form.Invoke(_form._DelegateAddObject, new Object[] { 8, s });
}
}
/// <summary>
///
/// </summary>
/// <param name="inputVector"></param>
void CalculateHessian()
{
// controls the Neural network's calculation if the diagonal Hessian for the Neural net
// This will be called from a thread, so although the calculation is lengthy, it should not interfere
// with the UI
// we need the neural net exclusively during this calculation, so grab it now
double[] inputVector = new double[841]; // note: 29x29, not 28x28
double[] targetOutputVector = new double[10];
double[] actualOutputVector = new double[10];
m_Mutexs[1].WaitOne();
for (int i = 0; i < 841; i++)
{
inputVector[i] = 0.0;
}
for (int j = 0; j < 10; j++)
{
targetOutputVector[j] = 0.0;
actualOutputVector[j] = 0.0;
}
byte label = 0;
int ii, jj;
uint kk;
// calculate the diagonal Hessian using 500 random patterns, per Yann LeCun 1998 "Gradient-Based Learning
// Applied To Document Recognition"
string s = "Commencing Caculation of Hessian...";
// Make synchronous call to main form.
// MainForm.AddString function runs in main thread.
// To make asynchronous call use BeginInvoke
if (_form != null)
{
_form.Invoke(_form._DelegateAddObject, new Object[] { 3, s });
}
// some of this code is similar to the BackpropagationThread() code
_NN.EraseHessianInformation();
uint numPatternsSampled = m_Preferences.m_nNumHessianPatterns;
for (kk = 0; kk < numPatternsSampled; ++kk)
{
int iRandomPatternNum;
iRandomPatternNum = _MnistDataSet.GetRandomPatternNumber();
label = _MnistDataSet.m_pImagePatterns[iRandomPatternNum].nLabel;
if (label < 0)
{
label = 0;
}
if (label > 9)
{
label = 9;
}
byte[] grayLevels = _MnistDataSet.m_pImagePatterns[iRandomPatternNum].pPattern;
// pad to 29x29, convert to double precision
for (ii = 0; ii < 841; ++ii)
{
inputVector[ii] = 1.0; // one is white, -one is black
}
// top row of inputVector is left as zero, left-most column is left as zero
for (ii = 0; ii < MyDefinations.g_cImageSize; ++ii)
{
for (jj = 0; jj < MyDefinations.g_cImageSize; ++jj)
{
inputVector[1 + jj + 29 * (ii + 1)] = (double)((int)(byte)grayLevels[jj + MyDefinations.g_cImageSize * ii]) / 128.0 - 1.0; // one is white, -one is black
}
}
// desired output vector
for (ii = 0; ii < 10; ++ii)
{
targetOutputVector[ii] = -1.0;
}
targetOutputVector[label] = 1.0;
// apply distortion map to inputVector. It's not certain that this is needed or helpful.
// The second derivatives do NOT rely on the output of the neural net (i.e., because the
// second derivative of the MSE function is exactly 1 (one), regardless of the actual output
// of the net). However, since the backpropagated second derivatives rely on the outputs of
// each neuron, distortion of the pattern might reveal previously-unseen information about the
// nature of the Hessian. But I am reluctant to give the full distortion, so I set the
// severityFactor to only 2/3 approx
GenerateDistortionMap(0.65);
ApplyDistortionMap(inputVector);
// forward calculate the neural network
_NN.Calculate(inputVector, 841, actualOutputVector, 10, null);
// backpropagate the second derivatives
_NN.BackpropagateSecondDervatives(actualOutputVector, targetOutputVector, 10);
//
// check if thread is cancelled
if (m_EventStop.WaitOne(0, true))
{
// clean-up operations may be placed here
// ...
string ss = "BackPropagation stoped";
// Make synchronous call to main form.
// MainForm.AddString function runs in main thread.
// To make asynchronous call use BeginInvoke
if (_form != null)
{
_form.Invoke(_form._DelegateAddObject, new Object[] { 3, ss });
}
// inform main thread that this thread stopped
m_EventStopped.Set();
return;
}
}
_NN.DivideHessianInformationBy((double)numPatternsSampled);
s = " Caculation of Hessian...completed";
if (_form != null)
{
_form.Invoke(_form._DelegateAddObject, new Object[] { 3, s });
}
m_Mutexs[1].ReleaseMutex();
}
