private double TrainEpoch(int epoch, VMatrix features, VMatrix labels)
{
double sse = 0;
Console.Write("TrainEpoch ");
int cl = Console.CursorLeft;
unsafe
{
for (var row = 0; row < features.Rows(); row++)
{
if (((row % 100) == 0) || (row == (features.Rows() - 1)))
{
Console.SetCursorPosition(cl, Console.CursorTop);
Console.Write(row);
}
// calculate the output
for (var layer = 0; layer < m_lCount.Length; layer++)
{
for (var n = 0; n < m_lCount[layer]; n++)
{
fixed(Node *node = &m_nodes[m_lBegIdx[layer] + n])
{
node->net = 0;
if (layer == 0)
{
// input layer
node->output = features.Get(row, n);
}
else
{
// calculate the net value
for (var w = 0; w < node->wEndIdx - node->wBegIdx; w++)
{
node->net += m_weights[node->wBegIdx + w] * m_nodes[m_lBegIdx[layer - 1] + w].output;
}
// add the bias
node->net += m_weights[node->wEndIdx];
node->output = 1.0 / (1.0 + Math.Exp(-node->net));
}
}
}
}
// calculate the error and weight changes
for (var layer = m_lCount.Length - 1; layer > 0; layer--)
{
for (var n = 0; n < m_lCount[layer]; n++)
{
fixed(Node *node = &m_nodes[m_lBegIdx[layer] + n])
{
double fPrime = node->output * (1.0 - node->output);
if (layer == m_lCount.Length - 1)
{
// output layer
double target = labels.Get(row, node->labelCol);
if (node->isContinuous == 0)
{
// nominal
if (target == node->labelVal)
{
target = 0.9;
}
else
{
target = 0.1;
}
}
var error = target - node->output;
node->error = error * fPrime;
sse += error * error;
}
else
{
// hidden layer
double sum = 0;
for (var tn = 0; tn < m_lCount[layer + 1]; tn++)
{
fixed(Node *tNode = &m_nodes[m_lBegIdx[layer + 1] + tn])
{
sum += tNode->error * m_weights[tNode->wBegIdx + n];
}
}
node->error = sum * fPrime;
}
// calculate the weight changes
double delta;
for (var w = 0; w < node->wEndIdx - node->wBegIdx; w++)
{
delta = m_rate * node->error * m_nodes[m_lBegIdx[layer - 1] + w].output;
delta += m_momentum * m_deltas[node->wBegIdx + w];
m_deltas[node->wBegIdx + w] = delta;
}
// calculate the bias weight change
delta = m_rate * node->error;
delta += m_momentum * m_deltas[node->wEndIdx];
m_deltas[node->wEndIdx] = delta;
}
}
}
// update the weights
for (var w = 0; w < m_weights.Length; w++)
{
m_weights[w] += m_deltas[w];
}
}
}
Console.WriteLine();
return(sse / features.Rows());
}
// Calculate the MSE
public override double VGetMSE(VMatrix features, VMatrix labels)
{
double sse = 0;
Console.Write("VGetMSE ");
int cl = Console.CursorLeft;
unsafe
{
for (var row = 0; row < features.Rows(); row++)
{
if (((row % 10) == 0) || (row == (features.Rows() - 1)))
{
Console.SetCursorPosition(cl, Console.CursorTop);
Console.Write(row);
}
// calculate the output
for (var layer = 0; layer < m_lCount.Length; layer++)
{
for (var n = 0; n < m_lCount[layer]; n++)
{
fixed(Node *node = &m_nodes[m_lBegIdx[layer] + n])
{
node->net = 0;
// calculate the net value
if (layer == 0)
{
// input layer
node->output = features.Get(row, n);
}
else
{
// calculate the net value
for (var w = 0; w < node->wEndIdx - node->wBegIdx; w++)
{
node->net += m_weights[node->wBegIdx + w] * m_nodes[m_lBegIdx[layer - 1] + w].output;
}
// add the bias
node->net += m_weights[node->wEndIdx];
node->output = 1.0 / (1.0 + Math.Exp(-node->net));
}
}
}
}
// calculate the error of the output layer
for (var n = 0; n < m_lCount[m_lCount.Length - 1]; n++)
{
fixed(Node *node = &m_nodes[m_lBegIdx[m_lCount.Length - 1] + n])
{
double target = labels.Get(row, node->labelCol);
if (node->isContinuous == 0)
{
// nominal
if (target == node->labelVal)
{
target = 0.9;
}
else
{
target = 0.1;
}
}
var error = target - node->output;
// update the error
sse += error * error;
}
}
}
}
Console.WriteLine();
return(sse / features.Rows());
}
public override void VTrain(VMatrix features, VMatrix labels, double[] colMin, double[] colMax)
{
if ((m_lCount == null) || (m_lCount.Length < 3))
{
m_lCount = new int[3] {
0, features.Cols() * 2, 0
};
}
List <Node> nodes = new List <Node>();
// add the input nodes
m_lCount[0] = features.Cols();
for (var n = 0; n < m_lCount[0]; n++)
{
nodes.Add(new Node(-1, -1, 0, 0, 0));
}
int numWeights = m_lCount[0] + 1;
int wBegIdx = 0;
// add the nodes for the hidden layers
for (var layer = 1; layer < m_lCount.Length - 1; layer++)
{
for (var n = 0; n < m_lCount[layer]; n++)
{
nodes.Add(new Node(wBegIdx, wBegIdx + numWeights - 1, 0, 0, 0));
wBegIdx += numWeights;
}
numWeights = m_lCount[layer] + 1;
}
// figure out how many outputs we need
int oCount = 0;
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
oCount++;
}
else
{
oCount += labelValueCount;
}
}
// update the layer arrays
m_lCount[m_lCount.Length - 1] = oCount;
m_lBegIdx = new int[m_lCount.Length];
for (var i = 0; i < m_lCount.Length; i++)
{
if (i == 0)
{
m_lBegIdx[i] = 0;
}
else
{
m_lBegIdx[i] = m_lBegIdx[i - 1] + m_lCount[i - 1];
}
}
// add the output nodes
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
nodes.Add(new Node(wBegIdx, wBegIdx + numWeights - 1, 1, col, -1));
wBegIdx += numWeights;
}
else
{
for (var n = 0; n < labelValueCount; n++)
{
nodes.Add(new Node(wBegIdx, wBegIdx + numWeights - 1, 0, col, n));
wBegIdx += numWeights;
}
}
}
m_nodes = nodes.ToArray();
// create the weights
m_weights = new double[wBegIdx];
m_bestWeights = new double[wBegIdx];
m_deltas = new double[wBegIdx];
for (var i = 0; i < wBegIdx; i++)
{
m_weights[i] = (double)(0.1 - (m_rand.NextDouble() * 0.2));
m_bestWeights[i] = m_weights[i];
m_deltas[i] = 0;
}
//m_weights[0] = 1.0;
//m_weights[1] = 0.5;
//m_weights[2] = 0;
//m_weights[3] = 1.2;
//m_weights[4] = 0.5;
//m_weights[5] = 0.5;
//m_weights[6] = 0.1;
//m_weights[7] = -0.8;
//m_weights[8] = -1.3;
if (!string.IsNullOrEmpty(OutputFileName))
{
m_outputFile = File.AppendText(OutputFileName);
}
int trainSize = (int)(0.75 * features.Rows());
double[,] trainFeatures = new double[trainSize, features.Cols()];
for (int r = 0; r < trainSize; r++)
{
for (int c = 0; c < features.Cols(); c++)
{
trainFeatures[r, c] = features.Get(r, c);
}
}
double[,] trainLabels = new double[trainSize, labels.Cols()];
for (int r = 0; r < trainSize; r++)
{
for (int c = 0; c < labels.Cols(); c++)
{
trainLabels[r, c] = labels.Get(r, c);
}
}
int[] fIdx = new int[trainSize];
for (int i = 0; i < fIdx.Length; i++)
{
fIdx[i] = i;
}
VMatrix validationFeatures = new VMatrix(features, trainSize, 0, features.Rows() - trainSize, features.Cols());
VMatrix validationLabels = new VMatrix(labels, trainSize, 0, labels.Rows() - trainSize, labels.Cols());
int epoch = 0; // current epoch number
int bestEpoch = 0; // epoch number of best MSE
int eCount = 0; // number of epochs since the best MSE
bool checkDone = false; // if true, check to see if we're done
double bestMSE = double.MaxValue; // best validation MSE so far
double bestAccuracy = double.MaxValue; // best validationa accuracy so far
Console.WriteLine("Epoch\tMSE (validation)\taccuracy (validation)");
if (m_outputFile != null)
{
m_outputFile.Write("Layers: ");
for (var l = 0; l < m_lCount.Length - 1; l++)
{
m_outputFile.Write(m_lCount[l]);
m_outputFile.Write('x');
}
m_outputFile.WriteLine(m_lCount[m_lCount.Length - 1]);
m_outputFile.WriteLine("Momentum: " + m_momentum);
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
m_outputFile.WriteLine("Epoch\tMSE (validation)\taccuracy (validation)");
}
CudafyModule km = CudafyTranslator.Cudafy();
GPGPU gpu = CudafyHost.GetDevice(CudafyModes.Target, CudafyModes.DeviceId);
gpu.LoadModule(km);
for (; ;)
{
// shuffle the training set
Shuffle(ref fIdx, m_rand);
double[,] g_trainFeatures = gpu.CopyToDevice(trainFeatures);
double[,] g_trainLabels = gpu.CopyToDevice(trainLabels);
int[] g_fIdx = gpu.CopyToDevice(fIdx);
int[] g_lCount = gpu.CopyToDevice(m_lCount);
int[] g_lBegIdx = gpu.CopyToDevice(m_lBegIdx);
Node[] g_nodes = gpu.CopyToDevice(m_nodes);
double[] g_weights = gpu.CopyToDevice(m_weights);
double[] g_deltas = gpu.CopyToDevice(m_deltas);
//// Launch trainSize blocks of 1 thread each
gpu.Launch(trainSize / 256, 256).TrainEpoch(g_trainFeatures, g_trainLabels, g_fIdx, g_lCount, g_lBegIdx, g_nodes, g_weights, g_deltas, m_rate, m_momentum);
//// copy the arrays back from the GPU to the CPU
gpu.CopyFromDevice(g_weights, m_weights);
gpu.CopyFromDevice(g_deltas, m_deltas);
gpu.CopyFromDevice(g_fIdx, fIdx);
// free the memory allocated on the GPU
gpu.FreeAll();
//TrainEpoch(trainFeatures, trainLabels, fIdx, m_lCount, m_lBegIdx, m_nodes, ref m_weights, ref m_deltas, m_rate, m_momentum, ref trainMSE);
// check the MSE after this epoch
double mse = VGetMSE(validationFeatures, validationLabels);
// check the validation accuracy after this epoch
double accuracy = VMeasureAccuracy(validationFeatures, validationLabels, null);
Console.WriteLine(string.Format("{0}-{1}\t{2}\t{3}", epoch, eCount, mse, accuracy));
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("{0}-{1}\t{2}\t{3}", epoch, eCount, mse, accuracy));
m_outputFile.Flush();
}
if ((mse == 0.0) || (epoch > 10000))
{
break;
}
else if ((epoch == 1) || (mse < bestMSE))
{
if (epoch == 1)
{
// save the initial MSE
bestMSE = mse;
}
else if ((mse / bestMSE) > 0.99)
{
if (!checkDone)
{
checkDone = true;
eCount = 0;
}
}
else
{
checkDone = false;
eCount = 0;
}
// save the best for later
bestMSE = mse;
bestAccuracy = accuracy;
bestEpoch = epoch;
SaveBestWeights();
}
else if (!checkDone)
{
checkDone = true;
eCount = 0;
}
if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 20)
{
break;
}
}
}
;
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
}
if ((bestEpoch > 0) && (bestEpoch != epoch))
{
RestoreBestWeights();
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine(string.Format("Best Weights (from Epoch {0}, valMSE={1}, valAcc={2})", bestEpoch, bestMSE, bestAccuracy));
PrintWeights();
}
}
if (m_outputFile != null)
{
m_outputFile.Close();
}
}