/// <summary>
/// This method accepts the training examples as input and performs the training of the MLP neural network
/// </summary>
/// <param name="data">training data pairs</param>
/// <param name="ctx">training data descriptions</param>
/// <returns>IScore</returns>
public override IScore Run(double[][] data, IContext ctx)
{
// Sum for every layer. hidLyrNeuronSum1 = x11*w11+x12*w21+..+x1N*wN1
double[][] hidLyrNeuronSum = new double[m_HiddenLayerNeurons.Length + 1][];
// outputs = ActFnx(hidLyrNeuronSum+Bias)
double[][] hidLyrOut = new double[m_HiddenLayerNeurons.Length + 1][];
double[][] trainingData = new double[(int)(data.Length * 0.8)][];
double[][] validationData = new double[(int)(data.Length * 0.2)][];
trainingData = data.Take((int)(data.Length * 0.8)).ToArray();
validationData = data.Skip((int)(data.Length * 0.8)).ToArray();
int numOfInputVectors = trainingData.Length;
m_InpDims = ctx.DataDescriptor.Features.Count();
m_OutputLayerNeurons = data[0].Length - m_InpDims;
m_Weights = new double[m_HiddenLayerNeurons.Length + 1][, ];
m_Biases = new double[m_HiddenLayerNeurons.Length + 1][];
InitializeWeightsandBiasesinputlayer(m_InpDims);
InitializeWeightsandBiaseshiddenlayers(m_HiddenLayerNeurons);
InitializeWeightsandBiasesoutputlayer(m_HiddenLayerNeurons);
var score = new MLPerceptronAlgorithmScore();
double lastLoss = 0;
double lastValidationLoss = 0;
#if TESTING
string path = Directory.GetCurrentDirectory() + "\\MLPerceptron\\TestFiles\\mnist_performance_params_" + this.TestCaseNumber.ToString() + ".csv";
if (!File.Exists(path))
{
File.Create(path).Dispose();
}
using (var performanceData = new StreamWriter(path))
#endif
{
Stopwatch watch = new Stopwatch();
double timeElapsed = 0;
#if TESTING
performanceData.WriteLine("{0},{1},{2},{3},{4},{5}", "Epoch", "Epoch Loss", "Epoch Accuracy", "Validation Loss", "Validation Accuracy", "Time Elapsed");
#endif
for (int i = 0; i < m_Iterations; i++)
{
watch.Restart();
score.Loss = 0;
double batchAccuracy = 0;
int miniBatchStartIndex = 0;
while (miniBatchStartIndex < numOfInputVectors)
{
BackPropagationNetwork backPropagation = new BackPropagationNetwork(m_Biases, m_HiddenLayerNeurons, m_OutputLayerNeurons, m_InpDims);
for (int inputVectIndx = miniBatchStartIndex; inputVectIndx < m_batchSize + miniBatchStartIndex && inputVectIndx < trainingData.Length; inputVectIndx++)
{
// Z2 = actFnc(X * W1)
CalcFirstHiddenLayer(trainingData[inputVectIndx], m_InpDims, out hidLyrOut[0], out hidLyrNeuronSum[0]);
// We use output of first layer as input of second layer.
CalcRemainingHiddenLayers(hidLyrOut[0], hidLyrNeuronSum[0], m_InpDims, out hidLyrOut, out hidLyrNeuronSum);
// Zk = ak-1 * Wk-1
CalculateResultatOutputlayer(hidLyrOut[m_HiddenLayerNeurons.Length - 1], m_InpDims, m_SoftMax, out hidLyrOut[m_HiddenLayerNeurons.Length], out hidLyrNeuronSum[m_HiddenLayerNeurons.Length]);
if (m_SoftMax == true)
{
backPropagation.CalcOutputErrorSoftMax(hidLyrOut[m_HiddenLayerNeurons.Length], m_HiddenLayerNeurons, trainingData[inputVectIndx], ctx);
}
else
{
// BackPropagationNetwork backPropagation = new BackPropagationNetwork(m_HiddenLayerNeurons.Length);
backPropagation.CalcOutputError(hidLyrOut[m_HiddenLayerNeurons.Length], m_HiddenLayerNeurons, hidLyrNeuronSum[m_HiddenLayerNeurons.Length], trainingData[inputVectIndx], ctx);
}
backPropagation.CalcHiddenLayersError(hidLyrOut, m_Weights, m_HiddenLayerNeurons, hidLyrNeuronSum, trainingData[inputVectIndx]);
backPropagation.CostFunctionChangeWithBiases(m_Biases, m_HiddenLayerNeurons, m_LearningRate);
backPropagation.CostFunctionChangeWithWeights(m_Weights, hidLyrOut, m_HiddenLayerNeurons, m_LearningRate, trainingData[inputVectIndx]);
}
backPropagation.UpdateBiases(m_Biases, m_HiddenLayerNeurons, m_LearningRate, out m_Biases);
backPropagation.UpdateWeights(m_Weights, hidLyrOut, m_HiddenLayerNeurons, m_LearningRate, m_InpDims, out m_Weights);
score.Errors = backPropagation.MiniBatchError[m_HiddenLayerNeurons.Length];
batchAccuracy += ((double)backPropagation.TrainingSetAccuracy / m_batchSize);
double sum = 0;
foreach (var outLyrErr in score.Errors)
{
sum += outLyrErr;
}
/*
* 1 - mean of errors
* score.Loss = 1 - (Math.Abs(sum) / score.Errors.Length);
*/
score.Loss += Math.Abs(sum);
miniBatchStartIndex = miniBatchStartIndex + m_batchSize;
}
double deltaLoss = lastLoss - score.Loss;
double accuracy = ((double)batchAccuracy * m_batchSize) / numOfInputVectors;
var result = ((MLPerceptronResult)Predict(validationData, ctx)).results;
int accurateResults = 0;
double validationSetLoss = 0.0;
// Check if the test data has been correctly classified by the neural network
for (int j = 0; j < validationData.Length; j++)
{
accurateResults++;
for (int k = 0; k < m_OutputLayerNeurons; k++)
{
validationSetLoss += Math.Abs(validationData[j][(validationData[j].Length - m_OutputLayerNeurons) + k] - result[j * m_OutputLayerNeurons + k]);
//Assert.True(testData[i][(testData[i].Length - numberOfOutputs) + j] == (result[i * numberOfOutputs + j] >= 0.5 ? 1 : 0));
if (validationData[j][(validationData[j].Length - m_OutputLayerNeurons) + k] != (result[j * m_OutputLayerNeurons + k] >= 0.5 ? 1 : 0))
{
accurateResults--;
break;
}
}
}
double deltaValidationLoss = lastValidationLoss - validationSetLoss;
double validationAccuracy = (double)accurateResults / validationData.Length;
watch.Stop();
timeElapsed += ((double)watch.ElapsedMilliseconds / 1000);
// Debug.WriteLine($"Loss: {score.Loss}, Last loss: {lastLoss}, Delta: {deltaLoss}, Accuracy: {accuracy}, ValidationLoss: {validationSetLoss}, Last Validationloss: {lastValidationLoss}, Delta: {deltaValidationLoss}, ValidationAccuracy: {validationAccuracy}, TimeElapsed: {timeElapsed}");
#if TESTING
performanceData.WriteLine("{0},{1},{2},{3},{4},{5}", i.ToString(), score.Loss.ToString("F3"), accuracy.ToString("F3"), validationSetLoss.ToString("F3"), validationAccuracy.ToString("F3"), timeElapsed.ToString("F3"));
#endif
lastLoss = score.Loss;
lastValidationLoss = validationSetLoss;
}
ctx.Score = score;
return(ctx.Score);
}
}
