/// <summary>
/// Mencari solusi model neural network
/// </summary>
private void searchSolution()
{
// Normalize Data
switch (this.selectedActivationFunction)
{
case ActivationFunctionEnumeration.SemiLinearFunction:
this.activationFunction = new SemiLinearFunction();
this.normalizeData(0.1, 0.9);
break;
case ActivationFunctionEnumeration.SigmoidFunction:
this.activationFunction = new SigmoidFunction();
this.normalizeData(0.1, 0.9);
break;
case ActivationFunctionEnumeration.BipolarSigmoidFunction:
this.activationFunction = new BipolarSigmoidFunction();
this.normalizeData(-0.9, 0.9);
break;
case ActivationFunctionEnumeration.HyperbolicTangentFunction:
this.activationFunction = new HyperbolicTangentFunction();
this.normalizeData(-0.9, 0.9);
break;
default:
this.activationFunction = new BipolarSigmoidFunction();
this.normalizeData(-0.9, 0.9);
break;
}
//create network
this.network = new BasicNetwork();
this.network.AddLayer(new FeedforwardLayer(this.activationFunction, this.inputLayerNeurons));
this.network.AddLayer(new FeedforwardLayer(this.activationFunction, this.hiddenLayerNeurons));
this.network.AddLayer(new FeedforwardLayer(this.activationFunction, this.outputLayerNeurons));
this.network.Reset();
//variable for looping
//needToStop = false;
double mse = 0.0, error = 0.0, mae=0.0;
int iteration = 1;
// parameters
double msle = 0.0, mspe = 0.0, generalizationLoss = 0.0, pq = 0.0;
double[] trainingErrors = new double[this.strip];
for (int i = 0; i < this.strip; i++) trainingErrors[i] = double.MaxValue / strip;
double lastMSE = double.MaxValue;
// advanced early stopping
int n = this.data.Length - this.network.InputLayer.NeuronCount;
int validationSet = (int)Math.Round(this.validationSetRatio * n);
int trainingSet = n - validationSet;
double[][] networkTrainingInput = new double[trainingSet][];
double[][] networkTrainingOutput = new double[trainingSet][];
for (int i = 0; i < trainingSet; i++)
{
networkTrainingInput[i] = new double[this.network.InputLayer.NeuronCount];
networkTrainingOutput[i] = new double[1];
}
for (int i = 0; i < trainingSet; i++)
{
for (int j = 0; j < this.network.InputLayer.NeuronCount; j++)
{
networkTrainingInput[i][j] = this.networkInput[i][j];
}
networkTrainingOutput[i][0] = this.networkOutput[i][0];
}
// validation set
double[] solutionValidation = new double[validationSet];
double[] inputForValidation = new double[this.network.InputLayer.NeuronCount];
double[] inputForValidationNetwork = new double[this.network.InputLayer.NeuronCount];
// array for saving neural weights and parameters
this.bestValidationError = double.MaxValue;
this.bestWeightMatrix = new double[this.network.Layers.Count -1][,];
this.bestSolution = new double[n];
for (int i = 0; i < this.network.Layers.Count - 1; i++)
{
this.bestWeightMatrix[i] = new double[this.network.Layers[i].WeightMatrix.Rows, this.network.Layers[i].WeightMatrix.Cols];
}
//best network criterion
double bestNetworkError = double.MaxValue, bestNetworkMSE = double.MaxValue, bestNetworkMAE = double.MaxValue;
// build array for graph
this.solutionData = new double[n];
this.predictedPoint = new cPoint[n];
this.validationPoint = new cPoint[validationSet];
//initialize point for graph
predictedDS.Samples = predictedPoint;
validationDS.Samples = validationPoint;
this.predictedDS.Active = true;
// prepare training data
INeuralDataSet dataset;
if (this.useAdvanceEarlyStopping)
dataset = new BasicNeuralDataSet(networkTrainingInput, networkTrainingOutput);
else
dataset = new BasicNeuralDataSet(this.networkInput, this.networkOutput);
// initialize trainer
this.learning = new Backpropagation(this.network, dataset, this.learningRate, this.momentum);
//training
while (!needToStop)
{
double sse = 0.0;
double sae = 0.0;
double ssle = 0.0;
double sspe = 0.0;
this.learning.Iteration();
error = learning.Error;
if (this.useAdvanceEarlyStopping)
{
this.validationDS.Active = true;
}
else
{
this.validationDS.Active = false;
}
for (int i = 0; i < n; i++)
{
INeuralData neuraldata = new BasicNeuralData(this.networkInput[i]);
this.solutionData[i] = (this.network.Compute(neuraldata)[0]
- this.minNormalizedData) / this.factor + this.minData;
this.predictedPoint[i].x = i + this.network.InputLayer.NeuronCount;
this.predictedPoint[i].y = (float)this.solutionData[i];
sse += Math.Pow(this.solutionData[i] - this.data[i + this.network.InputLayer.NeuronCount], 2);
sae += Math.Abs(this.solutionData[i] - this.data[i + this.network.InputLayer.NeuronCount]);
//calculate advance early stopping
if (this.useAdvanceEarlyStopping)
{
if (i < n - validationSet)
{
ssle += Math.Pow(this.solutionData[i] - this.data[i + this.network.InputLayer.NeuronCount], 2);
}
else
{
// initialize the first validation set input
if (i == n - validationSet)
{
for (int j = 0; j < this.network.InputLayer.NeuronCount; j++)
{
inputForValidation[this.network.InputLayer.NeuronCount - 1 - j] = this.data[this.data.Length - (n - i) - 1 - j];
}
}
for (int j = 0; j < this.network.InputLayer.NeuronCount; j++)
{
inputForValidationNetwork[j] = (inputForValidation[j] - this.minData) * this.factor + this.minNormalizedData;
}
INeuralData neuraldataval = new BasicNeuralData(inputForValidationNetwork);
solutionValidation[i - n + validationSet] = (this.network.Compute(neuraldataval)[0] - this.minNormalizedData) / this.factor + this.minData;
this.validationPoint[i - n + validationSet].x = i + this.network.InputLayer.NeuronCount;
this.validationPoint[i - n + validationSet].y = (float)solutionValidation[i - n + validationSet];
sspe += Math.Pow(this.data[i + this.network.InputLayer.NeuronCount] - solutionValidation[i - n + validationSet], 2);
// initialize the next validation set input from the current validation set input
for (int j = 0; j < this.network.InputLayer.NeuronCount - 1; j++)
{
inputForValidation[j] = inputForValidation[j + 1];
}
inputForValidation[this.network.InputLayer.NeuronCount - 1] = solutionValidation[i - n + validationSet];
}
}
}
mse = sse / this.solutionData.Length;
mae = sae / this.solutionData.Length;
//Console.WriteLine(error.ToString());
//Display it
this.iterationBox.Text = iteration.ToString();
this.maeBox.Text = mae.ToString("F5");
this.mseBox.Text = mse.ToString("F5");
this.errorBox.Text = error.ToString("F5");
seriesGraph.Refresh();
if (this.useAdvanceEarlyStopping)
{
//calculate advance early stopping 2
mspe = sspe / validationSet;
msle = ssle / (this.solutionData.Length - validationSet);
//save best weight
if (this.bestValidationError > mspe)
{
this.bestValidationError = mspe;
this.bestSolution = this.solutionData;
// weight matrix
for (int i = 0; i < this.network.Layers.Count - 1; i++)
for (int j = 0; j < this.network.Layers[i].WeightMatrix.Rows; j++)
for (int k = 0; k < this.network.Layers[i].WeightMatrix.Cols; k++)
this.bestWeightMatrix[i][j,k] = this.network.Layers[i].WeightMatrix[j, k];
bestNetworkError = error;
bestNetworkMAE = mae;
bestNetworkMSE = mse;
}
//calculate generalization loss &pq
generalizationLoss = 100 * (mspe / this.bestValidationError - 1);
trainingErrors[(iteration - 1) % this.strip] = msle;
double minStripTrainingError = double.MaxValue, sumStripTrainingError = 0.0;
for (int i = 0; i < this.strip; i++)
{
sumStripTrainingError += trainingErrors[i];
if (trainingErrors[i] < minStripTrainingError) minStripTrainingError = trainingErrors[i];
}
double trainingProgress = 1000 * ((sumStripTrainingError / (this.strip * minStripTrainingError)) - 1);
pq = generalizationLoss / trainingProgress;
//display advance early stopping
this.learningErrorBox.Text = msle.ToString("F5");
this.validationErrorBox.Text = mspe.ToString("F5");
this.generalizationLossBox.Text = generalizationLoss.ToString("F5");
this.pqBox.Text = pq.ToString("F5");
this.seriesGraph.Refresh();
//stopping
switch (this.advanceStoppingMethod)
{
case AdvanceStoppingMethodEnumeration.GeneralizationLoss:
if (generalizationLoss > this.generalizationLossTreshold) needToStop = true;
break;
case AdvanceStoppingMethodEnumeration.ProgressQuotient:
if (pq > this.pqTreshold) needToStop = true;
break;
}
}
if (this.withCheckingCycle && iteration % this.checkingCycle == 0)
{
switch (this.checkingMethod)
{
case CheckingMethodEnumeration.byMSEValue:
if (mse <= this.byMSEValueStopping) needToStop = true;
break;
case CheckingMethodEnumeration.byMSEChange:
if (lastMSE - mse <= this.byMSEChangeStopping) needToStop = true;
break;
}
lastMSE = mse;
}
if (iteration >= this.maxIteration)
{
needToStop = true;
}
iteration++;
}
//restore weight
if (this.useAdvanceEarlyStopping)
{
this.solutionData = this.bestSolution;
// weight matrix
for (int i = 0; i < this.network.Layers.Count - 1; i++)
for (int j = 0; j < this.network.Layers[i].WeightMatrix.Rows; j++)
for (int k = 0; k < this.network.Layers[i].WeightMatrix.Cols; k++)
this.network.Layers[i].WeightMatrix[j, k] = this.bestWeightMatrix[i][j, k];
//best network criterion
this.error = bestNetworkError;
this.mse = bestNetworkMSE;
this.mae = bestNetworkMAE;
}
else
{
this.error = error;
this.mse = mse;
this.mae = mae;
}
this.enableControls(true);
}
/// <summary>
/// Mencari solusi model neural network
/// </summary>
private void searchSolution()
{
// Normalize Data
switch (this.selectedActivationFunction)
{
case ActivationFunctionEnumeration.SemiLinearFunction:
this.activationFunction = new SemiLinearFunction();
this.normalizeData(0.1, 0.9);
break;
case ActivationFunctionEnumeration.SigmoidFunction:
this.activationFunction = new SigmoidFunction();
this.normalizeData(0.1, 0.9);
break;
case ActivationFunctionEnumeration.BipolarSigmoidFunction:
this.activationFunction = new BipolarSigmoidFunction();
this.normalizeData(-0.9, 0.9);
break;
case ActivationFunctionEnumeration.HyperbolicTangentFunction:
this.activationFunction = new HyperbolicTangentFunction();
this.normalizeData(-0.9, 0.9);
break;
default:
this.activationFunction = new BipolarSigmoidFunction();
this.normalizeData(-0.9, 0.9);
break;
}
//create network
this.network = new BasicNetwork();
this.network.AddLayer(new FeedforwardLayer(this.activationFunction, this.inputLayerNeurons));
this.network.AddLayer(new FeedforwardLayer(this.activationFunction, this.hiddenLayerNeurons));
this.network.AddLayer(new FeedforwardLayer(this.activationFunction, this.outputLayerNeurons));
this.network.Reset();
//variable for looping
//needToStop = false;
double mse = 0.0, error = 0.0, mae = 0.0;
int iteration = 1;
// parameters
double msle = 0.0, mspe = 0.0, generalizationLoss = 0.0, pq = 0.0;
double[] trainingErrors = new double[this.strip];
for (int i = 0; i < this.strip; i++)
{
trainingErrors[i] = double.MaxValue / strip;
}
double lastMSE = double.MaxValue;
// advanced early stopping
int n = this.data.Length - this.network.InputLayer.NeuronCount;
int validationSet = (int)Math.Round(this.validationSetRatio * n);
int trainingSet = n - validationSet;
double[][] networkTrainingInput = new double[trainingSet][];
double[][] networkTrainingOutput = new double[trainingSet][];
for (int i = 0; i < trainingSet; i++)
{
networkTrainingInput[i] = new double[this.network.InputLayer.NeuronCount];
networkTrainingOutput[i] = new double[1];
}
for (int i = 0; i < trainingSet; i++)
{
for (int j = 0; j < this.network.InputLayer.NeuronCount; j++)
{
networkTrainingInput[i][j] = this.networkInput[i][j];
}
networkTrainingOutput[i][0] = this.networkOutput[i][0];
}
// validation set
double[] solutionValidation = new double[validationSet];
double[] inputForValidation = new double[this.network.InputLayer.NeuronCount];
double[] inputForValidationNetwork = new double[this.network.InputLayer.NeuronCount];
// array for saving neural weights and parameters
this.bestValidationError = double.MaxValue;
this.bestWeightMatrix = new double[this.network.Layers.Count - 1][, ];
this.bestSolution = new double[n];
for (int i = 0; i < this.network.Layers.Count - 1; i++)
{
this.bestWeightMatrix[i] = new double[this.network.Layers[i].WeightMatrix.Rows, this.network.Layers[i].WeightMatrix.Cols];
}
//best network criterion
double bestNetworkError = double.MaxValue, bestNetworkMSE = double.MaxValue, bestNetworkMAE = double.MaxValue;
// build array for graph
this.solutionData = new double[n];
this.predictedPoint = new cPoint[n];
this.validationPoint = new cPoint[validationSet];
//initialize point for graph
predictedDS.Samples = predictedPoint;
validationDS.Samples = validationPoint;
this.predictedDS.Active = true;
// prepare training data
INeuralDataSet dataset;
if (this.useAdvanceEarlyStopping)
{
dataset = new BasicNeuralDataSet(networkTrainingInput, networkTrainingOutput);
}
else
{
dataset = new BasicNeuralDataSet(this.networkInput, this.networkOutput);
}
// initialize trainer
this.learning = new Backpropagation(this.network, dataset, this.learningRate, this.momentum);
//training
while (!needToStop)
{
double sse = 0.0;
double sae = 0.0;
double ssle = 0.0;
double sspe = 0.0;
this.learning.Iteration();
error = learning.Error;
if (this.useAdvanceEarlyStopping)
{
this.validationDS.Active = true;
}
else
{
this.validationDS.Active = false;
}
for (int i = 0; i < n; i++)
{
INeuralData neuraldata = new BasicNeuralData(this.networkInput[i]);
this.solutionData[i] = (this.network.Compute(neuraldata)[0]
- this.minNormalizedData) / this.factor + this.minData;
this.predictedPoint[i].x = i + this.network.InputLayer.NeuronCount;
this.predictedPoint[i].y = (float)this.solutionData[i];
sse += Math.Pow(this.solutionData[i] - this.data[i + this.network.InputLayer.NeuronCount], 2);
sae += Math.Abs(this.solutionData[i] - this.data[i + this.network.InputLayer.NeuronCount]);
//calculate advance early stopping
if (this.useAdvanceEarlyStopping)
{
if (i < n - validationSet)
{
ssle += Math.Pow(this.solutionData[i] - this.data[i + this.network.InputLayer.NeuronCount], 2);
}
else
{
// initialize the first validation set input
if (i == n - validationSet)
{
for (int j = 0; j < this.network.InputLayer.NeuronCount; j++)
{
inputForValidation[this.network.InputLayer.NeuronCount - 1 - j] = this.data[this.data.Length - (n - i) - 1 - j];
}
}
for (int j = 0; j < this.network.InputLayer.NeuronCount; j++)
{
inputForValidationNetwork[j] = (inputForValidation[j] - this.minData) * this.factor + this.minNormalizedData;
}
INeuralData neuraldataval = new BasicNeuralData(inputForValidationNetwork);
solutionValidation[i - n + validationSet] = (this.network.Compute(neuraldataval)[0] - this.minNormalizedData) / this.factor + this.minData;
this.validationPoint[i - n + validationSet].x = i + this.network.InputLayer.NeuronCount;
this.validationPoint[i - n + validationSet].y = (float)solutionValidation[i - n + validationSet];
sspe += Math.Pow(this.data[i + this.network.InputLayer.NeuronCount] - solutionValidation[i - n + validationSet], 2);
// initialize the next validation set input from the current validation set input
for (int j = 0; j < this.network.InputLayer.NeuronCount - 1; j++)
{
inputForValidation[j] = inputForValidation[j + 1];
}
inputForValidation[this.network.InputLayer.NeuronCount - 1] = solutionValidation[i - n + validationSet];
}
}
}
mse = sse / this.solutionData.Length;
mae = sae / this.solutionData.Length;
//Console.WriteLine(error.ToString());
//Display it
this.iterationBox.Text = iteration.ToString();
this.maeBox.Text = mae.ToString("F5");
this.mseBox.Text = mse.ToString("F5");
this.errorBox.Text = error.ToString("F5");
seriesGraph.Refresh();
if (this.useAdvanceEarlyStopping)
{
//calculate advance early stopping 2
mspe = sspe / validationSet;
msle = ssle / (this.solutionData.Length - validationSet);
//save best weight
if (this.bestValidationError > mspe)
{
this.bestValidationError = mspe;
this.bestSolution = this.solutionData;
// weight matrix
for (int i = 0; i < this.network.Layers.Count - 1; i++)
{
for (int j = 0; j < this.network.Layers[i].WeightMatrix.Rows; j++)
{
for (int k = 0; k < this.network.Layers[i].WeightMatrix.Cols; k++)
{
this.bestWeightMatrix[i][j, k] = this.network.Layers[i].WeightMatrix[j, k];
}
}
}
bestNetworkError = error;
bestNetworkMAE = mae;
bestNetworkMSE = mse;
}
//calculate generalization loss &pq
generalizationLoss = 100 * (mspe / this.bestValidationError - 1);
trainingErrors[(iteration - 1) % this.strip] = msle;
double minStripTrainingError = double.MaxValue, sumStripTrainingError = 0.0;
for (int i = 0; i < this.strip; i++)
{
sumStripTrainingError += trainingErrors[i];
if (trainingErrors[i] < minStripTrainingError)
{
minStripTrainingError = trainingErrors[i];
}
}
double trainingProgress = 1000 * ((sumStripTrainingError / (this.strip * minStripTrainingError)) - 1);
pq = generalizationLoss / trainingProgress;
//display advance early stopping
this.learningErrorBox.Text = msle.ToString("F5");
this.validationErrorBox.Text = mspe.ToString("F5");
this.generalizationLossBox.Text = generalizationLoss.ToString("F5");
this.pqBox.Text = pq.ToString("F5");
this.seriesGraph.Refresh();
//stopping
switch (this.advanceStoppingMethod)
{
case AdvanceStoppingMethodEnumeration.GeneralizationLoss:
if (generalizationLoss > this.generalizationLossTreshold)
{
needToStop = true;
}
break;
case AdvanceStoppingMethodEnumeration.ProgressQuotient:
if (pq > this.pqTreshold)
{
needToStop = true;
}
break;
}
}
if (this.withCheckingCycle && iteration % this.checkingCycle == 0)
{
switch (this.checkingMethod)
{
case CheckingMethodEnumeration.byMSEValue:
if (mse <= this.byMSEValueStopping)
{
needToStop = true;
}
break;
case CheckingMethodEnumeration.byMSEChange:
if (lastMSE - mse <= this.byMSEChangeStopping)
{
needToStop = true;
}
break;
}
lastMSE = mse;
}
if (iteration >= this.maxIteration)
{
needToStop = true;
}
iteration++;
}
//restore weight
if (this.useAdvanceEarlyStopping)
{
this.solutionData = this.bestSolution;
// weight matrix
for (int i = 0; i < this.network.Layers.Count - 1; i++)
{
for (int j = 0; j < this.network.Layers[i].WeightMatrix.Rows; j++)
{
for (int k = 0; k < this.network.Layers[i].WeightMatrix.Cols; k++)
{
this.network.Layers[i].WeightMatrix[j, k] = this.bestWeightMatrix[i][j, k];
}
}
}
//best network criterion
this.error = bestNetworkError;
this.mse = bestNetworkMSE;
this.mae = bestNetworkMAE;
}
else
{
this.error = error;
this.mse = mse;
this.mae = mae;
}
this.enableControls(true);
}
