public void RunEpochTest1()
{
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] { -1 },
new double[] { 1 },
new double[] { 1 },
new double[] { -1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 2, 1);
var teacher = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
double error = 1.0;
while (error > 1e-5)
error = teacher.RunEpoch(input, output);
for (int i = 0; i < input.Length; i++)
Assert.AreEqual(network.Compute(input[i])[0], output[i][0], 0.1);
}
public void MulticlassTest1()
{
Accord.Math.Tools.SetupGenerator(0);
Neuron.RandGenerator = new ThreadSafeRandom(0);
int numberOfInputs = 3;
int numberOfClasses = 4;
int hiddenNeurons = 5;
double[][] input =
{
new double[] { -1, -1, -1 }, // 0
new double[] { -1, 1, -1 }, // 1
new double[] { 1, -1, -1 }, // 1
new double[] { 1, 1, -1 }, // 0
new double[] { -1, -1, 1 }, // 2
new double[] { -1, 1, 1 }, // 3
new double[] { 1, -1, 1 }, // 3
new double[] { 1, 1, 1 } // 2
};
int[] labels =
{
0,
1,
1,
0,
2,
3,
3,
2,
};
double[][] outputs = Accord.Statistics.Tools
.Expand(labels, numberOfClasses, -1, 1);
var function = new BipolarSigmoidFunction(2);
var network = new ActivationNetwork(function,
numberOfInputs, hiddenNeurons, numberOfClasses);
new NguyenWidrow(network).Randomize();
var teacher = new LevenbergMarquardtLearning(network);
double error = Double.PositiveInfinity;
for (int i = 0; i < 10; i++)
error = teacher.RunEpoch(input, outputs);
for (int i = 0; i < input.Length; i++)
{
int answer;
double[] output = network.Compute(input[i]);
double response = output.Max(out answer);
int expected = labels[i];
Assert.AreEqual(expected, answer);
}
}
public void JacobianByChainRuleTest_MultipleOutput()
{
// Network with no hidden layers: 3-4
int numberOfInputs = 3;
int numberOfClasses = 4;
double[][] input =
{
new double[] { -1, -1, -1 }, // 0
new double[] { -1, 1, -1 }, // 1
new double[] { 1, -1, -1 }, // 1
new double[] { 1, 1, -1 }, // 0
new double[] { -1, -1, 1 }, // 2
new double[] { -1, 1, 1 }, // 3
new double[] { 1, -1, 1 }, // 3
new double[] { 1, 1, 1 } // 2
};
int[] labels =
{
0,
1,
1,
0,
2,
3,
3,
2,
};
double[][] output = Accord.Statistics.Tools
.Expand(labels, numberOfClasses, -1, 1);
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), numberOfInputs, numberOfClasses);
var teacher1 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
var teacher2 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByBackpropagation);
// Set lambda to lambda max so no iterations are performed
teacher1.LearningRate = 1e30f;
teacher2.LearningRate = 1e30f;
teacher1.RunEpoch(input, output);
teacher2.RunEpoch(input, output);
PrivateObject privateTeacher1 = new PrivateObject(teacher1);
PrivateObject privateTeacher2 = new PrivateObject(teacher2);
var jacobian1 = (float[][])privateTeacher1.GetField("jacobian");
var jacobian2 = (float[][])privateTeacher2.GetField("jacobian");
for (int i = 0; i < jacobian1.Length; i++)
{
for (int j = 0; j < jacobian1[i].Length; j++)
{
double j1 = jacobian1[i][j];
double j2 = jacobian2[i][j];
Assert.AreEqual(j1, j2, 1e-3);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
}
public void JacobianByChainRuleTest4()
{
// Network with no hidden layers: 3-1
double[][] input =
{
new double[] {-1, -1 },
new double[] {-1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] {-1 },
new double[] { 1 },
new double[] { 1 },
new double[] {-1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 1);
var teacher1 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
var teacher2 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByBackpropagation);
// Set lambda to lambda max so no iterations are performed
teacher1.LearningRate = 1e30f;
teacher2.LearningRate = 1e30f;
teacher1.RunEpoch(input, output);
teacher2.RunEpoch(input, output);
PrivateObject privateTeacher1 = new PrivateObject(teacher1);
PrivateObject privateTeacher2 = new PrivateObject(teacher2);
var jacobian1 = (float[][])privateTeacher1.GetField("jacobian");
var jacobian2 = (float[][])privateTeacher2.GetField("jacobian");
for (int i = 0; i < jacobian1.Length; i++)
{
for (int j = 0; j < jacobian1[i].Length; j++)
{
double j1 = jacobian1[i][j];
double j2 = jacobian2[i][j];
Assert.AreEqual(j1, j2, 1e-5);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
}
public void ConstructorTest()
{
// Four training samples of the xor function
// two inputs (x and y)
double[][] input =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
// one output (z = x ^ y)
double[][] output =
{
new double[] { -1 },
new double[] { 1 },
new double[] { 1 },
new double[] { -1 }
};
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), // use a bipolar sigmoid activation function
2, // two inputs
3, // three hidden neurons
1 // one output neuron
);
// create teacher
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(
network, // the neural network
false, // whether or not to use Bayesian regularization
JacobianMethod.ByBackpropagation // Jacobian calculation method
);
// set learning rate and momentum
teacher.LearningRate = 0.1f;
// start the supervisioned learning
for (int i = 0; i < 1000; i++)
{
double error = teacher.RunEpoch(input, output);
}
// If we reached here, the constructor test has passed.
}
public void RunEpochTest4()
{
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] { 0, 0 },
};
double[][] output =
{
new double[] { 0 },
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 1);
var teacher = new LevenbergMarquardtLearning(network,
true, JacobianMethod.ByBackpropagation);
double error = 1.0;
for (int i = 0; i < 1000; i++)
error = teacher.RunEpoch(input, output);
for (int i = 0; i < input.Length; i++)
Assert.AreEqual(network.Compute(input[i])[0], output[i][0], 0.1);
}
public void RunEpochTest3()
{
double[,] dataset = yinyang;
double[][] input = dataset.GetColumns(0, 1).ToArray();
double[][] output = dataset.GetColumn(2).ToArray();
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 5, 1);
var teacher = new LevenbergMarquardtLearning(network,
true, JacobianMethod.ByBackpropagation);
Assert.IsTrue(teacher.UseRegularization);
double error = 1.0;
for (int i = 0; i < 500; i++)
error = teacher.RunEpoch(input, output);
double[][] actual = new double[output.Length][];
for (int i = 0; i < input.Length; i++)
actual[i] = network.Compute(input[i]);
for (int i = 0; i < input.Length; i++)
Assert.AreEqual(Math.Sign(output[i][0]), Math.Sign(actual[i][0]));
}
public void JacobianByChainRuleTest3()
{
// Network with 3 hidden layers: 2-4-3-4-1
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] {-1, -1 },
new double[] {-1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] {-1 },
new double[] { 1 },
new double[] { 1 },
new double[] {-1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 4, 3, 4, 1);
var teacher1 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
var teacher2 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByBackpropagation);
// Set lambda to lambda max so no iterations are performed
teacher1.LearningRate = 1e30f;
teacher2.LearningRate = 1e30f;
teacher1.RunEpoch(input, output);
teacher2.RunEpoch(input, output);
var jacobian1 = teacher1.Jacobian;
var jacobian2 = teacher2.Jacobian;
for (int i = 0; i < jacobian1.Length; i++)
{
for (int j = 0; j < jacobian1[i].Length; j++)
{
double j1 = jacobian1[i][j];
double j2 = jacobian2[i][j];
Assert.AreEqual(j1, j2, 1e-4);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
}
// Worker thread
void SearchSolution()
{
// initialize input and output values
double[][] input = null;
double[][] output = null;
if (sigmoidType == 0)
{
// unipolar data
input = new double[4][] {
new double[] {0, 0},
new double[] {0, 1},
new double[] {1, 0},
new double[] {1, 1}
};
output = new double[4][] {
new double[] {0},
new double[] {1},
new double[] {1},
new double[] {0}
};
}
else
{
// bipolar data
input = new double[4][] {
new double[] {-1, -1},
new double[] {-1, 1},
new double[] { 1, -1},
new double[] { 1, 1}
};
output = new double[4][] {
new double[] {-1},
new double[] { 1},
new double[] { 1},
new double[] {-1}
};
}
// create neural network
ActivationNetwork network = new ActivationNetwork(
(sigmoidType == 0) ?
(IActivationFunction)new SigmoidFunction(sigmoidAlphaValue) :
(IActivationFunction)new BipolarSigmoidFunction(sigmoidAlphaValue),
2, 2, 1);
// create teacher
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network);
// set learning rate
teacher.LearningRate = learningRate;
// iterations
int iteration = 0;
// statistic files
StreamWriter errorsFile = null;
try
{
// check if we need to save statistics to files
if (saveStatisticsToFiles)
{
// open files
errorsFile = File.CreateText("errors.csv");
}
// erros list
ArrayList errorsList = new ArrayList();
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output);
errorsList.Add(error);
// save current error
if (errorsFile != null)
{
errorsFile.WriteLine(error);
}
// show current iteration & error
SetText(currentIterationBox, iteration.ToString());
SetText(currentErrorBox, error.ToString());
iteration++;
// check if we need to stop
if (error <= learningErrorLimit)
break;
}
// show error's dynamics
double[,] errors = new double[errorsList.Count, 2];
for (int i = 0, n = errorsList.Count; i < n; i++)
{
errors[i, 0] = i;
errors[i, 1] = (double)errorsList[i];
}
errorChart.RangeX = new Range(0, errorsList.Count - 1);
errorChart.UpdateDataSeries("error", errors);
}
catch (IOException)
{
MessageBox.Show("Failed writing file", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
finally
{
// close files
if (errorsFile != null)
errorsFile.Close();
}
// enable settings controls
EnableControls(true);
}
private static void network(double[][] inputs, int[] outputs)
{
// Since we would like to learn binary outputs in the form
// [-1,+1], we can use a bipolar sigmoid activation function
IActivationFunction function = new BipolarSigmoidFunction();
// In our problem, we have 2 inputs (x, y pairs), and we will
// be creating a network with 5 hidden neurons and 1 output:
//
var network = new ActivationNetwork(function,
inputsCount: 2, neuronsCount: new[] { 5, 1 });
// Create a Levenberg-Marquardt algorithm
var teacher = new LevenbergMarquardtLearning(network)
{
UseRegularization = true
};
// Because the network is expecting multiple outputs,
// we have to convert our single variable into arrays
//
var y = outputs.ToDouble().ToArray();
// Iterate until stop criteria is met
double error = double.PositiveInfinity;
double previous;
do
{
previous = error;
// Compute one learning iteration
error = teacher.RunEpoch(inputs, y);
} while (Math.Abs(previous - error) < 1e-10 * previous);
// Classify the samples using the model
int[] answers = inputs.Apply(network.Compute).GetColumn(0).Apply(System.Math.Sign);
// Plot the results
ScatterplotBox.Show("Expected results", inputs, outputs);
ScatterplotBox.Show("Network results", inputs, answers)
.Hold();
}
// Worker thread
void SearchSolution()
{
// number of learning samples
int samples = data.GetLength(0);
// data transformation factor
double yFactor = 1.7 / chart.RangeY.Length;
double yMin = chart.RangeY.Min;
double xFactor = 2.0 / chart.RangeX.Length;
double xMin = chart.RangeX.Min;
// prepare learning data
double[][] input = new double[samples][];
double[][] output = new double[samples][];
for (int i = 0; i < samples; i++)
{
input[i] = new double[1];
output[i] = new double[1];
// set input
input[i][0] = (data[i, 0] - xMin) * xFactor - 1.0;
// set output
output[i][0] = (data[i, 1] - yMin) * yFactor - 0.85;
}
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(sigmoidAlphaValue),
1, neuronsInFirstLayer, 1);
if (useNguyenWidrow)
{
NguyenWidrow initializer = new NguyenWidrow(network);
initializer.Randomize();
}
// create teacher
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, useRegularization);
// set learning rate and momentum
teacher.LearningRate = learningRate;
// iterations
int iteration = 1;
// solution array
double[,] solution = new double[50, 2];
double[] networkInput = new double[1];
// calculate X values to be used with solution function
for (int j = 0; j < 50; j++)
{
solution[j, 0] = chart.RangeX.Min + (double)j * chart.RangeX.Length / 49;
}
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output) / samples;
// calculate solution
for (int j = 0; j < 50; j++)
{
networkInput[0] = (solution[j, 0] - xMin) * xFactor - 1.0;
solution[j, 1] = (network.Compute(networkInput)[0] + 0.85) / yFactor + yMin;
}
chart.UpdateDataSeries("solution", solution);
// calculate error
double learningError = 0.0;
for (int j = 0, k = data.GetLength(0); j < k; j++)
{
networkInput[0] = input[j][0];
learningError += Math.Abs(data[j, 1] - ((network.Compute(networkInput)[0] + 0.85) / yFactor + yMin));
}
// set current iteration's info
SetText(currentIterationBox, iteration.ToString());
SetText(currentErrorBox, learningError.ToString("F3"));
// increase current iteration
iteration++;
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
// enable settings controls
EnableControls(true);
}
public void JacobianByChainRuleTest()
{
// Network with one hidden layer: 2-2-1
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] { -1 },
new double[] { 1 },
new double[] { 1 },
new double[] { -1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 2, 1);
var teacher1 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
var teacher2 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByBackpropagation);
// Set lambda to lambda max so no iterations are performed
teacher1.LearningRate = 1e30f;
teacher2.LearningRate = 1e30f;
teacher1.RunEpoch(input, output);
teacher2.RunEpoch(input, output);
PrivateObject privateTeacher1 = new PrivateObject(teacher1);
PrivateObject privateTeacher2 = new PrivateObject(teacher2);
var jacobian1 = (float[][])privateTeacher1.GetField("jacobian");
var jacobian2 = (float[][])privateTeacher2.GetField("jacobian");
Assert.AreEqual(jacobian1[0][0], -0.47895513745387097, 1e-6);
Assert.AreEqual(jacobian1[0][1], -0.05863886707282373, 1e-6);
Assert.AreEqual(jacobian1[0][2], 0.057751100929897485, 1e-6);
Assert.AreEqual(jacobian1[0][3], 0.0015185010717608583, 1e-6);
Assert.AreEqual(jacobian1[7][0], -0.185400783651892, 1e-6);
Assert.AreEqual(jacobian1[7][1], 0.025575161626462877, 1e-6);
Assert.AreEqual(jacobian1[7][2], 0.070494677797224889, 1e-6);
Assert.AreEqual(jacobian1[7][3], 0.037740463822781616, 1e-6);
Assert.AreEqual(jacobian2[0][0], -0.4789595904719437, 1e-6);
Assert.AreEqual(jacobian2[0][1], -0.058636153936941729, 1e-6);
Assert.AreEqual(jacobian2[0][2], 0.057748435491340212, 1e-6);
Assert.AreEqual(jacobian2[0][3], 0.0015184453425611988, 1e-6);
Assert.AreEqual(jacobian2[7][0], -0.1854008206574258, 1e-6);
Assert.AreEqual(jacobian2[7][1], 0.025575150379247645, 1e-6);
Assert.AreEqual(jacobian2[7][2], 0.070494269423259301, 1e-6);
Assert.AreEqual(jacobian2[7][3], 0.037740117733922635, 1e-6);
for (int i = 0; i < jacobian1.Length; i++)
{
for (int j = 0; j < jacobian1[i].Length; j++)
{
double j1 = jacobian1[i][j];
double j2 = jacobian2[i][j];
Assert.AreEqual(j1, j2, 1e-4);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
}
// Worker thread
void SearchSolution()
{
// number of learning samples
int samples = data.Length - predictionSize - windowSize;
// data transformation factor
double factor = 1.7 / chart.RangeY.Length;
double yMin = chart.RangeY.Min;
// prepare learning data
double[][] input = new double[samples][];
double[][] output = new double[samples][];
for (int i = 0; i < samples; i++)
{
input[i] = new double[windowSize];
output[i] = new double[1];
// set input
for (int j = 0; j < windowSize; j++)
{
input[i][j] = (data[i + j] - yMin) * factor - 0.85;
}
// set output
output[i][0] = (data[i + windowSize] - yMin) * factor - 0.85;
}
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(sigmoidAlphaValue),
windowSize, windowSize * 2, 1);
// create teacher
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, useRegularization);
// set learning rate
teacher.LearningRate = learningRate;
// iterations
int iteration = 1;
// solution array
int solutionSize = data.Length - windowSize;
double[,] solution = new double[solutionSize, 2];
double[] networkInput = new double[windowSize];
// calculate X values to be used with solution function
for (int j = 0; j < solutionSize; j++)
{
solution[j, 0] = j + windowSize;
}
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output) / samples;
// calculate solution and learning and prediction errors
double learningError = 0.0;
double predictionError = 0.0;
// go through all the data
for (int i = 0, n = data.Length - windowSize; i < n; i++)
{
// put values from current window as network's input
for (int j = 0; j < windowSize; j++)
{
networkInput[j] = (data[i + j] - yMin) * factor - 0.85;
}
// evalue the function
solution[i, 1] = (network.Compute(networkInput)[0] + 0.85) / factor + yMin;
// calculate prediction error
if (i >= n - predictionSize)
{
predictionError += Math.Abs(solution[i, 1] - data[windowSize + i]);
}
else
{
learningError += Math.Abs(solution[i, 1] - data[windowSize + i]);
}
}
// update solution on the chart
chart.UpdateDataSeries("solution", solution);
// set current iteration's info
SetText(currentIterationBox, iteration.ToString());
SetText(currentLearningErrorBox, learningError.ToString("F3"));
SetText(currentPredictionErrorBox, predictionError.ToString("F3"));
// increase current iteration
iteration++;
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
// show new solution
for (int j = windowSize, k = 0, n = data.Length; j < n; j++, k++)
{
AddSubItem(dataList, j, solution[k, 1].ToString());
}
// enable settings controls
EnableControls(true);
}
// Worker thread
void SearchSolution()
{
// number of learning samples
int samples = data.GetLength(0);
// prepare learning data
DoubleRange unit = new DoubleRange(-1, 1);
double[][] input = Tools.Scale(from: xRange, to: unit, x: data.GetColumn(0)).ToArray();
double[][] output = Tools.Scale(from: yRange, to: unit, x: data.GetColumn(1)).ToArray();
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(sigmoidAlphaValue),
1, neuronsInFirstLayer, 1);
if (useNguyenWidrow)
{
new NguyenWidrow(network).Randomize();
}
// create teacher
var teacher = new LevenbergMarquardtLearning(network, useRegularization);
// set learning rate and momentum
teacher.LearningRate = learningRate;
// iterations
int iteration = 1;
// solution array
double[,] solution = new double[samples, 2];
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output) / samples;
// calculate solution
for (int j = 0; j < samples; j++)
{
double x = input[j][0];
double y = network.Compute(new[] { x })[0];
solution[j, 0] = Tools.Scale(from: unit, to: xRange, x: x);
solution[j, 1] = Tools.Scale(from: unit, to: yRange, x: y);
}
chart.UpdateDataSeries("solution", solution);
// calculate error
double learningError = 0.0;
for (int j = 0; j < samples; j++)
{
double x = input[j][0];
double expected = data[j, 1];
double actual = network.Compute(new[] { x })[0];
learningError += Math.Abs(expected - actual);
}
// set current iteration's info
SetText(currentIterationBox, iteration.ToString());
SetText(currentErrorBox, learningError.ToString("F3"));
// increase current iteration
iteration++;
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
// enable settings controls
EnableControls(true);
}
public void BlockHessianTest1()
{
// Network with no hidden layers: 3-1
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] {-1, -1 },
new double[] {-1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] {-1 },
new double[] { 1 },
new double[] { 1 },
new double[] {-1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 1);
var teacher1 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
var teacher2 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByBackpropagation);
teacher2.Blocks = 2;
// Set lambda to lambda max so no iterations are performed
teacher1.LearningRate = 1e30f;
teacher2.LearningRate = 1e30f;
teacher1.RunEpoch(input, output);
teacher2.RunEpoch(input, output);
var hessian1 = teacher1.Hessian;
var hessian2 = teacher1.Hessian;
for (int i = 0; i < hessian1.Length; i++)
{
for (int j = 0; j < hessian1[i].Length; j++)
{
double j1 = hessian1[i][j];
double j2 = hessian2[i][j];
Assert.AreEqual(j1, j2, 1e-4);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
Assert.IsTrue(hessian1.IsUpperTriangular());
Assert.IsTrue(hessian2.IsUpperTriangular());
var gradient1 = teacher1.Gradient;
var gradient2 = teacher2.Gradient;
for (int i = 0; i < gradient1.Length; i++)
{
double j1 = gradient1[i];
double j2 = gradient2[i];
Assert.AreEqual(j1, j2, 1e-5);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
// Worker thread
void SearchSolution()
{
// number of learning samples
int samples = sourceMatrix.GetLength(0);
// prepare learning data
double[][] inputs = sourceMatrix.Submatrix(null, 0, 1).ToArray();
double[][] outputs = sourceMatrix.GetColumn(2).Transpose().ToArray();
// create multi-layer neural network
ann = new ActivationNetwork(new GaussianFunction(sigmoidAlphaValue),
//new BipolarSigmoidFunction(sigmoidAlphaValue),
2, neuronsInFirstLayer, 1);
if (useNguyenWidrow)
{
if (useSameWeights)
Accord.Math.Tools.SetupGenerator(0);
NguyenWidrow initializer = new NguyenWidrow(ann);
initializer.Randomize();
}
// create teacher
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(ann, useRegularization);
// set learning rate and momentum
teacher.LearningRate = learningRate;
// iterations
iteration = 1;
var ranges = Matrix.Range(sourceMatrix, 0);
double[][] map = Matrix.Mesh(ranges[0], ranges[1], 1, 1);
var sw = Stopwatch.StartNew();
// loop
while (!needToStop)
{
// run epoch of learning procedure
error = teacher.RunEpoch(inputs, outputs) / samples;
//var result = map.Apply(ann.Compute).GetColumn(0).Apply(Math.Sign);
var result = map.Apply(ann.Compute).GetColumn(0).Apply(d =>
{
if (d > 0.66) return 2;
else if (d < 0.33)
return 0;
return 1;
});
var graph = map.ToMatrix().InsertColumn(result.ToDouble());
CreateScatterplot(zedGraphControl2, graph);
// increase current iteration
iteration++;
elapsed = sw.Elapsed;
updateStatus();
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
sw.Stop();
// enable settings controls
EnableControls(true);
}
public void ZeroLambdaTest()
{
double[,] data = null;
// open selected file
using (TextReader stream = new StringReader(Properties.Resources.ZeroLambda))
using (CsvReader reader = new CsvReader(stream, false))
{
data = reader.ToTable().ToMatrix();
}
// number of learning samples
int samples = data.GetLength(0);
var ranges = data.Range(dimension: 0);
Assert.AreEqual(2, ranges.Length);
var rangeX = ranges[0];
var rangeY = ranges[1];
// data transformation factor
double yFactor = 1.7 / rangeY.Length;
double yMin = rangeY.Min;
double xFactor = 2.0 / rangeX.Length;
double xMin = rangeX.Min;
// prepare learning data
double[][] input = new double[samples][];
double[][] output = new double[samples][];
for (int i = 0; i < samples; i++)
{
input[i] = new double[1];
output[i] = new double[1];
input[i][0] = (data[i, 0] - xMin) * xFactor - 1.0; // set input
output[i][0] = (data[i, 1] - yMin) * yFactor - 0.85; // set output
}
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(5),
1, 12, 1);
// create teacher
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, true);
teacher.LearningRate = 1;
// iterations
int iteration = 1;
int iterations = 2000;
// solution array
double[,] solution = new double[samples, 2];
double[] networkInput = new double[1];
bool needToStop = false;
double learningError = 0;
// loop
while (!needToStop)
{
Assert.AreNotEqual(0, teacher.LearningRate);
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output) / samples;
// calculate solution
for (int j = 0; j < samples; j++)
{
networkInput[0] = (solution[j, 0] - xMin) * xFactor - 1.0;
solution[j, 1] = (network.Compute(networkInput)[0] + 0.85) / yFactor + yMin;
}
// calculate error
learningError = 0.0;
for (int j = 0; j < samples; j++)
{
networkInput[0] = input[j][0];
learningError += Math.Abs(data[j, 1] - ((network.Compute(networkInput)[0] + 0.85) / yFactor + yMin));
}
// increase current iteration
iteration++;
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
Assert.IsTrue(learningError < 0.13);
}
public BackPropogation()
{
InitializeComponent();
activation_nework = new ActivationNetwork(new BipolarSigmoidFunction(), 50, 50, 10, 1);
watch1 = new Stopwatch();
watch2 = new Stopwatch();
watch3 = new Stopwatch();
backgroundWorkerTrainer.Disposed += backgroundWorkerTrainer_Disposed;
backgroundWorkerTrainer.DoWork += backgroundWorkerTrainer_DoWork;
backgroundWorkerTrainer.ProgressChanged += backgroundWorkerTrainer_ProgressChanged;
backgroundWorkerTrainer.WorkerSupportsCancellation = true;
backgroundWorkerTrainer.WorkerReportsProgress = true;
saveFileDialog1.Filter = "feed forward network files (*.ffn)|*.ffn";
saveFileDialog1.Title = "Save neural networkfile";
saveFileDialog1.InitialDirectory = null;
saveFileDialog1.FileName = null;
openFileDialog1.Filter = "feed forward network files (*.ffn)|*.ffn";
openFileDialog1.Title = "Load neural network file";
openFileDialog1.InitialDirectory = null;
openFileDialog1.FileName = null;
backgroundWorkerSignal.Disposed += backgroundWorkerSignal_Disposed;
backgroundWorkerSignal.WorkerSupportsCancellation = true;
backgroundWorkerSignal.WorkerReportsProgress = true;
backgroundWorkerSignal.DoWork += backgroundWorkerSignal_DoWork;
backgroundWorkerSignal.RunWorkerCompleted += backgroundWorkerSignal_RunWorkerCompleted;//80, 70, 60, 50, 40,
network1 = activation_nework;
network2 = activation_nework; // new ActivationNetwork(new BipolarSigmoidFunction(), 50, 50, 40, 1);
network3 = activation_nework; // new ActivationNetwork(new BipolarSigmoidFunction(), 50, 50, 40, 1);
network4 = activation_nework; // new ActivationNetwork(new BipolarSigmoidFunction(), 50, 50, 40, 1);
network5 = new ActivationNetwork(new BipolarSigmoidFunction(), 50,1);
network6 = new ActivationNetwork(new BipolarSigmoidFunction(), 50, 1);
teacher = new BackPropagationLearning(network1);
evteacher = new EvolutionaryLearning(network2, 100);
reprop = new ResilientBackpropagationLearning(network3);
lbteacher = new LevenbergMarquardtLearning(network4);
delta = new DeltaRuleLearning(network5);
perceptron = new PerceptronLearning(network6);
delta.LearningRate = 1;
perceptron.LearningRate = 0.1;
myPane = new GraphPane();
listPointsOne = new PointPairList();
myPane = zedGraphControl1.GraphPane;
// set a title
myPane.Title.Text = "Error VS Time";
// set X and Y axis titles
myPane.XAxis.Title.Text = "Time in Milliseconds";
myPane.YAxis.Title.Text = "Error";
myCurveOne = myPane.AddCurve("Learning curve", listPointsOne, Color.Red, SymbolType.None);
// myCurveOne = myPane.AddCurve("Resilient Back Propagation", listPointstwo, Color.Green, SymbolType.None);
// myCurveOne = myPane.AddCurve("Genetic Learning", listPointsthree, Color.Blue, SymbolType.None);
}