public void RunEpochTest1()
        {
            Accord.Math.Tools.SetupGenerator(0);

            double[][] input = 
            {
                new double[] { -1, -1 },
                new double[] { -1,  1 },
                new double[] {  1, -1 },
                new double[] {  1,  1 }
            };

            double[][] output =
            {
                new double[] { -1 },
                new double[] {  1 },
                new double[] {  1 },
                new double[] { -1 }
            };

            Neuron.RandGenerator = new ThreadSafeRandom(0);

            ActivationNetwork network = new ActivationNetwork(
                   new BipolarSigmoidFunction(2), 2, 2, 1);

            var teacher = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByFiniteDifferences);

            double error = 1.0;
            while (error > 1e-5)
                error = teacher.RunEpoch(input, output);

            for (int i = 0; i < input.Length; i++)
                Assert.AreEqual(network.Compute(input[i])[0], output[i][0], 0.1);
        }
        public void MulticlassTest1()
        {
            Accord.Math.Tools.SetupGenerator(0);
            Neuron.RandGenerator = new ThreadSafeRandom(0);


            int numberOfInputs = 3;
            int numberOfClasses = 4;
            int hiddenNeurons = 5;

            double[][] input = 
            {
                new double[] { -1, -1, -1 }, // 0
                new double[] { -1,  1, -1 }, // 1
                new double[] {  1, -1, -1 }, // 1
                new double[] {  1,  1, -1 }, // 0
                new double[] { -1, -1,  1 }, // 2
                new double[] { -1,  1,  1 }, // 3
                new double[] {  1, -1,  1 }, // 3
                new double[] {  1,  1,  1 }  // 2
            };

            int[] labels =
            {
                0,
                1,
                1,
                0,
                2,
                3,
                3,
                2,
            };

            double[][] outputs = Accord.Statistics.Tools
                .Expand(labels, numberOfClasses, -1, 1);

            var function = new BipolarSigmoidFunction(2);
            var network = new ActivationNetwork(function,
                numberOfInputs, hiddenNeurons, numberOfClasses);

            new NguyenWidrow(network).Randomize();

            var teacher = new LevenbergMarquardtLearning(network);

            double error = Double.PositiveInfinity;
            for (int i = 0; i < 10; i++)
                error = teacher.RunEpoch(input, outputs);

            for (int i = 0; i < input.Length; i++)
            {
                int answer;
                double[] output = network.Compute(input[i]);
                double response = output.Max(out answer);

                int expected = labels[i];
                Assert.AreEqual(expected, answer);
            }
        }
        public void JacobianByChainRuleTest_MultipleOutput()
        {
            // Network with no hidden layers: 3-4

            int numberOfInputs = 3;
            int numberOfClasses = 4;

            double[][] input = 
            {
                new double[] { -1, -1, -1 }, // 0
                new double[] { -1,  1, -1 }, // 1
                new double[] {  1, -1, -1 }, // 1
                new double[] {  1,  1, -1 }, // 0
                new double[] { -1, -1,  1 }, // 2
                new double[] { -1,  1,  1 }, // 3
                new double[] {  1, -1,  1 }, // 3
                new double[] {  1,  1,  1 }  // 2
            };

            int[] labels =
            {
                0,
                1,
                1,
                0,
                2,
                3,
                3,
                2,
            };

            double[][] output = Accord.Statistics.Tools
                .Expand(labels, numberOfClasses, -1, 1);

            Neuron.RandGenerator = new ThreadSafeRandom(0);

            ActivationNetwork network = new ActivationNetwork(
                   new BipolarSigmoidFunction(2), numberOfInputs, numberOfClasses);

            var teacher1 = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByFiniteDifferences);

            var teacher2 = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByBackpropagation);

            // Set lambda to lambda max so no iterations are performed
            teacher1.LearningRate = 1e30f;
            teacher2.LearningRate = 1e30f;

            teacher1.RunEpoch(input, output);
            teacher2.RunEpoch(input, output);

            PrivateObject privateTeacher1 = new PrivateObject(teacher1);
            PrivateObject privateTeacher2 = new PrivateObject(teacher2);

            var jacobian1 = (float[][])privateTeacher1.GetField("jacobian");
            var jacobian2 = (float[][])privateTeacher2.GetField("jacobian");


            for (int i = 0; i < jacobian1.Length; i++)
            {
                for (int j = 0; j < jacobian1[i].Length; j++)
                {
                    double j1 = jacobian1[i][j];
                    double j2 = jacobian2[i][j];

                    Assert.AreEqual(j1, j2, 1e-3);

                    Assert.IsFalse(Double.IsNaN(j1));
                    Assert.IsFalse(Double.IsNaN(j2));
                }
            }
        }
        public void JacobianByChainRuleTest4()
        {
            // Network with no hidden layers: 3-1

            double[][] input = 
            {
                new double[] {-1, -1 },
                new double[] {-1,  1 },
                new double[] { 1, -1 },
                new double[] { 1,  1 }
            };

            double[][] output =
            {
                new double[] {-1 },
                new double[] { 1 },
                new double[] { 1 },
                new double[] {-1 }
            };

            Neuron.RandGenerator = new ThreadSafeRandom(0);

            ActivationNetwork network = new ActivationNetwork(
                   new BipolarSigmoidFunction(2), 2, 1);

            var teacher1 = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByFiniteDifferences);

            var teacher2 = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByBackpropagation);

            // Set lambda to lambda max so no iterations are performed
            teacher1.LearningRate = 1e30f;
            teacher2.LearningRate = 1e30f;

            teacher1.RunEpoch(input, output);
            teacher2.RunEpoch(input, output);

            PrivateObject privateTeacher1 = new PrivateObject(teacher1);
            PrivateObject privateTeacher2 = new PrivateObject(teacher2);

            var jacobian1 = (float[][])privateTeacher1.GetField("jacobian");
            var jacobian2 = (float[][])privateTeacher2.GetField("jacobian");


            for (int i = 0; i < jacobian1.Length; i++)
            {
                for (int j = 0; j < jacobian1[i].Length; j++)
                {
                    double j1 = jacobian1[i][j];
                    double j2 = jacobian2[i][j];

                    Assert.AreEqual(j1, j2, 1e-5);

                    Assert.IsFalse(Double.IsNaN(j1));
                    Assert.IsFalse(Double.IsNaN(j2));
                }
            }
        }
        public void ConstructorTest()
        {
            // Four training samples of the xor function

            // two inputs (x and y)
            double[][] input = 
            {
                new double[] { -1, -1 },
                new double[] { -1,  1 },
                new double[] {  1, -1 },
                new double[] {  1,  1 }
            };

            // one output (z = x ^ y)
            double[][] output = 
            {
                new double[] { -1 },
                new double[] {  1 },
                new double[] {  1 },
                new double[] { -1 }
            };


            // create multi-layer neural network
            ActivationNetwork network = new ActivationNetwork(
                   new BipolarSigmoidFunction(2), // use a bipolar sigmoid activation function
                   2, // two inputs
                   3, // three hidden neurons
                   1  // one output neuron
                   );

            // create teacher
            LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(
                network, // the neural network
                false,   // whether or not to use Bayesian regularization
                JacobianMethod.ByBackpropagation // Jacobian calculation method
                );


            // set learning rate and momentum
            teacher.LearningRate = 0.1f;

            // start the supervisioned learning
            for (int i = 0; i < 1000; i++)
            {
                double error = teacher.RunEpoch(input, output);
            }

            // If we reached here, the constructor test has passed.
        }
        public void RunEpochTest4()
        {
            Accord.Math.Tools.SetupGenerator(0);

            double[][] input = 
            {
                new double[] { 0, 0 },
            };

            double[][] output =
            {
                new double[] { 0 },
            };

            Neuron.RandGenerator = new ThreadSafeRandom(0);
            ActivationNetwork network = new ActivationNetwork(
                   new BipolarSigmoidFunction(2), 2, 1);

            var teacher = new LevenbergMarquardtLearning(network,
                true, JacobianMethod.ByBackpropagation);

            double error = 1.0;
            for (int i = 0; i < 1000; i++)
                error = teacher.RunEpoch(input, output);

            for (int i = 0; i < input.Length; i++)
                Assert.AreEqual(network.Compute(input[i])[0], output[i][0], 0.1);
        }
        public void RunEpochTest3()
        {
            double[,] dataset = yinyang;

            double[][] input = dataset.GetColumns(0, 1).ToArray();
            double[][] output = dataset.GetColumn(2).ToArray();

            Neuron.RandGenerator = new ThreadSafeRandom(0);

            ActivationNetwork network = new ActivationNetwork(
                   new BipolarSigmoidFunction(2), 2, 5, 1);

            var teacher = new LevenbergMarquardtLearning(network,
                true, JacobianMethod.ByBackpropagation);

            Assert.IsTrue(teacher.UseRegularization);

            double error = 1.0;
            for (int i = 0; i < 500; i++)
                error = teacher.RunEpoch(input, output);

            double[][] actual = new double[output.Length][];

            for (int i = 0; i < input.Length; i++)
                actual[i] = network.Compute(input[i]);

            for (int i = 0; i < input.Length; i++)
                Assert.AreEqual(Math.Sign(output[i][0]), Math.Sign(actual[i][0]));
        }
        public void JacobianByChainRuleTest3()
        {
            // Network with 3 hidden layers: 2-4-3-4-1

            Accord.Math.Tools.SetupGenerator(0);

            double[][] input = 
            {
                new double[] {-1, -1 },
                new double[] {-1,  1 },
                new double[] { 1, -1 },
                new double[] { 1,  1 }
            };

            double[][] output =
            {
                new double[] {-1 },
                new double[] { 1 },
                new double[] { 1 },
                new double[] {-1 }
            };

            Neuron.RandGenerator = new ThreadSafeRandom(0);

            ActivationNetwork network = new ActivationNetwork(
                   new BipolarSigmoidFunction(2), 2, 4, 3, 4, 1);

            var teacher1 = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByFiniteDifferences);

            var teacher2 = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByBackpropagation);

            // Set lambda to lambda max so no iterations are performed
            teacher1.LearningRate = 1e30f;
            teacher2.LearningRate = 1e30f;

            teacher1.RunEpoch(input, output);
            teacher2.RunEpoch(input, output);

            var jacobian1 = teacher1.Jacobian;
            var jacobian2 = teacher2.Jacobian;


            for (int i = 0; i < jacobian1.Length; i++)
            {
                for (int j = 0; j < jacobian1[i].Length; j++)
                {
                    double j1 = jacobian1[i][j];
                    double j2 = jacobian2[i][j];

                    Assert.AreEqual(j1, j2, 1e-4);

                    Assert.IsFalse(Double.IsNaN(j1));
                    Assert.IsFalse(Double.IsNaN(j2));
                }
            }

        }
Exemple #9
0
        // Worker thread
        void SearchSolution()
        {
            // initialize input and output values
            double[][] input = null;
            double[][] output = null;

            if (sigmoidType == 0)
            {
                // unipolar data
                input = new double[4][] {
                                            new double[] {0, 0},
                                            new double[] {0, 1},
                                            new double[] {1, 0},
                                            new double[] {1, 1}
                                        };
                output = new double[4][] {
                                             new double[] {0},
                                             new double[] {1},
                                             new double[] {1},
                                             new double[] {0}
                                         };
            }
            else
            {
                // bipolar data
                input = new double[4][] {
                                            new double[] {-1, -1},
                                            new double[] {-1,  1},
                                            new double[] { 1, -1},
                                            new double[] { 1,  1}
                                        };
                output = new double[4][] {
                                             new double[] {-1},
                                             new double[] { 1},
                                             new double[] { 1},
                                             new double[] {-1}
                                         };
            }

            // create neural network
            ActivationNetwork network = new ActivationNetwork(
                (sigmoidType == 0) ?
                    (IActivationFunction)new SigmoidFunction(sigmoidAlphaValue) :
                    (IActivationFunction)new BipolarSigmoidFunction(sigmoidAlphaValue),
                2, 2, 1);

            // create teacher
            LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network);

            // set learning rate 
            teacher.LearningRate = learningRate;


            // iterations
            int iteration = 0;

            // statistic files
            StreamWriter errorsFile = null;

            try
            {
                // check if we need to save statistics to files
                if (saveStatisticsToFiles)
                {
                    // open files
                    errorsFile = File.CreateText("errors.csv");
                }

                // erros list
                ArrayList errorsList = new ArrayList();

                // loop
                while (!needToStop)
                {
                    // run epoch of learning procedure
                    double error = teacher.RunEpoch(input, output);
                    errorsList.Add(error);

                    // save current error
                    if (errorsFile != null)
                    {
                        errorsFile.WriteLine(error);
                    }

                    // show current iteration & error
                    SetText(currentIterationBox, iteration.ToString());
                    SetText(currentErrorBox, error.ToString());
                    iteration++;

                    // check if we need to stop
                    if (error <= learningErrorLimit)
                        break;
                }

                // show error's dynamics
                double[,] errors = new double[errorsList.Count, 2];

                for (int i = 0, n = errorsList.Count; i < n; i++)
                {
                    errors[i, 0] = i;
                    errors[i, 1] = (double)errorsList[i];
                }

                errorChart.RangeX = new Range(0, errorsList.Count - 1);
                errorChart.UpdateDataSeries("error", errors);
            }
            catch (IOException)
            {
                MessageBox.Show("Failed writing file", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
            }
            finally
            {
                // close files
                if (errorsFile != null)
                    errorsFile.Close();
            }

            // enable settings controls
            EnableControls(true);
        }
Exemple #10
0
        private static void network(double[][] inputs, int[] outputs)
        {
            // Since we would like to learn binary outputs in the form
            // [-1,+1], we can use a bipolar sigmoid activation function
            IActivationFunction function = new BipolarSigmoidFunction();

            // In our problem, we have 2 inputs (x, y pairs), and we will 
            // be creating a network with 5 hidden neurons and 1 output:
            //
            var network = new ActivationNetwork(function,
                inputsCount: 2, neuronsCount: new[] { 5, 1 });

            // Create a Levenberg-Marquardt algorithm
            var teacher = new LevenbergMarquardtLearning(network)
            {
                UseRegularization = true
            };


            // Because the network is expecting multiple outputs,
            // we have to convert our single variable into arrays
            //
            var y = outputs.ToDouble().ToArray();

            // Iterate until stop criteria is met
            double error = double.PositiveInfinity;
            double previous;

            do
            {
                previous = error;

                // Compute one learning iteration
                error = teacher.RunEpoch(inputs, y);

            } while (Math.Abs(previous - error) < 1e-10 * previous);


            // Classify the samples using the model
            int[] answers = inputs.Apply(network.Compute).GetColumn(0).Apply(System.Math.Sign);

            // Plot the results
            ScatterplotBox.Show("Expected results", inputs, outputs);
            ScatterplotBox.Show("Network results", inputs, answers)
                .Hold();
        }
Exemple #11
0
        // Worker thread
        void SearchSolution()
        {
            // number of learning samples
            int samples = data.GetLength(0);
            // data transformation factor
            double yFactor = 1.7 / chart.RangeY.Length;
            double yMin = chart.RangeY.Min;
            double xFactor = 2.0 / chart.RangeX.Length;
            double xMin = chart.RangeX.Min;

            // prepare learning data
            double[][] input = new double[samples][];
            double[][] output = new double[samples][];

            for (int i = 0; i < samples; i++)
            {
                input[i] = new double[1];
                output[i] = new double[1];

                // set input
                input[i][0] = (data[i, 0] - xMin) * xFactor - 1.0;
                // set output
                output[i][0] = (data[i, 1] - yMin) * yFactor - 0.85;
            }

            // create multi-layer neural network
            ActivationNetwork network = new ActivationNetwork(
                new BipolarSigmoidFunction(sigmoidAlphaValue),
                1, neuronsInFirstLayer, 1);

            if (useNguyenWidrow)
            {
                NguyenWidrow initializer = new NguyenWidrow(network);
                initializer.Randomize();
            }

            // create teacher
            LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, useRegularization);

            // set learning rate and momentum
            teacher.LearningRate = learningRate;

            // iterations
            int iteration = 1;

            // solution array
            double[,] solution = new double[50, 2];
            double[] networkInput = new double[1];

            // calculate X values to be used with solution function
            for (int j = 0; j < 50; j++)
            {
                solution[j, 0] = chart.RangeX.Min + (double)j * chart.RangeX.Length / 49;
            }

            // loop
            while (!needToStop)
            {
                // run epoch of learning procedure
                double error = teacher.RunEpoch(input, output) / samples;

                // calculate solution
                for (int j = 0; j < 50; j++)
                {
                    networkInput[0] = (solution[j, 0] - xMin) * xFactor - 1.0;
                    solution[j, 1] = (network.Compute(networkInput)[0] + 0.85) / yFactor + yMin;
                }
                chart.UpdateDataSeries("solution", solution);
                // calculate error
                double learningError = 0.0;
                for (int j = 0, k = data.GetLength(0); j < k; j++)
                {
                    networkInput[0] = input[j][0];
                    learningError += Math.Abs(data[j, 1] - ((network.Compute(networkInput)[0] + 0.85) / yFactor + yMin));
                }

                // set current iteration's info
                SetText(currentIterationBox, iteration.ToString());
                SetText(currentErrorBox, learningError.ToString("F3"));

                // increase current iteration
                iteration++;

                // check if we need to stop
                if ((iterations != 0) && (iteration > iterations))
                    break;
            }


            // enable settings controls
            EnableControls(true);
        }
        public void JacobianByChainRuleTest()
        {
            // Network with one hidden layer: 2-2-1
            Accord.Math.Tools.SetupGenerator(0);

            double[][] input = 
            {
                new double[] { -1, -1 },
                new double[] { -1,  1 },
                new double[] {  1, -1 },
                new double[] {  1,  1 }
            };

            double[][] output =
            {
                new double[] { -1 },
                new double[] {  1 },
                new double[] {  1 },
                new double[] { -1 }
            };

            Neuron.RandGenerator = new ThreadSafeRandom(0);

            ActivationNetwork network = new ActivationNetwork(
                   new BipolarSigmoidFunction(2), 2, 2, 1);

            var teacher1 = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByFiniteDifferences);

            var teacher2 = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByBackpropagation);

            // Set lambda to lambda max so no iterations are performed
            teacher1.LearningRate = 1e30f;
            teacher2.LearningRate = 1e30f;

            teacher1.RunEpoch(input, output);
            teacher2.RunEpoch(input, output);

            PrivateObject privateTeacher1 = new PrivateObject(teacher1);
            PrivateObject privateTeacher2 = new PrivateObject(teacher2);

            var jacobian1 = (float[][])privateTeacher1.GetField("jacobian");
            var jacobian2 = (float[][])privateTeacher2.GetField("jacobian");

            Assert.AreEqual(jacobian1[0][0], -0.47895513745387097, 1e-6);
            Assert.AreEqual(jacobian1[0][1], -0.05863886707282373, 1e-6);
            Assert.AreEqual(jacobian1[0][2], 0.057751100929897485, 1e-6);
            Assert.AreEqual(jacobian1[0][3], 0.0015185010717608583, 1e-6);

            Assert.AreEqual(jacobian1[7][0], -0.185400783651892, 1e-6);
            Assert.AreEqual(jacobian1[7][1], 0.025575161626462877, 1e-6);
            Assert.AreEqual(jacobian1[7][2], 0.070494677797224889, 1e-6);
            Assert.AreEqual(jacobian1[7][3], 0.037740463822781616, 1e-6);


            Assert.AreEqual(jacobian2[0][0], -0.4789595904719437, 1e-6);
            Assert.AreEqual(jacobian2[0][1], -0.058636153936941729, 1e-6);
            Assert.AreEqual(jacobian2[0][2], 0.057748435491340212, 1e-6);
            Assert.AreEqual(jacobian2[0][3], 0.0015184453425611988, 1e-6);

            Assert.AreEqual(jacobian2[7][0], -0.1854008206574258, 1e-6);
            Assert.AreEqual(jacobian2[7][1], 0.025575150379247645, 1e-6);
            Assert.AreEqual(jacobian2[7][2], 0.070494269423259301, 1e-6);
            Assert.AreEqual(jacobian2[7][3], 0.037740117733922635, 1e-6);


            for (int i = 0; i < jacobian1.Length; i++)
            {
                for (int j = 0; j < jacobian1[i].Length; j++)
                {
                    double j1 = jacobian1[i][j];
                    double j2 = jacobian2[i][j];

                    Assert.AreEqual(j1, j2, 1e-4);

                    Assert.IsFalse(Double.IsNaN(j1));
                    Assert.IsFalse(Double.IsNaN(j2));
                }
            }

        }
Exemple #13
0
        // Worker thread
        void SearchSolution()
        {
            // number of learning samples
            int samples = data.Length - predictionSize - windowSize;
            // data transformation factor
            double factor = 1.7 / chart.RangeY.Length;
            double yMin = chart.RangeY.Min;
            // prepare learning data
            double[][] input = new double[samples][];
            double[][] output = new double[samples][];

            for (int i = 0; i < samples; i++)
            {
                input[i] = new double[windowSize];
                output[i] = new double[1];

                // set input
                for (int j = 0; j < windowSize; j++)
                {
                    input[i][j] = (data[i + j] - yMin) * factor - 0.85;
                }
                // set output
                output[i][0] = (data[i + windowSize] - yMin) * factor - 0.85;
            }

            // create multi-layer neural network
            ActivationNetwork network = new ActivationNetwork(
                new BipolarSigmoidFunction(sigmoidAlphaValue),
                windowSize, windowSize * 2, 1);

            // create teacher
            LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, useRegularization);

            // set learning rate
            teacher.LearningRate = learningRate;

            // iterations
            int iteration = 1;

            // solution array
            int solutionSize = data.Length - windowSize;
            double[,] solution = new double[solutionSize, 2];
            double[] networkInput = new double[windowSize];

            // calculate X values to be used with solution function
            for (int j = 0; j < solutionSize; j++)
            {
                solution[j, 0] = j + windowSize;
            }

            // loop
            while (!needToStop)
            {
                // run epoch of learning procedure
                double error = teacher.RunEpoch(input, output) / samples;

                // calculate solution and learning and prediction errors
                double learningError = 0.0;
                double predictionError = 0.0;
                // go through all the data
                for (int i = 0, n = data.Length - windowSize; i < n; i++)
                {
                    // put values from current window as network's input
                    for (int j = 0; j < windowSize; j++)
                    {
                        networkInput[j] = (data[i + j] - yMin) * factor - 0.85;
                    }

                    // evalue the function
                    solution[i, 1] = (network.Compute(networkInput)[0] + 0.85) / factor + yMin;

                    // calculate prediction error
                    if (i >= n - predictionSize)
                    {
                        predictionError += Math.Abs(solution[i, 1] - data[windowSize + i]);
                    }
                    else
                    {
                        learningError += Math.Abs(solution[i, 1] - data[windowSize + i]);
                    }
                }
                // update solution on the chart
                chart.UpdateDataSeries("solution", solution);

                // set current iteration's info
                SetText(currentIterationBox, iteration.ToString());
                SetText(currentLearningErrorBox, learningError.ToString("F3"));
                SetText(currentPredictionErrorBox, predictionError.ToString("F3"));

                // increase current iteration
                iteration++;

                // check if we need to stop
                if ((iterations != 0) && (iteration > iterations))
                    break;
            }

            // show new solution
            for (int j = windowSize, k = 0, n = data.Length; j < n; j++, k++)
            {
                AddSubItem(dataList, j, solution[k, 1].ToString());
            }

            // enable settings controls
            EnableControls(true);
        }
Exemple #14
0
        // Worker thread
        void SearchSolution()
        {
            // number of learning samples
            int samples = data.GetLength(0);

            // prepare learning data
            DoubleRange unit = new DoubleRange(-1, 1);
            double[][] input = Tools.Scale(from: xRange, to: unit, x: data.GetColumn(0)).ToArray();
            double[][] output = Tools.Scale(from: yRange, to: unit, x: data.GetColumn(1)).ToArray();


            // create multi-layer neural network
            ActivationNetwork network = new ActivationNetwork(
                new BipolarSigmoidFunction(sigmoidAlphaValue),
                1, neuronsInFirstLayer, 1);

            if (useNguyenWidrow)
            {
                new NguyenWidrow(network).Randomize();
            }

            // create teacher
            var teacher = new LevenbergMarquardtLearning(network, useRegularization);

            // set learning rate and momentum
            teacher.LearningRate = learningRate;

            // iterations
            int iteration = 1;

            // solution array
            double[,] solution = new double[samples, 2];


            // loop
            while (!needToStop)
            {
                // run epoch of learning procedure
                double error = teacher.RunEpoch(input, output) / samples;

                // calculate solution
                for (int j = 0; j < samples; j++)
                {
                    double x = input[j][0];
                    double y = network.Compute(new[] { x })[0];
                    solution[j, 0] = Tools.Scale(from: unit, to: xRange, x: x);
                    solution[j, 1] = Tools.Scale(from: unit, to: yRange, x: y);
                }

                chart.UpdateDataSeries("solution", solution);

                // calculate error
                double learningError = 0.0;
                for (int j = 0; j < samples; j++)
                {
                    double x = input[j][0];
                    double expected = data[j, 1];
                    double actual = network.Compute(new[] { x })[0];
                    learningError += Math.Abs(expected - actual);
                }

                // set current iteration's info
                SetText(currentIterationBox, iteration.ToString());
                SetText(currentErrorBox, learningError.ToString("F3"));

                // increase current iteration
                iteration++;

                // check if we need to stop
                if ((iterations != 0) && (iteration > iterations))
                    break;
            }


            // enable settings controls
            EnableControls(true);
        }
        public void BlockHessianTest1()
        {
            // Network with no hidden layers: 3-1

            Accord.Math.Tools.SetupGenerator(0);

            double[][] input = 
            {
                new double[] {-1, -1 },
                new double[] {-1,  1 },
                new double[] { 1, -1 },
                new double[] { 1,  1 }
            };

            double[][] output =
            {
                new double[] {-1 },
                new double[] { 1 },
                new double[] { 1 },
                new double[] {-1 }
            };

            Neuron.RandGenerator = new ThreadSafeRandom(0);


            ActivationNetwork network = new ActivationNetwork(
                   new BipolarSigmoidFunction(2), 2, 1);

            var teacher1 = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByFiniteDifferences);

            var teacher2 = new LevenbergMarquardtLearning(network,
                false, JacobianMethod.ByBackpropagation);
            teacher2.Blocks = 2;

            // Set lambda to lambda max so no iterations are performed
            teacher1.LearningRate = 1e30f;
            teacher2.LearningRate = 1e30f;

            teacher1.RunEpoch(input, output);
            teacher2.RunEpoch(input, output);

            var hessian1 = teacher1.Hessian;
            var hessian2 = teacher1.Hessian;

            for (int i = 0; i < hessian1.Length; i++)
            {
                for (int j = 0; j < hessian1[i].Length; j++)
                {
                    double j1 = hessian1[i][j];
                    double j2 = hessian2[i][j];

                    Assert.AreEqual(j1, j2, 1e-4);

                    Assert.IsFalse(Double.IsNaN(j1));
                    Assert.IsFalse(Double.IsNaN(j2));
                }
            }

            Assert.IsTrue(hessian1.IsUpperTriangular());
            Assert.IsTrue(hessian2.IsUpperTriangular());

            var gradient1 = teacher1.Gradient;
            var gradient2 = teacher2.Gradient;

            for (int i = 0; i < gradient1.Length; i++)
            {
                double j1 = gradient1[i];
                double j2 = gradient2[i];

                Assert.AreEqual(j1, j2, 1e-5);

                Assert.IsFalse(Double.IsNaN(j1));
                Assert.IsFalse(Double.IsNaN(j2));
            }
        }
Exemple #16
0
        // Worker thread
        void SearchSolution()
        {
            // number of learning samples
            int samples = sourceMatrix.GetLength(0);

            // prepare learning data
            double[][] inputs = sourceMatrix.Submatrix(null, 0, 1).ToArray();
            double[][] outputs = sourceMatrix.GetColumn(2).Transpose().ToArray();

            // create multi-layer neural network
            ann = new ActivationNetwork(new GaussianFunction(sigmoidAlphaValue),
                //new BipolarSigmoidFunction(sigmoidAlphaValue),
                2, neuronsInFirstLayer, 1);

            if (useNguyenWidrow)
            {
                if (useSameWeights)
                    Accord.Math.Tools.SetupGenerator(0);

                NguyenWidrow initializer = new NguyenWidrow(ann);
                initializer.Randomize();
            }

            // create teacher
            LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(ann, useRegularization);

            // set learning rate and momentum
            teacher.LearningRate = learningRate;

            // iterations
            iteration = 1;

            var ranges = Matrix.Range(sourceMatrix, 0);
            double[][] map = Matrix.Mesh(ranges[0], ranges[1], 1, 1);
            var sw = Stopwatch.StartNew();

            // loop
            while (!needToStop)
            {
                // run epoch of learning procedure
                error = teacher.RunEpoch(inputs, outputs) / samples;

                //var result = map.Apply(ann.Compute).GetColumn(0).Apply(Math.Sign);
                var result = map.Apply(ann.Compute).GetColumn(0).Apply(d =>
                {
                    if (d > 0.66) return 2;
                    else if (d < 0.33)
                        return 0;
                    return 1;
                });

                var graph = map.ToMatrix().InsertColumn(result.ToDouble());

                CreateScatterplot(zedGraphControl2, graph);

                // increase current iteration
                iteration++;

                elapsed = sw.Elapsed;

                updateStatus();

                // check if we need to stop
                if ((iterations != 0) && (iteration > iterations))
                    break;
            }

            sw.Stop();

            // enable settings controls
            EnableControls(true);
        }
        public void ZeroLambdaTest()
        {
            double[,] data = null;

            // open selected file
            using (TextReader stream = new StringReader(Properties.Resources.ZeroLambda))
            using (CsvReader reader = new CsvReader(stream, false))
            {
                data = reader.ToTable().ToMatrix();
            }

            // number of learning samples
            int samples = data.GetLength(0);

            var ranges = data.Range(dimension: 0);

            Assert.AreEqual(2, ranges.Length);

            var rangeX = ranges[0];
            var rangeY = ranges[1];

            // data transformation factor
            double yFactor = 1.7 / rangeY.Length;
            double yMin = rangeY.Min;
            double xFactor = 2.0 / rangeX.Length;
            double xMin = rangeX.Min;

            // prepare learning data
            double[][] input = new double[samples][];
            double[][] output = new double[samples][];

            for (int i = 0; i < samples; i++)
            {
                input[i] = new double[1];
                output[i] = new double[1];

                input[i][0] = (data[i, 0] - xMin) * xFactor - 1.0; // set input
                output[i][0] = (data[i, 1] - yMin) * yFactor - 0.85; // set output
            }

            // create multi-layer neural network
            ActivationNetwork network = new ActivationNetwork(
                new BipolarSigmoidFunction(5),
                1, 12, 1);

            // create teacher
            LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, true);

            teacher.LearningRate = 1;

            // iterations
            int iteration = 1;
            int iterations = 2000;

            // solution array
            double[,] solution = new double[samples, 2];
            double[] networkInput = new double[1];

            bool needToStop = false;

            double learningError = 0;

            // loop
            while (!needToStop)
            {
                Assert.AreNotEqual(0, teacher.LearningRate);

                // run epoch of learning procedure
                double error = teacher.RunEpoch(input, output) / samples;

                // calculate solution
                for (int j = 0; j < samples; j++)
                {
                    networkInput[0] = (solution[j, 0] - xMin) * xFactor - 1.0;
                    solution[j, 1] = (network.Compute(networkInput)[0] + 0.85) / yFactor + yMin;
                }


                // calculate error
                learningError = 0.0;
                for (int j = 0; j < samples; j++)
                {
                    networkInput[0] = input[j][0];
                    learningError += Math.Abs(data[j, 1] - ((network.Compute(networkInput)[0] + 0.85) / yFactor + yMin));
                }

                // increase current iteration
                iteration++;

                // check if we need to stop
                if ((iterations != 0) && (iteration > iterations))
                    break;
            }

            Assert.IsTrue(learningError < 0.13);
        }
        public BackPropogation()
        {
            InitializeComponent();

            activation_nework = new ActivationNetwork(new BipolarSigmoidFunction(), 50, 50,  10, 1);

            watch1 = new Stopwatch();

            watch2 = new Stopwatch();

            watch3 = new Stopwatch();

            backgroundWorkerTrainer.Disposed += backgroundWorkerTrainer_Disposed;
            backgroundWorkerTrainer.DoWork += backgroundWorkerTrainer_DoWork;
            backgroundWorkerTrainer.ProgressChanged += backgroundWorkerTrainer_ProgressChanged;
            backgroundWorkerTrainer.WorkerSupportsCancellation = true;
            backgroundWorkerTrainer.WorkerReportsProgress = true;
            saveFileDialog1.Filter = "feed forward network files (*.ffn)|*.ffn";
            saveFileDialog1.Title = "Save neural networkfile";
            saveFileDialog1.InitialDirectory = null;
            saveFileDialog1.FileName = null;

            openFileDialog1.Filter = "feed forward network files (*.ffn)|*.ffn";
            openFileDialog1.Title = "Load neural network file";
            openFileDialog1.InitialDirectory = null;
            openFileDialog1.FileName = null;

            backgroundWorkerSignal.Disposed += backgroundWorkerSignal_Disposed;
            backgroundWorkerSignal.WorkerSupportsCancellation = true;
            backgroundWorkerSignal.WorkerReportsProgress = true;
            backgroundWorkerSignal.DoWork += backgroundWorkerSignal_DoWork;
            backgroundWorkerSignal.RunWorkerCompleted += backgroundWorkerSignal_RunWorkerCompleted;//80, 70, 60, 50, 40,
            network1 = activation_nework;
            network2 = activation_nework; // new ActivationNetwork(new BipolarSigmoidFunction(), 50, 50, 40, 1);
            network3 = activation_nework; // new ActivationNetwork(new BipolarSigmoidFunction(), 50, 50, 40, 1);
            network4 = activation_nework; // new ActivationNetwork(new BipolarSigmoidFunction(), 50, 50, 40, 1);
            network5 = new ActivationNetwork(new BipolarSigmoidFunction(), 50,1);
            network6 = new ActivationNetwork(new BipolarSigmoidFunction(), 50, 1);
            teacher = new BackPropagationLearning(network1);
            evteacher = new EvolutionaryLearning(network2, 100);
            reprop = new ResilientBackpropagationLearning(network3);
            lbteacher = new LevenbergMarquardtLearning(network4);
            delta = new DeltaRuleLearning(network5);
            perceptron = new PerceptronLearning(network6);
            delta.LearningRate = 1;
            perceptron.LearningRate = 0.1;
            myPane = new GraphPane();
            listPointsOne = new PointPairList();

            myPane = zedGraphControl1.GraphPane;

            // set a title
            myPane.Title.Text = "Error VS Time";

            // set X and Y axis titles
            myPane.XAxis.Title.Text = "Time in Milliseconds";
            myPane.YAxis.Title.Text = "Error";
            myCurveOne = myPane.AddCurve("Learning curve", listPointsOne, Color.Red, SymbolType.None);
               // myCurveOne = myPane.AddCurve("Resilient Back Propagation", listPointstwo, Color.Green, SymbolType.None);
               // myCurveOne = myPane.AddCurve("Genetic Learning", listPointsthree, Color.Blue, SymbolType.None);
        }