public void RunEpochTest1() { Accord.Math.Tools.SetupGenerator(0); double[][] input = { new double[] { -1, -1 }, new double[] { -1, 1 }, new double[] { 1, -1 }, new double[] { 1, 1 } }; double[][] output = { new double[] { -1 }, new double[] { 1 }, new double[] { 1 }, new double[] { -1 } }; Neuron.RandGenerator = new ThreadSafeRandom(0); ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), 2, 2, 1); var teacher = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByFiniteDifferences); double error = 1.0; while (error > 1e-5) error = teacher.RunEpoch(input, output); for (int i = 0; i < input.Length; i++) Assert.AreEqual(network.Compute(input[i])[0], output[i][0], 0.1); }
/// <summary> /// Initializes a new instance of the <see cref="BackPropagationLearning"/> class /// </summary> /// /// <param name="network">Network to teach</param> /// public BackPropagationLearning( ActivationNetwork network ) { this.network = network; // create error and deltas arrays neuronErrors = new double[network.LayersCount][]; weightsUpdates = new double[network.LayersCount][][]; thresholdsUpdates = new double[network.LayersCount][]; // initialize errors and deltas arrays for each layer for ( int i = 0, n = network.LayersCount; i < n; i++ ) { Layer layer = network[i]; neuronErrors[i] = new double[layer.NeuronsCount]; weightsUpdates[i] = new double[layer.NeuronsCount][]; thresholdsUpdates[i] = new double[layer.NeuronsCount]; // for each neuron for ( int j = 0; j < layer.NeuronsCount; j++ ) { weightsUpdates[i][j] = new double[layer.InputsCount]; } } }
public void MulticlassTest1() { Accord.Math.Tools.SetupGenerator(0); Neuron.RandGenerator = new ThreadSafeRandom(0); int numberOfInputs = 3; int numberOfClasses = 4; int hiddenNeurons = 5; double[][] input = { new double[] { -1, -1, -1 }, // 0 new double[] { -1, 1, -1 }, // 1 new double[] { 1, -1, -1 }, // 1 new double[] { 1, 1, -1 }, // 0 new double[] { -1, -1, 1 }, // 2 new double[] { -1, 1, 1 }, // 3 new double[] { 1, -1, 1 }, // 3 new double[] { 1, 1, 1 } // 2 }; int[] labels = { 0, 1, 1, 0, 2, 3, 3, 2, }; double[][] outputs = Accord.Statistics.Tools .Expand(labels, numberOfClasses, -1, 1); var function = new BipolarSigmoidFunction(2); var network = new ActivationNetwork(function, numberOfInputs, hiddenNeurons, numberOfClasses); new NguyenWidrow(network).Randomize(); var teacher = new LevenbergMarquardtLearning(network); double error = Double.PositiveInfinity; for (int i = 0; i < 10; i++) error = teacher.RunEpoch(input, outputs); for (int i = 0; i < input.Length; i++) { int answer; double[] output = network.Compute(input[i]); double response = output.Max(out answer); int expected = labels[i]; Assert.AreEqual(expected, answer); } }
/// <summary> /// Constructs a new Gaussian Weight initialization. /// </summary> /// /// <param name="network">The activation network whose weights will be initialized.</param> /// <param name="stdDev">The standard deviation to be used. Common values lie in the 0.001- /// 0.1 range. Default is 0.1.</param> /// public GaussianWeights(ActivationNetwork network, double stdDev = 0.1) { this.network = network; this.random = new GaussianGenerator(0f, (float)stdDev, Accord.Math.Tools.Random.Next()); this.UpdateThresholds = false; }
/// <summary> /// Generates points trained points from learning. /// </summary> private async Task <double[][]> ComputeTrainedPoints(double[][] correctPoints) { return(await Task.Run(() => { lock (_threadLocker) { try { _network = (ActivationNetwork)Network.Load(ConfigurationPath()); } catch (Exception ex) { throw ex; } Scaler scaler = new Scaler(correctPoints); var inputScaledData = correctPoints.Select(a => a.ToArray()).ToArray(); scaler.Scale(ref inputScaledData); var solution = inputScaledData.Select(a => a.ToArray()).ToArray(); var networkInput = new double[2 * _inputPoints]; for (int j = _inputPoints; j < solution.GetLength(0); j++) { for (int k = _inputPoints, l = 0; k > 0; k--, l = l + 2) { networkInput[l] = solution[j - k][0]; networkInput[l + 1] = solution[j - k][1]; } try { var result = _network.Compute(networkInput); solution[j][0] = result[0]; solution[j][1] = result[1]; } catch { solution[j][0] = double.NaN; solution[j][1] = double.NaN; } } for (int i = 0; i < solution.GetLength(0); i++) { if (double.IsNaN(solution[i][0]) || double.IsNaN(solution[i][1])) { continue; } solution[i][0] = scaler.Rescale(solution[i][0], XYEnum.X); solution[i][1] = scaler.Rescale(solution[i][1], XYEnum.Y); } _network.Save(ConfigurationPath()); _network = null; return solution; } })); }
public void JacobianByChainRuleTest3() { // Network with 3 hidden layers: 2-4-3-4-1 Accord.Math.Tools.SetupGenerator(0); double[][] input = { new double[] { -1, -1 }, new double[] { -1, 1 }, new double[] { 1, -1 }, new double[] { 1, 1 } }; double[][] output = { new double[] { -1 }, new double[] { 1 }, new double[] { 1 }, new double[] { -1 } }; Neuron.RandGenerator = new ThreadSafeRandom(0); ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), 2, 4, 3, 4, 1); var teacher1 = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByFiniteDifferences); var teacher2 = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByBackpropagation); // Set lambda to lambda max so no iterations are performed teacher1.LearningRate = 1e30f; teacher2.LearningRate = 1e30f; teacher1.RunEpoch(input, output); teacher2.RunEpoch(input, output); var jacobian1 = teacher1.Jacobian; var jacobian2 = teacher2.Jacobian; for (int i = 0; i < jacobian1.Length; i++) { for (int j = 0; j < jacobian1[i].Length; j++) { double j1 = jacobian1[i][j]; double j2 = jacobian2[i][j]; Assert.AreEqual(j1, j2, 1e-4); Assert.IsFalse(Double.IsNaN(j1)); Assert.IsFalse(Double.IsNaN(j2)); } } }
static void Main(string[] args) { string fn = @"C:\Users\semio\Desktop\nefile.csv"; var f = File.ReadLines(fn); var data = from z in f.Skip(1) let zz = z.Split(';') select new Digit { Label = int.Parse(zz.Last()), Width = int.Parse(zz[1]), Height = int.Parse(zz[2]), Image = zz.FirstOrDefault().Split(' ').Select(double.Parse).ToArray() }; Digit[] train = data.Take(9).ToArray(); Digit[] test = data.Skip(9).ToArray(); double prErr = 10000000; double error = 100; var network = new ActivationNetwork(new SigmoidFunction(), 50 * 50, 100, 100, 2); var learner = new BackPropagationLearning(network) { LearningRate = 0.005 }; Console.WriteLine("Start Learning"); new GaussianWeights(network, 0.1).Randomize(); double err = 1; int i = 1; while (err > 0.004) { err = learner.RunEpoch((from x in train select x.Image.Select(t => (double)t).ToArray()).ToArray(), (from x in train select ToOneHotEnc(x.Label, 2)).ToArray()); Console.WriteLine($"EpochNum = {i} --- Error={err}"); i++; if (Math.Abs(error - prErr) < 0.0001) { // Уменьшаем коэффициент скорости обучения на 2 //learner.LearningRate /= 2; //if (learner.LearningRate < 0.001) // learner.LearningRate = 0.001; } prErr = error; } foreach (Digit v in test) { var n = network.Compute(v.Image); //ImageBox.Show(v.Image.Select(x => (double)x / 2).ToArray(), 50, 50); var z = getMaxInex(n); Console.WriteLine("{0} => {1}; 2 - {2}; 3 - {3}", v.Label, z + 2, n[0], n[1]); } Console.ReadKey(); }
public NeuralNetwork(double[][] inputData, double[][] outputData) { _inputData = inputData; _outputData = outputData; Network = new ActivationNetwork(new SigmoidFunction(), inputData[0].Length, // Input neuron count outputData[0].Length); // Output neuron count Network.Randomize(); }
public AttackStateNN() { var activationFunction = new Accord.Neuro.ActivationFunctions.GaussianFunction(); network = new ActivationNetwork(new SigmoidFunction(0.01), 4 + 4 + 4 * 5 * 5 + 25, new[] { 100, 100, 1 }); new GaussianWeights(network, 0.3).Randomize(); teacher = new ResilientBackpropagationLearning(network); teacher.LearningRate = 0.1; }
public static ActivationNetwork LoadNetwork(string file) { IFormatter formatter = new BinaryFormatter(); Stream stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read); ActivationNetwork network = (ActivationNetwork)formatter.Deserialize(stream); stream.Close(); return(network); }
/// <summary> /// Реиницализация сети - создаём заново сеть с указанной структурой /// </summary> /// <param name="structure">Массив с указанием нейронов на каждом слое (включая сенсорный)</param> /// <param name="initialLearningRate">Начальная скорость обучения</param> public override void ReInit(int[] structure, double initialLearningRate = 0.25) { // Создаём сеть - вроде того network = new ActivationNetwork((IActivationFunction) new SigmoidFunction(2.0), structure[0], structure.Skip(1).ToArray <int>()); // Встряска "мозгов" сети - рандомизируем веса связей new NguyenWidrow(network).Randomize(); }
public void CreateNetwork() { // create neural network network = new ActivationNetwork( new BipolarSigmoidFunction(), 1, // inputs in the network 10, // neurons in the first (hidden) layer 1); // neuron in the second (output) layer }
public void Accept(INet sourceNet) { _network = Network.Load(sourceNet.Persistence()) as ActivationNetwork; _teacher = new BackPropagationLearning(_network) { LearningRate = _learningRate, Momentum = 0.9 }; }
private void button2_Click_1(object sender, EventArgs e) { button2.Enabled = false; network2 = null; reprop = null; network2 = new ActivationNetwork(new BipolarSigmoidFunction(param1), 100, 100, 90, 80, 70, 60, 50, 40, 30, 20, 10, 1); reprop = new ResilientBackpropagationLearning(network3); network2.Randomize(); }
/// <summary> /// Load a network /// </summary> /// <param name="FilePath">The path to the binary network file</param> /// <returns></returns> private void LoadNet(string FilePath) { FileStream fs = new FileStream(FilePath, FileMode.Open); BinaryFormatter formatter = new BinaryFormatter(); ActivationNetwork net = (ActivationNetwork)formatter.Deserialize(fs); fs.Close(); network = net; }
private void button3_Click_1(object sender, EventArgs e) { button3.Enabled = false; network1 = null; teacher = null; network1 = new ActivationNetwork(new BipolarSigmoidFunction(param2), 100, 100, 90, 80, 70, 60, 50, 40, 30, 20, 10, 1); teacher = new BackPropagationLearning(network1); network1.Randomize(); }
private void button4_Click_1(object sender, EventArgs e) { button4.Enabled = false; network3 = null; evteacher = null; network3 = new ActivationNetwork(new BipolarSigmoidFunction(param3), 100, 100, 90, 80, 70, 60, 50, 40, 30, 20, 10, 1); evteacher = new EvolutionaryLearning(network2, 100); network3.Randomize(); }
internal override void Think(EvoEngine engine) { if (Network == null) { Network = new ActivationNetwork(new BipolarSigmoidFunction(), WorldScene.NeuralNetworkShape[0], WorldScene.NeuralNetworkShape.Skip(1).ToArray()); // Nw = (I+1)*H1 +(H1+1)*H2 +(H2+1)*O // I = inputs // H1 = neurons in hidden layer 1 // H2 = neurons in hidden layer 2 // O = Number of outputs // Nw = (5+1)*0 + (0+1)*0 + (5+1)*4 = 24 // 5in 4out // Nw = (5+1)*4 + (4+1)*0 + (4+1)*4 = 44 // 5in 1hl4 4out // Nw = (11+1)*4 + (4+1)*0 + (4+1)*4 = 68 // 11in 1hl4 4out var dna = GetCurrentGene(); SetNetworkWeights(dna); //float scale = (float)dna.Genes[scene.NeuralNetworkWeightsCount - 1 + 0]; //if (scale < 0.4) scale = 0.4f; //if (scale > 0.6) scale = 0.6f; //Snake.SetScale(scale); //int div = scene.NeuralNetworkWeightsCount / 3; //Color c = new Color( // (int)Math.Max((dna.Genes[div*0] * 255), 128), // (int)Math.Max((dna.Genes[div*1] * 255), 128), // (int)Math.Max((dna.Genes[div*2] * 255), 128) // ); //Snake.Color = c; } if (engine.KeyState.GetPressedKeys().Length > 0) { return; } double[] netout = Network.Compute(convertToDouble(Snake.GetFoodSensorsActivation())); //if (Math.Max(netout[0], netout[1]) > 0.5) if (netout[0] > netout[1]) { Snake.Accelerate(); } else { Snake.Brake(); } //if (Math.Max(netout[2], netout[3]) > 0.5) if (netout[2] > netout[3]) { Snake.RotateLeft(); } else { Snake.RotateRight(); } }
public void ExecuteFold(int k) { int LengthOfInput = this.FormattedData[0].Input.Count(); int LengthOfOutput = this.FormattedData[0].Output.Count(); ActivationNetwork NeuralNetwork = new ActivationNetwork( new SigmoidFunction(2), LengthOfInput, this.NumberOfHiddenLayerNeurons(LengthOfInput, LengthOfOutput), LengthOfOutput); NguyenWidrow weights = new NguyenWidrow(NeuralNetwork); weights.Randomize(); ResilientBackpropagationLearning BackProp = new ResilientBackpropagationLearning(NeuralNetwork); BackProp.LearningRate = this.LearningRate; //BackProp.Momentum = 0.5; List <NetIO> TrainingData = new List <NetIO>(); List <NetIO> ValidationData = new List <NetIO>(); ReadWrite.RemoveKFold(this.FormattedData, ref TrainingData, ref ValidationData, k); // for each epoch int epoch = 0; int maxEpochs = int.MaxValue; EarlyStoppingTools netError = new EarlyStoppingTools(this.patience); do { ++epoch; double internalError = BackProp.RunEpoch(TrainingData.Select(l => l.Input.ToArray()).ToArray(), TrainingData.Select(l => l.Output.ToArray()).ToArray()); this.RssError = EarlyStoppingTools.RssError(NeuralNetwork, ValidationData.Select(l => l.Input.ToArray()).ToArray(), ValidationData.Select(l => l.Output.ToArray()).ToArray()); //Console.WriteLine("Epochs: " + epoch); //Console.WriteLine("Training error: " + internalError); //Console.WriteLine("CV Error: " + this.RssError); } while (!netError.ExceedsPatience(RssError) && epoch < maxEpochs); Console.Write("Target: "); ValidationData[0].Output.ForEach(i => Console.Write(i)); Console.WriteLine(); Console.WriteLine("Result: " + string.Join(",", NeuralNetwork.Compute(ValidationData[0].Input.ToArray()))); this.NumberOfEpochs = epoch; Console.WriteLine("Epochs required: " + epoch); Console.WriteLine("Error: " + RssError); }
static void Main(string[] args) { //vybrat excel s hodnotami //vyparsovat ho, ulozit si hodnoty //rozbehat neuronku //naucit neuronku //testovat neuronku //skusit pustit data on-the-fly OpenFileDialog ofd = new OpenFileDialog() { Multiselect = false }; string excelFileName; if (ofd.ShowDialog() == DialogResult.OK) { excelFileName = ofd.FileName; } else { throw new Exception("Je nutne vybrat subor"); } var excelManager = new ExcelParser(excelFileName); excelManager.ParseSourceData(); double[][] input = NeuronData.GetInputMatrix(); double[][] output = NeuronData.GetOutputMatrix(); // create neural network ActivationNetwork network = new ActivationNetwork( new SigmoidFunction(3), 3, // two inputs in the network 3, // two neurons in the first layer 2); // one neuron in the second layer // create teacher BackPropagationLearning teacher = new BackPropagationLearning(network); bool needToStop = false; // loop while (!needToStop) { // run epoch of learning procedure System.Threading.Thread.Sleep(1000); double error = teacher.RunEpoch(input, output); Console.WriteLine(error); // check error value to see if we need to stop // ... } }
private void BtnLoadData_Click(object sender, EventArgs e) { network = new ActivationNetwork(new SigmoidFunction(2), 1024, 256, 128, 10); new NguyenWidrow(network).Randomize(); StringReader reader = new StringReader(File.ReadAllText("Data\\optdigits-tra.txt")); int trainingStart = 0; int trainingCount = 1000; int testingStart = 1000; int testingCount = 1000; int c1 = 0; int c2 = 1; int trainingSet = 0; int testingSet = 0; dgvLearnSet.Rows.Clear(); dgvLearnSet.RowTemplate.Height = 40; dgvVerifySet.Rows.Clear(); dgvVerifySet.RowTemplate.Height = 40; while (true) { char[] buffer = new char[33 * 32]; int read = reader.ReadBlock(buffer, 0, buffer.Length); string label = reader.ReadLine(); if (read < buffer.Length || label == null) { break; } if (c1 > trainingStart && c1 <= trainingStart + trainingCount) { Bitmap bitmap = Ultilities.Extract(new string(buffer)); double[] features = Ultilities.Extract(bitmap); int clabel1 = Int32.Parse(label); dgvLearnSet.Rows.Add(c1, bitmap, clabel1, features); trainingSet++; } else if (c1 > testingStart && c1 <= testingStart + testingCount) { Bitmap bitmap = Ultilities.Extract(new string(buffer)); double[] features = Ultilities.Extract(bitmap); int clabel1 = Int32.Parse(label); dgvVerifySet.Rows.Add(c2, bitmap, "", features, clabel1); testingSet++; c2++; } c1++; } btnLearn.Enabled = true; btnVerify.Enabled = true; }
private static void Training() { var csv = new CsvReader(new StreamReader("C:\\Users\\jantk_000\\Documents\\SearchResultRelevance\\trainextracted.csv")); List <LabeledData> data = csv.GetRecords <LabeledData>().ToList(); List <LabeledData> train = new List <LabeledData>(); List <LabeledData> validation = new List <LabeledData>(); for (int i = 0; i < data.Count; i++) { if (i % 5 == 0) { validation.Add(data[i]); } else { train.Add(data[i]); } } double[][] trainInput = train.Select(labeledData => labeledData.GetInputVector()).ToArray(); double[][] trainOutput = train.Select(labeledData => new[] { labeledData.GetOutput() }).ToArray(); double[][] validationInput = validation.Select(labeledData => labeledData.GetInputVector()).ToArray(); double[][] validationOutput = validation.Select(labeledData => new[] { labeledData.GetOutput() }).ToArray(); ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(), 3, 4, 1); BackPropagationLearning rprop = new BackPropagationLearning(network) { LearningRate = 0.01, Momentum = 0.01 }; double lastError = double.MaxValue; for (int i = 0;; i++) { rprop.RunEpoch(trainInput, trainOutput); double err = 0; for (int j = 0; j < validationInput.Length; j++) { double[] input = validationInput[j]; double[] output = network.Compute(input); err += Math.Sqrt(Math.Pow(validationOutput[j][0] - output[0], 2)); } err = err / validationInput.Length; var improvement = lastError - err; lastError = err; Console.WriteLine("Iteration {0} RMSE {1} Improvement {2}", i, err, improvement); if (improvement < 0.000001) { break; } } network.Save("Network.bin"); }
private ActivationNetwork Copy(ActivationNetwork network) { using (var memoryStream = new MemoryStream()) { network.Save(memoryStream); memoryStream.Position = 0; return((ActivationNetwork)Network.Load(memoryStream)); } }
public Network(int input_sz, int output_sz = 10) { input_size = input_sz; output_size = output_sz; net = new ActivationNetwork(new Accord.Neuro.BipolarSigmoidFunction(), input_size, input_size * 3, input_size * 2, input_size, 100, output_size); backprop = new ParallelResilientBackpropagationLearning(net); nguen = new NguyenWidrow(net); nguen.Randomize(); }
/// <summary> /// Deserialization Constructor /// </summary> /// <param name="info"></param> /// <param name="ctxt"></param> public BackPropagationLearning(SerializationInfo info, StreamingContext ctxt) { //Get the values from info and assign them to the appropriate properties network = (ActivationNetwork)info.GetValue("network", typeof(ActivationNetwork)); learningRate = (double)info.GetValue("learningRate", typeof(double)); momentum = (double)info.GetValue("momentum", typeof(double)); neuronErrors = (double[][])info.GetValue("neuronErrors", typeof(double[][])); weightsUpdates = (double[][][])info.GetValue("weightsUpdates", typeof(double[][][])); thresholdsUpdates = (double[][])info.GetValue("thresholdsUpdates", typeof(double[][])); }
/// <summary> /// Initializes a new instance of the <see cref="DeltaRuleLearning"/> class. /// </summary> /// /// <param name="network">Network to teach.</param> /// /// <exception cref="ArgumentException">Invalid nuaral network. It should have one layer only.</exception> /// public DeltaRuleLearning( ActivationNetwork network ) { // check layers count if ( network.Layers.Length != 1 ) { throw new ArgumentException( "Invalid nuaral network. It should have one layer only." ); } this.network = network; }
protected override void CreateNetwork() { network = new ActivationNetwork(new Tanh(0.1), 1, 7, 7, 6, 1); network.ForEachWeight(z => rnd.NextDouble() * 2 - 1); teacher = new BackPropagationLearning(network); teacher.LearningRate = 1; teacher.Momentum = 0.3; }
/// <summary> /// Constructs a new Nguyen-Widrow Weight initialization. /// </summary> /// /// <param name="network">The activation network whose weights will be initialized.</param> /// public NguyenWidrow(ActivationNetwork network) { this.network = network; int hiddenNodes = network.Layers[0].Neurons.Length; int inputNodes = network.Layers[0].InputsCount; randRange = new Range(-0.5f, 0.5f); beta = 0.7 * Math.Pow(hiddenNodes, 1.0 / inputNodes); }
/// <summary> /// Initializes a new instance of the <see cref="DeltaRuleLearning"/> class. /// </summary> /// /// <param name="network">Network to teach.</param> /// /// <exception cref="ArgumentException">Invalid nuaral network. It should have one layer only.</exception> /// public DeltaRuleLearning(ActivationNetwork network) { // check layers count if (network.Layers.Length != 1) { throw new ArgumentException("Invalid nuaral network. It should have one layer only."); } this.network = network; }
/// <summary> /// Initializes a new instance of the <see cref="LevenbergMarquardtLearning"/> class. /// </summary> /// /// <param name="network">Network to teach.</param> /// <param name="useRegularization">True to use Bayesian regularization, false otherwise.</param> /// <param name="method">The method by which the Jacobian matrix will be calculated.</param> /// public LevenbergMarquardtLearning(ActivationNetwork network, bool useRegularization, JacobianMethod method) { this.ParallelOptions = new ParallelOptions(); this.network = network; this.numberOfParameters = getNumberOfParameters(network); this.outputCount = network.Layers[network.Layers.Length - 1].Neurons.Length; this.useBayesianRegularization = useRegularization; this.method = method; this.weights = new float[numberOfParameters]; this.hessian = new float[numberOfParameters][]; for (int i = 0; i < hessian.Length; i++) { hessian[i] = new float[numberOfParameters]; } this.diagonal = new float[numberOfParameters]; this.gradient = new float[numberOfParameters]; this.jacobian = new float[numberOfParameters][]; // Will use Backpropagation method for Jacobian computation if (method == JacobianMethod.ByBackpropagation) { // create weight derivatives arrays this.weightDerivatives = new float[network.Layers.Length][][]; this.thresholdsDerivatives = new float[network.Layers.Length][]; // initialize arrays for (int i = 0; i < network.Layers.Length; i++) { ActivationLayer layer = (ActivationLayer)network.Layers[i]; this.weightDerivatives[i] = new float[layer.Neurons.Length][]; this.thresholdsDerivatives[i] = new float[layer.Neurons.Length]; for (int j = 0; j < layer.Neurons.Length; j++) { this.weightDerivatives[i][j] = new float[layer.InputsCount]; } } } else // Will use finite difference method for Jacobian computation { // create differential coefficient arrays this.differentialCoefficients = createCoefficients(3); this.derivativeStepSize = new double[numberOfParameters]; // initialize arrays for (int i = 0; i < numberOfParameters; i++) { this.derivativeStepSize[i] = derivativeStep; } } }
public NeuralNetwork(Accord.Neuro.IActivationFunction function, int numberOfInputs, params int[] numberOfNeurons) { inputSize = numberOfInputs; outputSize = numberOfNeurons[numberOfNeurons.Length - 1]; activationFunction = function; this.numberOfNeurons = numberOfNeurons; network = new ActivationNetwork(function, numberOfInputs, numberOfNeurons); //InitalizeInputs(numberOfInputs); //InitalizeOutputs(numberOfNeurons[numberOfNeurons.Length - 1]); }
public void Reset() { PersonOccurenceDatabase = new WordOccurenceDatabase(); OrganizationOccurenceDatabase = new WordOccurenceDatabase(); LocationsOccurenceDatabase = new WordOccurenceDatabase(); SportSpecificWordsOccurenceDatabase = new WordOccurenceDatabase(); int hiddenLayerCount = (10 * _categoriesCount) / 3; _classifier = new ActivationNetwork(new SigmoidFunction(), 4 * _categoriesCount, hiddenLayerCount, _categoriesCount); _teacher = new BackPropagationLearning(_classifier); }
public BackPropagationLearning(ActivationNetwork network) { this.network = network; neuronErrors = new double[network.LayersCount][]; for (var i = 0; i < network.LayersCount; i++) { neuronErrors[i] = new double[network[i].NeuronsCount]; } }
private static PredictiveValues CalculatePredictiveValues(ActivationNetwork ann, Data data) { var annOutputs = new double[data.ValidateOutputs.Length][]; for (int i = 0; i < data.ValidateInputs.Length; i++) { var vi = data.ValidateInputs[i]; annOutputs[i] = new double[] { Math.Round(ann.Compute(vi)[0], 0) }; } return CalculatePredictiveValues(data.ValidateOutputs, annOutputs); }
public double Treinar() { //var activationNetwork = new ActivationNetwork(new ThresholdFunction(), tamanhoEntrada, _classes.Count); //var supervisedLearning = new PerceptronLearning(activationNetwork); activationNetwork = new ActivationNetwork(new SigmoidFunction(0.5), tamanhoEntrada, _classes.Count); //var supervisedLearning = new PerceptronLearning(activationNetwork); var supervisedLearning = new BackPropagationLearning(activationNetwork); supervisedLearning.LearningRate = 0.5; int indice = 0; int treinamento = -1; var entrada = new List <double[]>(); var saida = new List <double[]>(); foreach (var modelo in _classes) { modelo.Value.SaidaEsperada = ObterSaidaEsperada(indice++); int contadorTreinamento = 0; foreach (var caracteristicas in modelo.Value.ObterCaracteristicas(msSegmentos)) { entrada.Add(caracteristicas); saida.Add(modelo.Value.SaidaEsperada); //var erroAbsoluto = supervisedLearning.Run(caracteristicas, modelo.Value.SaidaEsperada); if (treinamento == contadorTreinamento++) { break; } } } var entradaArray = entrada.ToArray(); var saidaArray = saida.ToArray(); return(supervisedLearning.RunEpoch(entradaArray, saidaArray)); //var resultado = supervisedLearning.RunEpoch(entradaArray, saidaArray); //Testar(activationNetwork, msSegmentos, @".\Audios\Giovanni", @".\Audios\Giovanni\audio_02.wav"); //Testar(activationNetwork, msSegmentos, @".\Audios\Giovanni", @".\Audios\Giovanni\audio_03.wav"); //Testar(activationNetwork, msSegmentos, @".\Audios\Sidney", @".\Audios\Sidney\audio_03.wav"); //Testar(activationNetwork, msSegmentos, @".\Audios\Sidney", @".\Audios\Sidney\audio_04.wav"); //Testar(activationNetwork, msSegmentos, @".\Audios\Wellington", @".\Audios\Wellington\audio_03.wav"); }
public NeuralNetworkChromosome(ActivationNetwork activationNetwork) { this._activationNetwork = activationNetwork; this._genes = new double[CountNumberOfGenesInNeuralNetwork()]; for (int i = 0; i < _genes.Length; i++) { _genes[i] = GenerateGene(); } SetNeurons(); }
public AnnAgent(bool learn, int boardSize, byte player = 1) { learning = learn; playerNumber = player; int boardFields = boardSize * boardSize; if(File.Exists("ann" + boardSize + ".bin")) network = (ActivationNetwork)Serialization.LoadNetwork("ann" + boardSize + ".bin"); else network = new ActivationNetwork(new BipolarSigmoidFunction(), boardFields, 5, boardFields * 2); backProp = new BackPropagationLearning(network); teacher = new MinimaxAgent(2, player); }
/// <summary> /// Initializes a new instance of the <see cref="EvolutionaryFitness"/> class. /// </summary> /// /// <param name="network">Neural network for which fitness will be calculated.</param> /// <param name="input">Input data samples for neural network.</param> /// <param name="output">Output data sampels for neural network (desired output).</param> /// /// <exception cref="ArgumentException">Length of inputs and outputs arrays must be equal and greater than 0.</exception> /// <exception cref="ArgumentException">Length of each input vector must be equal to neural network's inputs count.</exception> /// public EvolutionaryFitness( ActivationNetwork network, double[][] input, double[][] output ) { if ( ( input.Length == 0 ) || ( input.Length != output.Length ) ) { throw new ArgumentException( "Length of inputs and outputs arrays must be equal and greater than 0." ); } if ( network.InputsCount != input[0].Length ) { throw new ArgumentException( "Length of each input vector must be equal to neural network's inputs count." ); } this.network = network; this.input = input; this.output = output; }
public void RunEpochTest1() { Accord.Math.Tools.SetupGenerator(0); double[][] input = { new double[] { -1, -1 }, new double[] { -1, 1 }, new double[] { 1, -1 }, new double[] { 1, 1 } }; double[][] output = { new double[] { -1 }, new double[] { 1 }, new double[] { 1 }, new double[] { -1 } }; Neuron.RandGenerator = new ThreadSafeRandom(0); ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), 2, 2, 1); var teacher = new ParallelResilientBackpropagationLearning(network); double error = 1.0; while (error > 1e-5) error = teacher.RunEpoch(input, output); for (int i = 0; i < input.Length; i++) { double actual = network.Compute(input[i])[0]; double expected = output[i][0]; Assert.AreEqual(expected, actual, 0.01); Assert.IsFalse(Double.IsNaN(actual)); } }
public IForecastingModel TrainNewModel(double[][] iInput, double[][] iOutput) { int inputSize = iInput[0].Length, samplesNum = iOutput.Length; if (samplesNum != iInput.Length) throw new ArgumentException(); for (int i = 0; i < samplesNum;++i) if (iInput[i].Length != inputSize || iOutput[i].Length != 1) //iInput isn't a square matrix or iOutput isn't a vector throw new ArgumentException(); int[] neuronsCount = (int[]) ModelParametersDict[NeuronsInLayersKey]; string activationFunction = (string) ModelParametersDict[ActivationFunctionKey]; long maxIterNum = (long) ModelParametersDict[MaxIterationsNumberKey]; double stopError = (double)ModelParametersDict[StopErrorKey]; ActivationNetwork netToTrain = new ActivationNetwork(ActivationFunctionsDict[activationFunction], inputSize, neuronsCount); DataNormalizer normalizer = new DataNormalizer(iInput.Concat(iOutput).ToArray()); IForecastingModel aModel = new ANNforecastingModel(netToTrain, normalizer); ISupervisedLearning teacher = new ResilientBackpropagationLearning(netToTrain); double[][] trainInputSet, trainOutputSet; TrainingSubsetGenerator.GenerateRandomly(iInput, iOutput, out trainInputSet, out trainOutputSet, iMultiplier: TrainSubsetMultiplier); trainInputSet = normalizer.Normalize(trainInputSet); trainOutputSet = normalizer.Normalize(trainOutputSet); long epochsCount = 0; double nextError = ErrorCalculator.CalculateMSE(aModel, iInput, iOutput), prevError; do { prevError = nextError; teacher.RunEpoch(trainInputSet, trainOutputSet); nextError = ErrorCalculator.CalculateMSE(aModel, iInput, iOutput); } while (epochsCount++ <= maxIterNum && Math.Abs(prevError - nextError) >= stopError); return aModel; }
// Worker thread void SearchSolution( ) { // initialize input and output values double[][] input = null; double[][] output = null; if ( sigmoidType == 0 ) { // unipolar data input = new double[4][] { new double[] {0, 0}, new double[] {0, 1}, new double[] {1, 0}, new double[] {1, 1} }; output = new double[4][] { new double[] {0}, new double[] {1}, new double[] {1}, new double[] {0} }; } else { // bipolar data input = new double[4][] { new double[] {-1, -1}, new double[] {-1, 1}, new double[] { 1, -1}, new double[] { 1, 1} }; output = new double[4][] { new double[] {-1}, new double[] { 1}, new double[] { 1}, new double[] {-1} }; } // create neural network ActivationNetwork network = new ActivationNetwork( ( sigmoidType == 0 ) ? (IActivationFunction) new SigmoidFunction( sigmoidAlphaValue ) : (IActivationFunction) new BipolarSigmoidFunction( sigmoidAlphaValue ), 2, 2, 1 ); // create teacher var teacher = new ParallelResilientBackpropagationLearning(network); // set learning rate and momentum teacher.Reset(initialStep); // iterations int iteration = 0; // statistic files StreamWriter errorsFile = null; try { // check if we need to save statistics to files if ( saveStatisticsToFiles ) { // open files errorsFile = File.CreateText( "errors.csv" ); } // erros list ArrayList errorsList = new ArrayList( ); // loop while ( !needToStop ) { // run epoch of learning procedure double error = teacher.RunEpoch( input, output ); errorsList.Add( error ); // save current error if ( errorsFile != null ) { errorsFile.WriteLine( error ); } // show current iteration & error SetText( currentIterationBox, iteration.ToString( ) ); SetText( currentErrorBox, error.ToString( ) ); iteration++; // check if we need to stop if ( error <= learningErrorLimit ) break; } // show error's dynamics double[,] errors = new double[errorsList.Count, 2]; for ( int i = 0, n = errorsList.Count; i < n; i++ ) { errors[i, 0] = i; errors[i, 1] = (double) errorsList[i]; } errorChart.RangeX = new Range( 0, errorsList.Count - 1 ); errorChart.UpdateDataSeries( "error", errors ); } catch ( IOException ) { MessageBox.Show( "Failed writing file", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error ); } finally { // close files if ( errorsFile != null ) errorsFile.Close( ); } // enable settings controls EnableControls( true ); }
public void ConstructorTest() { // Four training samples of the xor function // two inputs (x and y) double[][] input = { new double[] { -1, -1 }, new double[] { -1, 1 }, new double[] { 1, -1 }, new double[] { 1, 1 } }; // one output (z = x ^ y) double[][] output = { new double[] { -1 }, new double[] { 1 }, new double[] { 1 }, new double[] { -1 } }; // create multi-layer neural network ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), // use a bipolar sigmoid activation function 2, // two inputs 3, // three hidden neurons 1 // one output neuron ); // create teacher LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning( network, // the neural network false, // whether or not to use Bayesian regularization JacobianMethod.ByBackpropagation // Jacobian calculation method ); // set learning rate and momentum teacher.LearningRate = 0.1f; // start the supervisioned learning for (int i = 0; i < 1000; i++) { double error = teacher.RunEpoch(input, output); } // If we reached here, the constructor test has passed. }
public void RunEpochTest4() { Accord.Math.Tools.SetupGenerator(0); double[][] input = { new double[] { 0, 0 }, }; double[][] output = { new double[] { 0 }, }; Neuron.RandGenerator = new ThreadSafeRandom(0); ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), 2, 1); var teacher = new LevenbergMarquardtLearning(network, true, JacobianMethod.ByBackpropagation); double error = 1.0; for (int i = 0; i < 1000; i++) error = teacher.RunEpoch(input, output); for (int i = 0; i < input.Length; i++) Assert.AreEqual(network.Compute(input[i])[0], output[i][0], 0.1); }
public void JacobianByChainRuleTest() { // Network with one hidden layer: 2-2-1 Accord.Math.Tools.SetupGenerator(0); double[][] input = { new double[] { -1, -1 }, new double[] { -1, 1 }, new double[] { 1, -1 }, new double[] { 1, 1 } }; double[][] output = { new double[] { -1 }, new double[] { 1 }, new double[] { 1 }, new double[] { -1 } }; Neuron.RandGenerator = new ThreadSafeRandom(0); ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), 2, 2, 1); var teacher1 = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByFiniteDifferences); var teacher2 = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByBackpropagation); // Set lambda to lambda max so no iterations are performed teacher1.LearningRate = 1e30f; teacher2.LearningRate = 1e30f; teacher1.RunEpoch(input, output); teacher2.RunEpoch(input, output); PrivateObject privateTeacher1 = new PrivateObject(teacher1); PrivateObject privateTeacher2 = new PrivateObject(teacher2); var jacobian1 = (float[][])privateTeacher1.GetField("jacobian"); var jacobian2 = (float[][])privateTeacher2.GetField("jacobian"); Assert.AreEqual(jacobian1[0][0], -0.47895513745387097, 1e-6); Assert.AreEqual(jacobian1[0][1], -0.05863886707282373, 1e-6); Assert.AreEqual(jacobian1[0][2], 0.057751100929897485, 1e-6); Assert.AreEqual(jacobian1[0][3], 0.0015185010717608583, 1e-6); Assert.AreEqual(jacobian1[7][0], -0.185400783651892, 1e-6); Assert.AreEqual(jacobian1[7][1], 0.025575161626462877, 1e-6); Assert.AreEqual(jacobian1[7][2], 0.070494677797224889, 1e-6); Assert.AreEqual(jacobian1[7][3], 0.037740463822781616, 1e-6); Assert.AreEqual(jacobian2[0][0], -0.4789595904719437, 1e-6); Assert.AreEqual(jacobian2[0][1], -0.058636153936941729, 1e-6); Assert.AreEqual(jacobian2[0][2], 0.057748435491340212, 1e-6); Assert.AreEqual(jacobian2[0][3], 0.0015184453425611988, 1e-6); Assert.AreEqual(jacobian2[7][0], -0.1854008206574258, 1e-6); Assert.AreEqual(jacobian2[7][1], 0.025575150379247645, 1e-6); Assert.AreEqual(jacobian2[7][2], 0.070494269423259301, 1e-6); Assert.AreEqual(jacobian2[7][3], 0.037740117733922635, 1e-6); for (int i = 0; i < jacobian1.Length; i++) { for (int j = 0; j < jacobian1[i].Length; j++) { double j1 = jacobian1[i][j]; double j2 = jacobian2[i][j]; Assert.AreEqual(j1, j2, 1e-4); Assert.IsFalse(Double.IsNaN(j1)); Assert.IsFalse(Double.IsNaN(j2)); } } }
public void BlockHessianTest1() { // Network with no hidden layers: 3-1 Accord.Math.Tools.SetupGenerator(0); double[][] input = { new double[] {-1, -1 }, new double[] {-1, 1 }, new double[] { 1, -1 }, new double[] { 1, 1 } }; double[][] output = { new double[] {-1 }, new double[] { 1 }, new double[] { 1 }, new double[] {-1 } }; Neuron.RandGenerator = new ThreadSafeRandom(0); ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), 2, 1); var teacher1 = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByFiniteDifferences); var teacher2 = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByBackpropagation); teacher2.Blocks = 2; // Set lambda to lambda max so no iterations are performed teacher1.LearningRate = 1e30f; teacher2.LearningRate = 1e30f; teacher1.RunEpoch(input, output); teacher2.RunEpoch(input, output); var hessian1 = teacher1.Hessian; var hessian2 = teacher1.Hessian; for (int i = 0; i < hessian1.Length; i++) { for (int j = 0; j < hessian1[i].Length; j++) { double j1 = hessian1[i][j]; double j2 = hessian2[i][j]; Assert.AreEqual(j1, j2, 1e-4); Assert.IsFalse(Double.IsNaN(j1)); Assert.IsFalse(Double.IsNaN(j2)); } } Assert.IsTrue(hessian1.IsUpperTriangular()); Assert.IsTrue(hessian2.IsUpperTriangular()); var gradient1 = teacher1.Gradient; var gradient2 = teacher2.Gradient; for (int i = 0; i < gradient1.Length; i++) { double j1 = gradient1[i]; double j2 = gradient2[i]; Assert.AreEqual(j1, j2, 1e-5); Assert.IsFalse(Double.IsNaN(j1)); Assert.IsFalse(Double.IsNaN(j2)); } }
/// <summary> /// Initializes a new instance of the <see cref="EvolutionaryLearning"/> class. /// </summary> /// /// <param name="activationNetwork">Activation network to be trained.</param> /// <param name="populationSize">Size of genetic population.</param> /// <param name="chromosomeGenerator">Random numbers generator used for initialization of genetic /// population representing neural network's weights and thresholds (see <see cref="DoubleArrayChromosome.chromosomeGenerator"/>).</param> /// <param name="mutationMultiplierGenerator">Random numbers generator used to generate random /// factors for multiplication of network's weights and thresholds during genetic mutation /// (ses <see cref="DoubleArrayChromosome.mutationMultiplierGenerator"/>.)</param> /// <param name="mutationAdditionGenerator">Random numbers generator used to generate random /// values added to neural network's weights and thresholds during genetic mutation /// (see <see cref="DoubleArrayChromosome.mutationAdditionGenerator"/>).</param> /// <param name="selectionMethod">Method of selection best chromosomes in genetic population.</param> /// <param name="crossOverRate">Crossover rate in genetic population (see /// <see cref="Population.CrossoverRate"/>).</param> /// <param name="mutationRate">Mutation rate in genetic population (see /// <see cref="Population.MutationRate"/>).</param> /// <param name="randomSelectionRate">Rate of injection of random chromosomes during selection /// in genetic population (see <see cref="Population.RandomSelectionPortion"/>).</param> /// public EvolutionaryLearning( ActivationNetwork activationNetwork, int populationSize, IRandomNumberGenerator chromosomeGenerator, IRandomNumberGenerator mutationMultiplierGenerator, IRandomNumberGenerator mutationAdditionGenerator, ISelectionMethod selectionMethod, double crossOverRate, double mutationRate, double randomSelectionRate ) { // Check of assumptions during debugging only Debug.Assert( activationNetwork != null ); Debug.Assert( populationSize > 0 ); Debug.Assert( chromosomeGenerator != null ); Debug.Assert( mutationMultiplierGenerator != null ); Debug.Assert( mutationAdditionGenerator != null ); Debug.Assert( selectionMethod != null ); Debug.Assert( crossOverRate >= 0.0 && crossOverRate <= 1.0 ); Debug.Assert( mutationRate >= 0.0 && crossOverRate <= 1.0 ); Debug.Assert( randomSelectionRate >= 0.0 && randomSelectionRate <= 1.0 ); // networks's parameters this.network = activationNetwork; this.numberOfNetworksWeights = CalculateNetworkSize( activationNetwork ); // population parameters this.populationSize = populationSize; this.chromosomeGenerator = chromosomeGenerator; this.mutationMultiplierGenerator = mutationMultiplierGenerator; this.mutationAdditionGenerator = mutationAdditionGenerator; this.selectionMethod = selectionMethod; this.crossOverRate = crossOverRate; this.mutationRate = mutationRate; this.randomSelectionRate = randomSelectionRate; }
// Worker thread void SearchSolution() { // number of learning samples int samples = sourceMatrix.GetLength(0); // prepare learning data double[][] inputs = sourceMatrix.Submatrix(null, 0, 1).ToArray(); double[][] outputs = sourceMatrix.GetColumn(2).Transpose().ToArray(); // create multi-layer neural network ann = new ActivationNetwork(new GaussianFunction(sigmoidAlphaValue), //new BipolarSigmoidFunction(sigmoidAlphaValue), 2, neuronsInFirstLayer, 1); if (useNguyenWidrow) { if (useSameWeights) Accord.Math.Tools.SetupGenerator(0); NguyenWidrow initializer = new NguyenWidrow(ann); initializer.Randomize(); } // create teacher LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(ann, useRegularization); // set learning rate and momentum teacher.LearningRate = learningRate; // iterations iteration = 1; var ranges = Matrix.Range(sourceMatrix, 0); double[][] map = Matrix.Mesh(ranges[0], ranges[1], 1, 1); var sw = Stopwatch.StartNew(); // loop while (!needToStop) { // run epoch of learning procedure error = teacher.RunEpoch(inputs, outputs) / samples; //var result = map.Apply(ann.Compute).GetColumn(0).Apply(Math.Sign); var result = map.Apply(ann.Compute).GetColumn(0).Apply(d => { if (d > 0.66) return 2; else if (d < 0.33) return 0; return 1; }); var graph = map.ToMatrix().InsertColumn(result.ToDouble()); CreateScatterplot(zedGraphControl2, graph); // increase current iteration iteration++; elapsed = sw.Elapsed; updateStatus(); // check if we need to stop if ((iterations != 0) && (iteration > iterations)) break; } sw.Stop(); // enable settings controls EnableControls(true); }
private static double computeError(double[][] inputs, double[][] outputs, ActivationNetwork ann) { // Compute the machine outputs int miss = 0; for (int i = 0; i < inputs.Length; i++) { var y = System.Math.Sign(ann.Compute(inputs[i])[0]); var o = outputs[i][0]; if (y != o) miss++; } return (double)miss / inputs.Length; }
// Worker thread void SearchSolution() { // number of learning samples int samples = data.GetLength(0); // data transformation factor double yFactor = 1.7 / chart.RangeY.Length; double yMin = chart.RangeY.Min; double xFactor = 2.0 / chart.RangeX.Length; double xMin = chart.RangeX.Min; // prepare learning data double[][] input = new double[samples][]; double[][] output = new double[samples][]; for (int i = 0; i < samples; i++) { input[i] = new double[1]; output[i] = new double[1]; // set input input[i][0] = (data[i, 0] - xMin) * xFactor - 1.0; // set output output[i][0] = (data[i, 1] - yMin) * yFactor - 0.85; } // create multi-layer neural network ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(sigmoidAlphaValue), 1, neuronsInFirstLayer, 1); if (useNguyenWidrow) { NguyenWidrow initializer = new NguyenWidrow(network); initializer.Randomize(); } // create teacher var teacher = new ParallelResilientBackpropagationLearning(network); // iterations int iteration = 1; // solution array double[,] solution = new double[50, 2]; double[] networkInput = new double[1]; // calculate X values to be used with solution function for (int j = 0; j < 50; j++) { solution[j, 0] = chart.RangeX.Min + (double)j * chart.RangeX.Length / 49; } // loop while (!needToStop) { // run epoch of learning procedure double error = teacher.RunEpoch(input, output) / samples; // calculate solution for (int j = 0; j < 50; j++) { networkInput[0] = (solution[j, 0] - xMin) * xFactor - 1.0; solution[j, 1] = (network.Compute(networkInput)[0] + 0.85) / yFactor + yMin; } chart.UpdateDataSeries("solution", solution); // calculate error double learningError = 0.0; for (int j = 0, k = data.GetLength(0); j < k; j++) { networkInput[0] = input[j][0]; learningError += Math.Abs(data[j, 1] - ((network.Compute(networkInput)[0] + 0.85) / yFactor + yMin)); } // set current iteration's info SetText(currentIterationBox, iteration.ToString()); SetText(currentErrorBox, learningError.ToString("F3")); // increase current iteration iteration++; // check if we need to stop if ((iterations != 0) && (iteration > iterations)) break; } // enable settings controls EnableControls(true); }
public BinarySplitForm() { InitializeComponent(); watch = new Stopwatch(); backgroundWorkerTrainer.Disposed += backgroundWorkerTrainer_Disposed; backgroundWorkerTrainer.DoWork += backgroundWorkerTrainer_DoWork; backgroundWorkerTrainer.ProgressChanged += backgroundWorkerTrainer_ProgressChanged; backgroundWorkerTrainer.WorkerSupportsCancellation = true; backgroundWorkerTrainer.WorkerReportsProgress = true; saveFileDialog1.Filter = "feed forward network files (*.ffn)|*.ffn"; saveFileDialog1.Title = "Save neural networkfile"; saveFileDialog1.InitialDirectory = null; saveFileDialog1.FileName = null; openFileDialog1.Filter = "feed forward network files (*.ffn)|*.ffn"; openFileDialog1.Title = "Load neural network file"; openFileDialog1.InitialDirectory = null; openFileDialog1.FileName = null; backgroundWorkerSignal.Disposed += backgroundWorkerSignal_Disposed; backgroundWorkerSignal.WorkerSupportsCancellation = true; backgroundWorkerSignal.WorkerReportsProgress = true; backgroundWorkerSignal.DoWork += backgroundWorkerSignal_DoWork; backgroundWorkerSignal.RunWorkerCompleted += backgroundWorkerSignal_RunWorkerCompleted; // initialize input and output values input = new double[4][] { new double[] {0, 0}, new double[] {0, 1}, new double[] {1, 0}, new double[] {1, 1} }; output = new double[4][] { new double[] {0}, new double[] {1}, new double[] {1}, new double[] {0} }; network = new ActivationNetwork(new SigmoidFunction(2), 2, 2, 1); teacher = new BackPropagationLearning(network); //logg.Show(); // pane used to draw your chart myPane = new GraphPane(); // poing pair lists listPointsOne = new PointPairList(); }
// Worker thread void SearchSolution() { // number of learning samples int samples = data.GetLength(0); // prepare learning data DoubleRange unit = new DoubleRange(-1, 1); double[][] input = Tools.Scale(from: xRange, to: unit, x: data.GetColumn(0)).ToArray(); double[][] output = Tools.Scale(from: yRange, to: unit, x: data.GetColumn(1)).ToArray(); // create multi-layer neural network ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(sigmoidAlphaValue), 1, neuronsInFirstLayer, 1); if (useNguyenWidrow) { new NguyenWidrow(network).Randomize(); } // create teacher var teacher = new ParallelResilientBackpropagationLearning(network); // iterations int iteration = 1; // solution array double[,] solution = new double[samples, 2]; // loop while (!needToStop) { // run epoch of learning procedure double error = teacher.RunEpoch(input, output) / samples; // calculate solution for (int j = 0; j < samples; j++) { double x = input[j][0]; double y = network.Compute(new[] { x })[0]; solution[j, 0] = Tools.Scale(from: unit, to: xRange, x: x); solution[j, 1] = Tools.Scale(from: unit, to: yRange, x: y); } chart.UpdateDataSeries("solution", solution); // calculate error double learningError = 0.0; for (int j = 0; j < samples; j++) { double x = input[j][0]; double expected = data[j, 1]; double actual = network.Compute(new[] { x })[0]; learningError += Math.Abs(expected - actual); } // set current iteration's info SetText(currentIterationBox, iteration.ToString()); SetText(currentErrorBox, learningError.ToString("F3")); // increase current iteration iteration++; // check if we need to stop if ((iterations != 0) && (iteration > iterations)) break; } // enable settings controls EnableControls(true); }
public void JacobianByChainRuleTest4() { // Network with no hidden layers: 3-1 double[][] input = { new double[] {-1, -1 }, new double[] {-1, 1 }, new double[] { 1, -1 }, new double[] { 1, 1 } }; double[][] output = { new double[] {-1 }, new double[] { 1 }, new double[] { 1 }, new double[] {-1 } }; Neuron.RandGenerator = new ThreadSafeRandom(0); ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), 2, 1); var teacher1 = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByFiniteDifferences); var teacher2 = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByBackpropagation); // Set lambda to lambda max so no iterations are performed teacher1.LearningRate = 1e30f; teacher2.LearningRate = 1e30f; teacher1.RunEpoch(input, output); teacher2.RunEpoch(input, output); PrivateObject privateTeacher1 = new PrivateObject(teacher1); PrivateObject privateTeacher2 = new PrivateObject(teacher2); var jacobian1 = (float[][])privateTeacher1.GetField("jacobian"); var jacobian2 = (float[][])privateTeacher2.GetField("jacobian"); for (int i = 0; i < jacobian1.Length; i++) { for (int j = 0; j < jacobian1[i].Length; j++) { double j1 = jacobian1[i][j]; double j2 = jacobian2[i][j]; Assert.AreEqual(j1, j2, 1e-5); Assert.IsFalse(Double.IsNaN(j1)); Assert.IsFalse(Double.IsNaN(j2)); } } }
public void JacobianByChainRuleTest_MultipleOutput() { // Network with no hidden layers: 3-4 int numberOfInputs = 3; int numberOfClasses = 4; double[][] input = { new double[] { -1, -1, -1 }, // 0 new double[] { -1, 1, -1 }, // 1 new double[] { 1, -1, -1 }, // 1 new double[] { 1, 1, -1 }, // 0 new double[] { -1, -1, 1 }, // 2 new double[] { -1, 1, 1 }, // 3 new double[] { 1, -1, 1 }, // 3 new double[] { 1, 1, 1 } // 2 }; int[] labels = { 0, 1, 1, 0, 2, 3, 3, 2, }; double[][] output = Accord.Statistics.Tools .Expand(labels, numberOfClasses, -1, 1); Neuron.RandGenerator = new ThreadSafeRandom(0); ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), numberOfInputs, numberOfClasses); var teacher1 = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByFiniteDifferences); var teacher2 = new LevenbergMarquardtLearning(network, false, JacobianMethod.ByBackpropagation); // Set lambda to lambda max so no iterations are performed teacher1.LearningRate = 1e30f; teacher2.LearningRate = 1e30f; teacher1.RunEpoch(input, output); teacher2.RunEpoch(input, output); PrivateObject privateTeacher1 = new PrivateObject(teacher1); PrivateObject privateTeacher2 = new PrivateObject(teacher2); var jacobian1 = (float[][])privateTeacher1.GetField("jacobian"); var jacobian2 = (float[][])privateTeacher2.GetField("jacobian"); for (int i = 0; i < jacobian1.Length; i++) { for (int j = 0; j < jacobian1[i].Length; j++) { double j1 = jacobian1[i][j]; double j2 = jacobian2[i][j]; Assert.AreEqual(j1, j2, 1e-3); Assert.IsFalse(Double.IsNaN(j1)); Assert.IsFalse(Double.IsNaN(j2)); } } }
/// <summary> /// Initializes a new instance of the <see cref="EvolutionaryLearning"/> class. /// </summary> /// /// <param name="activationNetwork">Activation network to be trained.</param> /// <param name="populationSize">Size of genetic population.</param> /// /// <remarks><para>This version of constructor is used to create genetic population /// for searching optimal neural network's weight using default set of parameters, which are: /// <list type="bullet"> /// <item>Selection method - elite;</item> /// <item>Crossover rate - 0.75;</item> /// <item>Mutation rate - 0.25;</item> /// <item>Rate of injection of random chromosomes during selection - 0.20;</item> /// <item>Random numbers generator for initializing new chromosome - /// <c>UniformGenerator( new Range( -1, 1 ) )</c>;</item> /// <item>Random numbers generator used during mutation for genes' multiplication - /// <c>ExponentialGenerator( 1 )</c>;</item> /// <item>Random numbers generator used during mutation for adding random value to genes - /// <c>UniformGenerator( new Range( -0.5f, 0.5f ) )</c>.</item> /// </list></para> /// /// <para>In order to have full control over the above default parameters, it is possible to /// used extended version of constructor, which allows to specify all of the parameters.</para> /// </remarks> /// public EvolutionaryLearning( ActivationNetwork activationNetwork, int populationSize ) { // Check of assumptions during debugging only Debug.Assert( activationNetwork != null ); Debug.Assert( populationSize > 0 ); // networks's parameters this.network = activationNetwork; this.numberOfNetworksWeights = CalculateNetworkSize( activationNetwork ); // population parameters this.populationSize = populationSize; this.chromosomeGenerator = new UniformGenerator( new Range( -1, 1 ) ); this.mutationMultiplierGenerator = new ExponentialGenerator( 1 ); this.mutationAdditionGenerator = new UniformGenerator( new Range( -0.5f, 0.5f ) ); this.selectionMethod = new EliteSelection( ); this.crossOverRate = 0.75; this.mutationRate = 0.25; this.randomSelectionRate = 0.2; }
public void ZeroLambdaTest() { double[,] data = null; // open selected file using (TextReader stream = new StringReader(Properties.Resources.ZeroLambda)) using (CsvReader reader = new CsvReader(stream, false)) { data = reader.ToTable().ToMatrix(); } // number of learning samples int samples = data.GetLength(0); var ranges = data.Range(dimension: 0); Assert.AreEqual(2, ranges.Length); var rangeX = ranges[0]; var rangeY = ranges[1]; // data transformation factor double yFactor = 1.7 / rangeY.Length; double yMin = rangeY.Min; double xFactor = 2.0 / rangeX.Length; double xMin = rangeX.Min; // prepare learning data double[][] input = new double[samples][]; double[][] output = new double[samples][]; for (int i = 0; i < samples; i++) { input[i] = new double[1]; output[i] = new double[1]; input[i][0] = (data[i, 0] - xMin) * xFactor - 1.0; // set input output[i][0] = (data[i, 1] - yMin) * yFactor - 0.85; // set output } // create multi-layer neural network ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(5), 1, 12, 1); // create teacher LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, true); teacher.LearningRate = 1; // iterations int iteration = 1; int iterations = 2000; // solution array double[,] solution = new double[samples, 2]; double[] networkInput = new double[1]; bool needToStop = false; double learningError = 0; // loop while (!needToStop) { Assert.AreNotEqual(0, teacher.LearningRate); // run epoch of learning procedure double error = teacher.RunEpoch(input, output) / samples; // calculate solution for (int j = 0; j < samples; j++) { networkInput[0] = (solution[j, 0] - xMin) * xFactor - 1.0; solution[j, 1] = (network.Compute(networkInput)[0] + 0.85) / yFactor + yMin; } // calculate error learningError = 0.0; for (int j = 0; j < samples; j++) { networkInput[0] = input[j][0]; learningError += Math.Abs(data[j, 1] - ((network.Compute(networkInput)[0] + 0.85) / yFactor + yMin)); } // increase current iteration iteration++; // check if we need to stop if ((iterations != 0) && (iteration > iterations)) break; } Assert.IsTrue(learningError < 0.13); }
// Create and initialize genetic population private int CalculateNetworkSize( ActivationNetwork activationNetwork ) { // caclculate total amount of weight in neural network int networkSize = 0; for ( int i = 0; i < network.Layers.Length; i++ ) { Layer layer = network.Layers[i]; for ( int j = 0; j < layer.Neurons.Length; j++ ) { // sum all weights and threshold networkSize += layer.Neurons[j].Weights.Length + 1; } } return networkSize; }
private void loadFileToolStripMenuItem_Click(object sender, EventArgs e) { var r = openFileDialog1.ShowDialog(); if (r != System.Windows.Forms.DialogResult.Cancel) { try { var n = Network.Load(openFileDialog1.FileName); network = n as ActivationNetwork; } catch (Exception eg) { MessageBox.Show("Error occured!"); } } }
public void RunEpochTest3() { double[,] dataset = yinyang; double[][] input = dataset.GetColumns(0, 1).ToArray(); double[][] output = dataset.GetColumn(2).ToArray(); Neuron.RandGenerator = new ThreadSafeRandom(0); ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), 2, 5, 1); var teacher = new LevenbergMarquardtLearning(network, true, JacobianMethod.ByBackpropagation); Assert.IsTrue(teacher.UseRegularization); double error = 1.0; for (int i = 0; i < 500; i++) error = teacher.RunEpoch(input, output); double[][] actual = new double[output.Length][]; for (int i = 0; i < input.Length; i++) actual[i] = network.Compute(input[i]); for (int i = 0; i < input.Length; i++) Assert.AreEqual(Math.Sign(output[i][0]), Math.Sign(actual[i][0])); }
// Create and initialize genetic population private int CalculateNetworkSize( ActivationNetwork activationNetwork ) { // caclculate total amount of weight in neural network int networkSize = 0; for ( int i = 0, layersCount = network.LayersCount; i < layersCount; i++ ) { ActivationLayer layer = network[i]; for ( int j = 0, neuronsCount = layer.NeuronsCount; j < neuronsCount; j++ ) { // sum all weights and threshold networkSize += layer[j].InputsCount + 1; } } return networkSize; }
/// <summary> /// Initializes a new instance of the <see cref="ResilientBackpropagationLearning"/> class. /// </summary> /// /// <param name="network">Network to teach.</param> /// public ResilientBackpropagationLearning( ActivationNetwork network ) { this.network = network; int layersCount = network.Layers.Length; neuronErrors = new double[layersCount][]; weightsDerivatives = new double[layersCount][][]; thresholdsDerivatives = new double[layersCount][]; weightsPreviousDerivatives = new double[layersCount][][]; thresholdsPreviousDerivatives = new double[layersCount][]; weightsUpdates = new double[layersCount][][]; thresholdsUpdates = new double[layersCount][]; // initialize errors, derivatives and steps for ( int i = 0; i < network.Layers.Length; i++ ) { Layer layer = network.Layers[i]; int neuronsCount = layer.Neurons.Length; neuronErrors[i] = new double[neuronsCount]; weightsDerivatives[i] = new double[neuronsCount][]; weightsPreviousDerivatives[i] = new double[neuronsCount][]; weightsUpdates[i] = new double[neuronsCount][]; thresholdsDerivatives[i] = new double[neuronsCount]; thresholdsPreviousDerivatives[i] = new double[neuronsCount]; thresholdsUpdates[i] = new double[neuronsCount]; // for each neuron for ( int j = 0; j < layer.Neurons.Length; j++ ) { weightsDerivatives[i][j] = new double[layer.InputsCount]; weightsPreviousDerivatives[i][j] = new double[layer.InputsCount]; weightsUpdates[i][j] = new double[layer.InputsCount]; } } // intialize steps ResetUpdates( learningRate ); }