public override void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet, Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParameters = null) { //Func<Math.IActivatorFunction, IActivationFunction> to = (fun) => //{ // if (fun is Math.SigmoidFunction) // return new SigmoidFunction(); // if (fun is Math.IdentityFunction) // return new IdentityFunction(); // return new SigmoidFunction(); //}; //IList<ActivationLayer> layers = new List<ActivationLayer>(); //layers.Add(new ActivationLayer(neuralNetwork.HiddenWeights[0].RowCount, trainingSet.ColumnCount, to(neuralNetwork.Layers[0].Applier.ActivatorFunction))); //for (int i = 1; i < neuralNetwork.HiddenWeights.Count; i++) //{ // layers.Add(new ActivationLayer(neuralNetwork.HiddenWeights[i - 1].RowCount, trainingSet.ColumnCount, to(neuralNetwork.Layers[i].Applier.ActivatorFunction))); //} double maxError = 0.01, error = 5, lr = 0.01; int maxEpochs = 1000, epochs = 0; if (hyperParameters != null) { if (maxEpochs <= 0) { throw new ArgumentException("Max Epochs cannot be negative"); } if (maxError > 2 || maxError < 0) { throw new ArgumentException("Max error cannot be negative or very large"); } maxError = hyperParameters.MaxError; maxEpochs = hyperParameters.MaxEpochs; lr = hyperParameters.Lr; } ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(2), trainingSet.ColumnCount, neuralNetwork.Layers.Select(x => x.NeuronsNumber).ToArray()); LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, true) { LearningRate = lr }; TrainingErrorMessage message = new TrainingErrorMessage() { NeuralNetwork = neuralNetwork, TrainingSet = trainingSet, CrossValidationSet = crossValidationSet, TrainingSetOutput = trainingSetOutput, CrossValidationSetOutput = crossValidationSetOutput }; int iterations = 1; double[][] inputs = new double[trainingSet.RowCount][], crossInputs = new double[crossValidationSet.RowCount][], outputs = new double[trainingSetOutput.RowCount][], crossOutputs = new double[crossValidationSetOutput.RowCount][]; for (int i = 0; i < trainingSet.RowCount; i++) { inputs[i] = new double[trainingSet.ColumnCount]; for (int j = 0; j < trainingSet.ColumnCount; j++) { inputs[i][j] = trainingSet[i, j]; } } for (int i = 0; i < trainingSetOutput.RowCount; i++) { outputs[i] = new double[trainingSetOutput.ColumnCount]; for (int j = 0; j < trainingSet.ColumnCount; j++) { outputs[i][j] = trainingSetOutput[i, j]; } } for (int i = 0; i < crossValidationSet.RowCount; i++) { crossInputs[i] = new double[crossValidationSet.ColumnCount]; for (int j = 0; j < crossValidationSet.ColumnCount; j++) { crossInputs[i][j] = crossValidationSet[i, j]; } } for (int i = 0; i < crossValidationSetOutput.RowCount; i++) { crossOutputs[i] = new double[crossValidationSetOutput.ColumnCount]; for (int j = 0; j < crossValidationSetOutput.ColumnCount; j++) { crossOutputs[i][j] = crossValidationSetOutput[i, j]; } } while (error > maxError && iterations++ <= maxEpochs) { message.Epochs = iterations; error = teacher.RunEpoch(inputs, outputs); message.TrainError = error; message.CrossError = teacher.ComputeError(crossInputs, crossOutputs); base.Notify(message); } //double mue = 0.001, mue_adj = 10, max_mue = 1e10; //base.Notify(message); //double currentError = message.Error; //while (currentError >= maxError && epochs++ < maxEpochs) //{ // message.Epochs = epochs; // var temp = HissienAndGragient(neuralNetwork, trainingSet, trainingSetOutput); // var hessien = temp.Item1; // var gradient = temp.Item2; // Matrix<double> blendingMatrix = Matrix<double>.Build.DenseDiagonal(hessien.RowCount, hessien.ColumnCount); // var prevW = neuralNetwork.HiddenWeights.ToList(); // //Console.WriteLine("prev :"); // //prevW.ForEach(Console.WriteLine); // double nextError = 100000; // while (true) // { // var term = hessien + mue*blendingMatrix; // var det = term.Determinant(); // if (System.Math.Abs(det) > 0) // { // var deltaW = term*gradient; // neuralNetwork.UpdateWeightsFromVector(deltaW); // //Console.WriteLine("updated :"); // //neuralNetwork.HiddenWeights.ForEach(Console.WriteLine); // base.Notify(message); // nextError = message.Error; // } // if (!(System.Math.Abs(det) > 0) || nextError >= currentError) // { // neuralNetwork.SetWeights(prevW); // //Console.WriteLine("set to prev :"); // //neuralNetwork.HiddenWeights.ForEach(Console.WriteLine); // mue *= mue_adj; // if (mue > max_mue) // { // mue = max_mue; // break; // } // } // else // { // mue /= mue_adj; // currentError = nextError; // //Console.WriteLine("the shit is here"); // break; // } // } //} }
public override void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet, Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParameters = null) { double maxError = 0.01, error = 5, momentum = 0.9; int maxEpochs = 1000, epochs = 0; if (hyperParameters != null) { if (maxEpochs <= 0) { throw new ArgumentException("Max Epochs cannot be negative"); } if (maxError > 2 || maxError < 0) { throw new ArgumentException("Max error cannot be negative or very large"); } maxError = hyperParameters.MaxError; maxEpochs = hyperParameters.MaxEpochs; momentum = hyperParameters.Momentum; } TrainingErrorMessage message = new TrainingErrorMessage() { NeuralNetwork = neuralNetwork, TrainingSet = trainingSet, CrossValidationSet = crossValidationSet, TrainingSetOutput = trainingSetOutput, CrossValidationSetOutput = crossValidationSetOutput }; var layers = neuralNetwork.Layers; var weights = neuralNetwork.HiddenWeights; //for momentum List <Matrix <double> > prevDeltaW = new List <Matrix <double> >(); for (int i = 0; i < weights.Count; ++i) { prevDeltaW.Add(Matrix <double> .Build.Dense(layers[i + 1].NeuronsNumber, layers[i].NeuronsNumber + 1)); } //end //epochs++ => ++epochs while (error >= maxError && ++epochs <= maxEpochs) { prevDeltaW.ForEach(e => e.Clear()); for (int i = 0; i < trainingSet.RowCount; i++) { Vector <double> input = trainingSet.Row(i), output = trainingSetOutput.Column(i); var temp = neuralNetwork.ForwardInput(input); IList <Vector <double> > acs = temp.Item1, gs = temp.Item2; var D = (output - acs[acs.Count - 1]).PointwiseMultiply(gs[gs.Count - 1]).ToColumnMatrix(); // n(output) * 1 var deltaW = D * acs[acs.Count - 2].ToRowMatrix() * layers[layers.Count - 1].LearningRate; // (n(output) * 1) * ((n(output-1)+1) * 1)' = n(output) * (n(output-1)+1) //for momentum deltaW += computeAdditionalTerms(prevDeltaW[weights.Count - 1], momentum); prevDeltaW[weights.Count - 1] = deltaW; //end neuralNetwork.UpdateWeightsAt(deltaW, weights.Count - 1); for (int j = layers.Count - 2; j > 0; j--) { D = (weights[j].Transpose() * D).RemoveRow(0).PointwiseMultiply(gs[j - 1].ToColumnMatrix()); // (n(j+1) * (n(j)+1))' * (n(j+1) * 1) = (n(j)+1) * 1, then => (n(j) * 1) .* (n(j) * 1) deltaW = D * acs[j - 1].ToRowMatrix() * layers[j].LearningRate; // (n(j) * 1) * ((n(j-1)+1) * 1)' = n(j) * (n(j-1)+1) //for momentum deltaW += computeAdditionalTerms(prevDeltaW[j - 1], momentum); prevDeltaW[j - 1] = deltaW; //end neuralNetwork.UpdateWeightsAt(deltaW, j - 1); } } message.Epochs = epochs; base.Notify(message); error = message.Error; Console.WriteLine(error); } base.OnComplete(); }