public override void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet,
Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput,
HyperParameters hyperParameters = null)
{
//Func<Math.IActivatorFunction, IActivationFunction> to = (fun) =>
//{
// if (fun is Math.SigmoidFunction)
// return new SigmoidFunction();
// if (fun is Math.IdentityFunction)
// return new IdentityFunction();
// return new SigmoidFunction();
//};
//IList<ActivationLayer> layers = new List<ActivationLayer>();
//layers.Add(new ActivationLayer(neuralNetwork.HiddenWeights[0].RowCount, trainingSet.ColumnCount, to(neuralNetwork.Layers[0].Applier.ActivatorFunction)));
//for (int i = 1; i < neuralNetwork.HiddenWeights.Count; i++)
//{
// layers.Add(new ActivationLayer(neuralNetwork.HiddenWeights[i - 1].RowCount, trainingSet.ColumnCount, to(neuralNetwork.Layers[i].Applier.ActivatorFunction)));
//}
double maxError = 0.01, error = 5, lr = 0.01;
int maxEpochs = 1000, epochs = 0;
if (hyperParameters != null)
{
if (maxEpochs <= 0)
{
throw new ArgumentException("Max Epochs cannot be negative");
}
if (maxError > 2 || maxError < 0)
{
throw new ArgumentException("Max error cannot be negative or very large");
}
maxError = hyperParameters.MaxError;
maxEpochs = hyperParameters.MaxEpochs;
lr = hyperParameters.Lr;
}
ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(2), trainingSet.ColumnCount, neuralNetwork.Layers.Select(x => x.NeuronsNumber).ToArray());
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, true)
{
LearningRate = lr
};
TrainingErrorMessage message = new TrainingErrorMessage()
{
NeuralNetwork = neuralNetwork,
TrainingSet = trainingSet,
CrossValidationSet = crossValidationSet,
TrainingSetOutput = trainingSetOutput,
CrossValidationSetOutput = crossValidationSetOutput
};
int iterations = 1;
double[][] inputs = new double[trainingSet.RowCount][],
crossInputs = new double[crossValidationSet.RowCount][],
outputs = new double[trainingSetOutput.RowCount][],
crossOutputs = new double[crossValidationSetOutput.RowCount][];
for (int i = 0; i < trainingSet.RowCount; i++)
{
inputs[i] = new double[trainingSet.ColumnCount];
for (int j = 0; j < trainingSet.ColumnCount; j++)
{
inputs[i][j] = trainingSet[i, j];
}
}
for (int i = 0; i < trainingSetOutput.RowCount; i++)
{
outputs[i] = new double[trainingSetOutput.ColumnCount];
for (int j = 0; j < trainingSet.ColumnCount; j++)
{
outputs[i][j] = trainingSetOutput[i, j];
}
}
for (int i = 0; i < crossValidationSet.RowCount; i++)
{
crossInputs[i] = new double[crossValidationSet.ColumnCount];
for (int j = 0; j < crossValidationSet.ColumnCount; j++)
{
crossInputs[i][j] = crossValidationSet[i, j];
}
}
for (int i = 0; i < crossValidationSetOutput.RowCount; i++)
{
crossOutputs[i] = new double[crossValidationSetOutput.ColumnCount];
for (int j = 0; j < crossValidationSetOutput.ColumnCount; j++)
{
crossOutputs[i][j] = crossValidationSetOutput[i, j];
}
}
while (error > maxError && iterations++ <= maxEpochs)
{
message.Epochs = iterations;
error = teacher.RunEpoch(inputs, outputs);
message.TrainError = error;
message.CrossError = teacher.ComputeError(crossInputs, crossOutputs);
base.Notify(message);
}
//double mue = 0.001, mue_adj = 10, max_mue = 1e10;
//base.Notify(message);
//double currentError = message.Error;
//while (currentError >= maxError && epochs++ < maxEpochs)
//{
// message.Epochs = epochs;
// var temp = HissienAndGragient(neuralNetwork, trainingSet, trainingSetOutput);
// var hessien = temp.Item1;
// var gradient = temp.Item2;
// Matrix<double> blendingMatrix = Matrix<double>.Build.DenseDiagonal(hessien.RowCount, hessien.ColumnCount);
// var prevW = neuralNetwork.HiddenWeights.ToList();
// //Console.WriteLine("prev :");
// //prevW.ForEach(Console.WriteLine);
// double nextError = 100000;
// while (true)
// {
// var term = hessien + mue*blendingMatrix;
// var det = term.Determinant();
// if (System.Math.Abs(det) > 0)
// {
// var deltaW = term*gradient;
// neuralNetwork.UpdateWeightsFromVector(deltaW);
// //Console.WriteLine("updated :");
// //neuralNetwork.HiddenWeights.ForEach(Console.WriteLine);
// base.Notify(message);
// nextError = message.Error;
// }
// if (!(System.Math.Abs(det) > 0) || nextError >= currentError)
// {
// neuralNetwork.SetWeights(prevW);
// //Console.WriteLine("set to prev :");
// //neuralNetwork.HiddenWeights.ForEach(Console.WriteLine);
// mue *= mue_adj;
// if (mue > max_mue)
// {
// mue = max_mue;
// break;
// }
// }
// else
// {
// mue /= mue_adj;
// currentError = nextError;
// //Console.WriteLine("the shit is here");
// break;
// }
// }
//}
}
public override void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet, Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParameters = null)
{
double maxError = 0.01, error = 5, momentum = 0.9;
int maxEpochs = 1000, epochs = 0;
if (hyperParameters != null)
{
if (maxEpochs <= 0)
{
throw new ArgumentException("Max Epochs cannot be negative");
}
if (maxError > 2 || maxError < 0)
{
throw new ArgumentException("Max error cannot be negative or very large");
}
maxError = hyperParameters.MaxError;
maxEpochs = hyperParameters.MaxEpochs;
momentum = hyperParameters.Momentum;
}
TrainingErrorMessage message = new TrainingErrorMessage()
{
NeuralNetwork = neuralNetwork, TrainingSet = trainingSet, CrossValidationSet = crossValidationSet, TrainingSetOutput = trainingSetOutput, CrossValidationSetOutput = crossValidationSetOutput
};
var layers = neuralNetwork.Layers;
var weights = neuralNetwork.HiddenWeights;
//for momentum
List <Matrix <double> > prevDeltaW = new List <Matrix <double> >();
for (int i = 0; i < weights.Count; ++i)
{
prevDeltaW.Add(Matrix <double> .Build.Dense(layers[i + 1].NeuronsNumber, layers[i].NeuronsNumber + 1));
}
//end
//epochs++ => ++epochs
while (error >= maxError && ++epochs <= maxEpochs)
{
prevDeltaW.ForEach(e => e.Clear());
for (int i = 0; i < trainingSet.RowCount; i++)
{
Vector <double> input = trainingSet.Row(i),
output = trainingSetOutput.Column(i);
var temp = neuralNetwork.ForwardInput(input);
IList <Vector <double> > acs = temp.Item1, gs = temp.Item2;
var D = (output - acs[acs.Count - 1]).PointwiseMultiply(gs[gs.Count - 1]).ToColumnMatrix(); // n(output) * 1
var deltaW = D * acs[acs.Count - 2].ToRowMatrix() * layers[layers.Count - 1].LearningRate; // (n(output) * 1) * ((n(output-1)+1) * 1)' = n(output) * (n(output-1)+1)
//for momentum
deltaW += computeAdditionalTerms(prevDeltaW[weights.Count - 1], momentum);
prevDeltaW[weights.Count - 1] = deltaW;
//end
neuralNetwork.UpdateWeightsAt(deltaW, weights.Count - 1);
for (int j = layers.Count - 2; j > 0; j--)
{
D = (weights[j].Transpose() * D).RemoveRow(0).PointwiseMultiply(gs[j - 1].ToColumnMatrix()); // (n(j+1) * (n(j)+1))' * (n(j+1) * 1) = (n(j)+1) * 1, then => (n(j) * 1) .* (n(j) * 1)
deltaW = D * acs[j - 1].ToRowMatrix() * layers[j].LearningRate; // (n(j) * 1) * ((n(j-1)+1) * 1)' = n(j) * (n(j-1)+1)
//for momentum
deltaW += computeAdditionalTerms(prevDeltaW[j - 1], momentum);
prevDeltaW[j - 1] = deltaW;
//end
neuralNetwork.UpdateWeightsAt(deltaW, j - 1);
}
}
message.Epochs = epochs;
base.Notify(message);
error = message.Error;
Console.WriteLine(error);
}
base.OnComplete();
}