public void OnNext(TrainingErrorMessage value)
{
NeuralNetwork.NeuralNetwork neuralNetwork = value.NeuralNetwork;
Matrix <double> trainingSet = value.TrainingSet,
trainingSetOutput = value.TrainingSetOutput,
crossValidationSet = value.CrossValidationSet,
crossValidationSetOutput = value.CrossValidationSetOutput;
//var error = trainingSet.EnumerateRows().Select(row =>
//{
// var temp = neuralNetwork.ForwardInput(row);
// return temp.Item1[temp.Item1.Count - 1];
//}).Zip(trainingSetOutput.EnumerateColumns(), (first, second) => computeError(first, second)).Sum();
//TrainingErrors.Add(error);
//CrossValidationErrors.Add(crossValidationSet.EnumerateRows().Select(row =>
//{
// var temp = neuralNetwork.ForwardInput(row);
// return temp.Item1[temp.Item1.Count - 1];
//}).Zip(crossValidationSetOutput.EnumerateColumns(), (first, second) => computeError(first, second)).Sum());
//value.Error = error;
if (_isLeven)
{
TrainingErrors.Add(value.TrainError);
CrossValidationErrors.Add(value.CrossError);
NotifyResults(new ResultMesssage(value.TrainError, value.CrossError, value.Epochs));
return;
}
/*Trainer as observer*/
double validationERR = 0, trainingERR = 0;
//int epochsNUM=0;
var Terror = trainingSet.EnumerateRows().Select(row =>
{
var temp = neuralNetwork.ForwardInput(row);
return(temp.Item1[temp.Item1.Count - 1]);
}).Zip(trainingSetOutput.EnumerateColumns(), (first, second) => computeError(first, second)).Sum();
TrainingErrors.Add(Terror);
trainingERR = Terror;
var Verror = crossValidationSet.EnumerateRows().Select(row =>
{
var temp = neuralNetwork.ForwardInput(row);
//epochsNUM = temp.Item1.Count;
return(temp.Item1[temp.Item1.Count - 1]);
}).Zip(crossValidationSetOutput.EnumerateColumns(), (first, second) => computeError(first, second)).Sum();
value.Error = Terror;
CrossValidationErrors.Add(Terror);
validationERR = Verror;
Console.WriteLine("{0}\t{1}\t*****{2}", value.Epochs, validationERR, trainingERR);
//Trainer as observer
ResultMesssage messsage = new ResultMesssage(trainingERR, validationERR, value.Epochs);
NotifyResults(messsage);
/*Trainer as observer*/
}
private Tuple <Matrix <double>, Vector <double> > HissienAndGragient(NeuralNetwork.NeuralNetwork nn, Matrix <double> trainingSet, Matrix <double> trainingSetOutput)
{
var weights = nn.HiddenWeights;
var layers = nn.Layers;
List <Matrix <double> > gradient2D = new List <Matrix <double> >();
for (int i = 0; i < weights.Count; ++i)
{
gradient2D.Add(Matrix <double> .Build.Dense(layers[i + 1].NeuronsNumber, layers[i].NeuronsNumber + 1));
}
Dictionary <int, List <double> > gradients = new Dictionary <int, List <double> >();
List <double> errors = new List <double>();
for (int i = 0; i < trainingSet.RowCount; i++)
{
//conpute gradient for one training example
Vector <double> input = trainingSet.Row(i),
output = trainingSetOutput.Column(i);
var temp = nn.ForwardInput(input);
IList <Vector <double> > acs = temp.Item1, gs = temp.Item2;
var D = (output - acs[acs.Count - 1]).PointwiseMultiply(gs[gs.Count - 1]).ToColumnMatrix();
var deltaW = D * acs[acs.Count - 2].ToRowMatrix();
gradient2D[weights.Count - 1] = deltaW;
for (int j = layers.Count - 2; j > 0; j--)
{
D = (nn.HiddenWeights[j].Transpose() * D).RemoveRow(0).PointwiseMultiply(gs[j - 1].ToColumnMatrix());
deltaW = D * acs[j - 1].ToRowMatrix();
gradient2D[j - 1] = deltaW;
}
//end
//put the gradient in 1D list
int m = i;
gradients.Add(m, new List <double>());
for (int j = 0; j < weights.Count; j++)
{
gradient2D[j].Enumerate().ForEach(e => gradients[m].Add(e));
}
//end
//compute error for one training example
var t = output - acs[acs.Count - 1];
errors.Add(t.PointwiseMultiply(t).Sum() / 2);
//end
}
//initialize jaccobian matrix
Matrix <double> jac = Matrix <double> .Build.Dense(trainingSet.RowCount, gradients[0].Count);
for (int i = 0; i < jac.RowCount; i++)
{
for (int k = 0; k < jac.ColumnCount; ++k)
{
jac[i, k] = gradients[i][k];
}
}
//end
//errors vector
var err = Vector <double> .Build.DenseOfEnumerable(errors.AsEnumerable());
return(new Tuple <Matrix <double>, Vector <double> >(jac.Transpose() * jac, jac.Transpose() * err));
}
public override void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet, Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParameters = null)
{
double maxError = 0.01, error = 5, momentum = 0.9;
int maxEpochs = 1000, epochs = 0;
if (hyperParameters != null)
{
if (maxEpochs <= 0)
{
throw new ArgumentException("Max Epochs cannot be negative");
}
if (maxError > 2 || maxError < 0)
{
throw new ArgumentException("Max error cannot be negative or very large");
}
maxError = hyperParameters.MaxError;
maxEpochs = hyperParameters.MaxEpochs;
momentum = hyperParameters.Momentum;
}
TrainingErrorMessage message = new TrainingErrorMessage()
{
NeuralNetwork = neuralNetwork, TrainingSet = trainingSet, CrossValidationSet = crossValidationSet, TrainingSetOutput = trainingSetOutput, CrossValidationSetOutput = crossValidationSetOutput
};
var layers = neuralNetwork.Layers;
var weights = neuralNetwork.HiddenWeights;
//for momentum
List <Matrix <double> > prevDeltaW = new List <Matrix <double> >();
for (int i = 0; i < weights.Count; ++i)
{
prevDeltaW.Add(Matrix <double> .Build.Dense(layers[i + 1].NeuronsNumber, layers[i].NeuronsNumber + 1));
}
//end
//epochs++ => ++epochs
while (error >= maxError && ++epochs <= maxEpochs)
{
prevDeltaW.ForEach(e => e.Clear());
for (int i = 0; i < trainingSet.RowCount; i++)
{
Vector <double> input = trainingSet.Row(i),
output = trainingSetOutput.Column(i);
var temp = neuralNetwork.ForwardInput(input);
IList <Vector <double> > acs = temp.Item1, gs = temp.Item2;
var D = (output - acs[acs.Count - 1]).PointwiseMultiply(gs[gs.Count - 1]).ToColumnMatrix(); // n(output) * 1
var deltaW = D * acs[acs.Count - 2].ToRowMatrix() * layers[layers.Count - 1].LearningRate; // (n(output) * 1) * ((n(output-1)+1) * 1)' = n(output) * (n(output-1)+1)
//for momentum
deltaW += computeAdditionalTerms(prevDeltaW[weights.Count - 1], momentum);
prevDeltaW[weights.Count - 1] = deltaW;
//end
neuralNetwork.UpdateWeightsAt(deltaW, weights.Count - 1);
for (int j = layers.Count - 2; j > 0; j--)
{
D = (weights[j].Transpose() * D).RemoveRow(0).PointwiseMultiply(gs[j - 1].ToColumnMatrix()); // (n(j+1) * (n(j)+1))' * (n(j+1) * 1) = (n(j)+1) * 1, then => (n(j) * 1) .* (n(j) * 1)
deltaW = D * acs[j - 1].ToRowMatrix() * layers[j].LearningRate; // (n(j) * 1) * ((n(j-1)+1) * 1)' = n(j) * (n(j-1)+1)
//for momentum
deltaW += computeAdditionalTerms(prevDeltaW[j - 1], momentum);
prevDeltaW[j - 1] = deltaW;
//end
neuralNetwork.UpdateWeightsAt(deltaW, j - 1);
}
}
message.Epochs = epochs;
base.Notify(message);
error = message.Error;
Console.WriteLine(error);
}
base.OnComplete();
}