public EvaluationModel Evaluate(InputModel[] testData)
{
var correctSolutions = 0;
for (var i = 0; i < testData.Length; i++)
{
var model = testData[i];
var decision = Compute(model.Values);
if (decision == model.Label)
{
correctSolutions++;
}
}
var result = new EvaluationModel
{
Correct = correctSolutions,
All = testData.Length
};
return(result);
}
public TrainingResult Train(TrainingModel trainingModel)
{
var trainingSet = trainingModel.TrainingSet;
var testSet = trainingModel.TestSet;
var validationSet = trainingModel.ValidationSet;
var errorTreshold = trainingModel.ErrorThreshold;
var maxEpochs = trainingModel.MaxEpochs;
var batchSize = Math.Min(trainingModel.BatchSize, trainingModel.TrainingSet.Length);
var learningRate = trainingModel.LearningRate;
var lambda = trainingModel.Lambda;
var momentum = trainingModel.Momentum;
var isEncoder = trainingModel.IsEncoder;
var takeBest = trainingModel.TakeBest;
var isVerbose = trainingModel.IsVerbose;
var evaluateOnEachEpoch = trainingModel.EvaluateOnEachEpoch;
//Debugger.Launch();
IList <double> epochErrors = new List <double>(maxEpochs);
var epochEvaluations = new List <EvaluationModel>(maxEpochs);
var errorSum = double.PositiveInfinity;
var epoch = 0;
var layersCount = Sizes.Length - 1;
#region create nablas arrays
var nablaWeights = new Matrix <double> [layersCount];
var nablaBiases = new Vector <double> [layersCount];
for (var i = 0; i < layersCount; i++)
{
var nextLayer = Layers[i];
nablaBiases[i] = nextLayer.GetNewBiasesVector(true);
nablaWeights[i] = nextLayer.GetNewWeightsMatrix(true);
}
#endregion
if (isVerbose)
{
var activationFunctions = Layers.Select(l => l.CurrentActivationFunction.ToString()).ToArray();
var distributions = Layers.Select(l => l.InitialWeightsRange.ToString("#0.00")).ToArray();
Console.WriteLine("Starting with params:");
Console.WriteLine($"\tsizes- {JsonConvert.SerializeObject(Sizes)}");
Console.WriteLine($"\tlearning rate - {learningRate}");
Console.WriteLine($"\tmomentum- {momentum}");
Console.WriteLine($"\terror threshold - {errorTreshold}");
Console.WriteLine($"\tmax epochs - {maxEpochs}");
Console.WriteLine($"\tactivation functions - {JsonConvert.SerializeObject(activationFunctions, Formatting.None)}");
Console.WriteLine($"\tinitial weights ranges- {JsonConvert.SerializeObject(distributions, Formatting.None)}");
}
InputModel sample = null;
if (isEncoder)
{
Directory.CreateDirectory("encoder_logs");
sample = trainingSet[DateTime.Now.Ticks % trainingSet.Length];
var recreatedImage = MnistViewer.ToImage(sample.Values, 7);
recreatedImage.Save($"encoder_logs/_original.png", ImageFormat.Png);
}
var prevWeightsChange = new Matrix <double> [layersCount];
var prevBiasChange = new Vector <double> [layersCount];
if (isVerbose && !isEncoder)
{
var initialPercentage = Evaluate(testSet).Percentage;
Console.WriteLine($"Initial state, {initialPercentage.ToString("#0.00")}");
}
#region log data
//log data
var path = "log.csv";
var log = new StringBuilder("sep=|");
log.AppendLine();
log.Append("epoch|evaluation_0|error_0");
log.AppendLine();
#endregion
string bestNNJson = null;
double bestError = double.PositiveInfinity;
while (errorSum > errorTreshold && epoch < maxEpochs)
{
epoch++;
errorSum = 0;
var batch = HelperFunctions.RandomPermutation(trainingSet).Take(batchSize).ToList();
//Vector<double> activationsSum = new DenseVector(_hiddenLayers[0].NeuronsCount);
//foreach (var item in batch)
//{
// var activation = item.Values;
// foreach (var layer in _hiddenLayers)
// {
// activation = layer.Feedforward(activation);
// }
// activationsSum += activation;
//}
//var avgActivations = activationsSum.Divide(batch.Count);
foreach (var item in batch)
{
//const double beta = 0.5;
//const double rho = 0.05;
//var divergence = beta * (-rho / avgActivations + (1 - rho) / (1 - avgActivations));
//var bpResult = Backpropagate(item.Values, item.ExpectedSolution, divergence);
var bpResult = Backpropagate(item.Values, item.ExpectedSolution);
for (var i = 0; i < nablaWeights.Length; i++)
{
nablaBiases[i] = bpResult.Biases[i] + nablaBiases[i];
nablaWeights[i] = bpResult.Weights[i] + nablaWeights[i];
}
var solution = bpResult.Solution;
var expectedSolution = item.ExpectedSolution;
errorSum += solution.Map2((y, o) => Math.Abs(y - o), expectedSolution).Sum();
}
errorSum /= batchSize;
#region update parameters
for (var i = 0; i < layersCount; i++)
{
var weightsChange = learningRate / batchSize * nablaWeights[i];
if (prevWeightsChange[i] != null)
{
weightsChange += momentum * prevWeightsChange[i];
}
//L2
var weights = Layers[i].Weights;
if (lambda != 0)
{
weights = (1 - learningRate * lambda / batchSize) * weights;
}
Layers[i].Weights = weights - weightsChange;
var biasesChange = learningRate / batchSize * nablaBiases[i];
if (prevBiasChange[i] != null)
{
biasesChange += momentum * prevBiasChange[i];
}
//L2
var biases = Layers[i].Biases;
if (lambda != 0)
{
biases = (1 - (learningRate * lambda) / batchSize) * biases;
}
Layers[i].Biases = biases - biasesChange;
prevWeightsChange[i] = weightsChange;
prevBiasChange[i] = biasesChange;
}
#endregion
EvaluationModel epochEvaluation = null;
if (evaluateOnEachEpoch)
{
epochEvaluation = Evaluate(testSet);
epochEvaluations.Add(epochEvaluation);
}
if (isVerbose)
{
var percentage = isEncoder
? 0
: (epochEvaluation ?? Evaluate(testSet)).Percentage;
Console.WriteLine($"Epoch - {epoch}," +
$" error - {errorSum.ToString("#0.000")}," +
$" test - {percentage.ToString("#0.00")}");
if (isEncoder)
{
var recreatedData = FeedForward(sample.Values);
var recreatedImage = MnistViewer.ToImage(recreatedData, 7);
recreatedImage.Save($"encoder_logs/{epoch}.png", ImageFormat.Png);
}
}
#region dump data
var eval = isEncoder
? 0
: (epochEvaluation ?? Evaluate(testSet)).Percentage;
log.AppendLine(epoch + "|" + eval + "|" + errorSum);
#endregion
#region set nablas to zeroes
for (var i = 0; i < layersCount; i++)
{
nablaBiases[i].Clear();
nablaWeights[i].Clear();
}
#endregion
epochErrors.Add(errorSum);
if (takeBest && errorSum < bestError)
{
bestNNJson = ToJson();
}
}
#region log data
File.WriteAllText(path, log.ToString());
#endregion
var trainingResult = new TrainingResult
{
NeuralNetwork = takeBest ? FromJson(bestNNJson) : this,
Epochs = epoch,
EpochErrors = epochErrors.ToArray(),
Evaluations = epochEvaluations.ToArray()
};
return(trainingResult);
}
