private void Mutate(FullyConnectedNeuralNetworkModel model)
{
var biasCount = model.BiasLayers.Sum(biasLayer => biasLayer.Length);
var weightCount = model.WeightLayers.Sum(weightLayer => weightLayer.Length);
var totalDnaCount = biasCount + weightCount;
var dnaToMutateCount = totalDnaCount * fractionOfChromosomeToMutate;
for (var mutationCount = 0; mutationCount < dnaToMutateCount; mutationCount++)
{
if (mutationCount + 1 > dnaToMutateCount && random.NextDouble() > dnaToMutateCount - mutationCount)
{
break;
}
var biasOrWeight = random.Next(totalDnaCount);
if (biasOrWeight < biasCount)
{
var layerIndex = random.Next(model.BiasLayers.Length);
var biasIndex = random.Next(model.BiasLayers[layerIndex].Length);
model.BiasLayers[layerIndex][biasIndex] = random.NextDouble();
}
else
{
var layerIndex = random.Next(model.WeightLayers.Length);
var weightInputIndex = random.Next(model.WeightLayers[layerIndex].GetLength(0));
var weightOutputIndex = random.Next(model.WeightLayers[layerIndex].GetLength(1));
model.WeightLayers[layerIndex][weightInputIndex, weightOutputIndex] = random.NextDouble();
}
}
}
public FullyConnectedNeuralNetworkModel[] CreateNextGeneration(FullyConnectedNeuralNetworkModel[] parents, int populationSize)
{
var children = new FullyConnectedNeuralNetworkModel[populationSize - parents.Length];
var indexesBred = new ConcurrentDictionary <int, List <int> >();
Parallel.For(0, children.Length, childIndex =>
{
var motherIndex = 0;
var fatherIndex = 0;
var motherHasChildren = false;
while (motherIndex == fatherIndex || motherHasChildren && indexesBred[motherIndex].Contains(fatherIndex))
{
var parentIndexes = new[] { random.Next(parents.Length), random.Next(parents.Length) }.OrderBy(i => i);
motherIndex = parentIndexes.ElementAt(0);
fatherIndex = parentIndexes.ElementAt(1);
motherHasChildren = indexesBred.ContainsKey(motherIndex);
}
children[childIndex] = modelBreeder.Breed(parents[motherIndex], parents[fatherIndex]);
indexesBred.AddOrUpdate(motherIndex, new List <int>()
{
fatherIndex
}, (key, value) =>
{
value.Add(fatherIndex);
return(value);
});
});
return(children);
}
private void ApplyLayerDeltas(InputOutputPairModel[] batch)
{
var newModel = new FullyConnectedNeuralNetworkModel
{
ActivationCountsPerLayer = Model.ActivationCountsPerLayer,
ActivationFunction = Model.ActivationFunction,
BiasLayers = new double[Model.BiasLayers.Length][],
WeightLayers = new double[Model.WeightLayers.Length][, ]
};
for (var layerIndex = 0; layerIndex < Model.WeightLayers.Length; layerIndex++)
{
var inputActivationCount = Model.WeightLayers[layerIndex].GetLength(0);
var outputActivationCount = Model.WeightLayers[layerIndex].GetLength(1);
newModel.WeightLayers[layerIndex] = new double[inputActivationCount, outputActivationCount];
newModel.BiasLayers[layerIndex] = new double[outputActivationCount];
for (var o = 0; o < outputActivationCount; o++)
{
for (var i = 0; i < inputActivationCount; i++)
{
var weightLayerDeltaAverage = layerWeightDeltaTotals[layerIndex][i, o] / batch.Length;
newModel.WeightLayers[layerIndex][i, o] = Model.WeightLayers[layerIndex][i, o] - weightLayerDeltaAverage;
}
var biasLayerDeltaAverage = layerBiasDeltaTotals[layerIndex][o] / batch.Length;
newModel.BiasLayers[layerIndex][o] = Model.BiasLayers[layerIndex][o] - biasLayerDeltaAverage;
}
}
Model = newModel;
}
private void PropagateBackwards(double[][] inputActivationLayers, FullyConnectedNeuralNetworkModel model, double[] deltaOutputActivations)
{
for (var layerIndex = inputActivationLayers.Length - 1; layerIndex >= 0; layerIndex--)
{
deltaOutputActivations = PropagateLayerBackwards(layerIndex, inputActivationLayers[layerIndex], model, deltaOutputActivations);
}
}
public void CorrectResultsAreReturnedWhenActivationFunctionIsNone()
{
// Arrange
var network = new FullyConnectedNeuralNetworkModel
{
BiasLayers = new[] { new[] { -0.84167747099030643, 0.66957177392652811, 0.78872177740033789 } },
WeightLayers = new[]
{
new[, ]
{
{ -0.554521067791861, 0.370098950979346, -0.79384722830441179 },
{ 0.60650056395982421, -0.95214650032676129, 0.67987006608390721 },
{ -0.35679294977187781, -0.81505192248851621, 0.14074044914019312 },
{ 0.60933753829884219, 0.11109885252597684, -0.59281794335358673 }
}
},
ActivationFunction = ActivationFunction.None
};
var activations = new[] { 0.34, 0.56, 0.78, 0.90 };
// Act
var results = sut.Execute(network, activations);
// Assert
results.Should().HaveCount(2);
results[0].Should().BeEquivalentTo(activations);
results[1].Should().BeEquivalentTo(new[] { -0.42046903457514428, -0.27354815519114406, 0.47578235809494851 });
}
private double[] FlattenWeightsAndBiases(FullyConnectedNeuralNetworkModel model)
{
var flatWeightsAndBiases = new List <double>();
flatWeightsAndBiases.AddRange(model.WeightLayers.SelectMany(l => l.Cast <double>()));
flatWeightsAndBiases.AddRange(model.BiasLayers.SelectMany(l => l));
return(flatWeightsAndBiases.ToArray());
}
public void BreedProducesAChildModelThatHasTheCorrectPercentageOfChromosomesFromEachParentAndMutations()
{
// Arrange
var activationCountsPerLayer = new[] { 5, 4, 3 };
var activationFunction = ActivationFunction.Sigmoid;
var sampleSize = 10000;
var mothers = new FullyConnectedNeuralNetworkModel[sampleSize].Select(m => CreateFullyConnectedNeuralNetworkModel(activationCountsPerLayer, activationFunction)).ToArray();
var fathers = new FullyConnectedNeuralNetworkModel[sampleSize].Select(m => CreateFullyConnectedNeuralNetworkModel(activationCountsPerLayer, activationFunction)).ToArray();
var totalMotherChromosomeCount = 0;
var totalFatherChromosomeCount = 0;
var totalMutationChromosomeCount = 0;
mockModelInitializer.Setup(m => m.CreateModel(It.IsAny <int[]>(), It.IsAny <ActivationFunction>(), It.IsAny <double[][]>(), It.IsAny <double[][, ]>()))
.Returns <int[], ActivationFunction, double[][], double[][, ]>((arg1, arg2, arg3, arg4) => new FullyConnectedNeuralNetworkModel
{
ActivationCountsPerLayer = arg1,
ActivationFunction = arg2,
BiasLayers = arg3,
WeightLayers = arg4
});
// Act
for (var i = 0; i < sampleSize; i++)
{
var child = sut.Breed(mothers[i], fathers[i]);
var childDna = FlattenWeightsAndBiases(child);
var motherDna = FlattenWeightsAndBiases(mothers[i]);
var fatherDna = FlattenWeightsAndBiases(fathers[i]);
for (var j = 0; j < motherDna.Length; j++)
{
if (childDna[j] == motherDna[j])
{
totalMotherChromosomeCount++;
}
else if (childDna[j] == fatherDna[j])
{
totalFatherChromosomeCount++;
}
else
{
totalMutationChromosomeCount++;
}
}
}
var totalChromosomeCount = (double)totalMotherChromosomeCount + totalFatherChromosomeCount + totalMutationChromosomeCount;
var motherChromosomePercentage = totalMotherChromosomeCount / totalChromosomeCount;
var fatherChromosomePercentage = totalFatherChromosomeCount / totalChromosomeCount;
var mutationChromosomePercentage = totalMutationChromosomeCount / totalChromosomeCount;
// Assert
mockModelInitializer.Verify(m => m.CreateModel(It.IsAny <int[]>(), It.IsAny <ActivationFunction>(), It.IsAny <double[][]>(), It.IsAny <double[][, ]>()), Times.Exactly(sampleSize));
motherChromosomePercentage.Should().BeInRange(0.4491, 0.5489); // should be around 0.499
fatherChromosomePercentage.Should().BeInRange(0.4491, 0.5489); // should be around 0.499
mutationChromosomePercentage.Should().BeInRange(0.0009, 0.0011); // should be around 0.001
}
public void Initialize(FullyConnectedNeuralNetworkModel model)
{
population[0] = new CostModel <FullyConnectedNeuralNetworkModel>(model);
for (var populationIndex = 1; populationIndex < populationSize; populationIndex++)
{
var member = modelInitializer.CreateModel(model.ActivationCountsPerLayer, model.ActivationFunction);
population[populationIndex] = new CostModel <FullyConnectedNeuralNetworkModel>(member);
}
}
private FullyConnectedNeuralNetworkModel[] CreateManyFullyConnectedNeuralNetworkModel(int count)
{
var models = new FullyConnectedNeuralNetworkModel[count];
for (var i = 0; i < count; i++)
{
models[i] = CreateFullyConnectedNeuralNetworkModel();
}
return(models);
}
public void InitializeSetsTheModelCorrectly()
{
// Arrange
var model = new FullyConnectedNeuralNetworkModel();
// Act
sut.Initialize(model);
// Assert
sut.Model.Should().BeSameAs(model);
}
public FullyConnectedNeuralNetworkModel Breed(FullyConnectedNeuralNetworkModel mother, FullyConnectedNeuralNetworkModel father)
{
var childBiasLayers = MergeBiasLayers(mother.BiasLayers, father.BiasLayers);
var childWeightLayers = MergeWeightLayers(mother.WeightLayers, father.WeightLayers);
var childModel = modelInitializer.CreateModel(mother.ActivationCountsPerLayer, mother.ActivationFunction, childBiasLayers, childWeightLayers);
var mutationChance = random.NextDouble();
if (mutationChance < mutationThreshold)
{
Mutate(childModel);
}
return(childModel);
}
public double[][] Execute(FullyConnectedNeuralNetworkModel model, double[] activations)
{
var layerCount = model.WeightLayers.Length;
var allActivations = new double[layerCount + 1][];
allActivations[0] = activations;
for (var layerIndex = 0; layerIndex < layerCount; layerIndex++)
{
allActivations[layerIndex + 1] = ExecuteLayer(model.BiasLayers[layerIndex], model.WeightLayers[layerIndex], allActivations[layerIndex], model.ActivationFunction);
}
return(allActivations);
}
public void SingleIterationOfLearnWithActivationFunctionReturnCorrectResult()
{
// Arrange
var activationFunction = ActivationFunction.Sigmoid;
var model = CreateFullyConnectedNeuralNetworkModel(activationFunction);
var batch = CreateBatch();
var expectedResult = new FullyConnectedNeuralNetworkModel
{
ActivationCountsPerLayer = new[] { 4, 3 },
ActivationFunction = activationFunction,
BiasLayers = new[]
{
new[] { -1.2943006116609224, 0.77069126468580773, 0.41209730715748505 }
},
WeightLayers = new[]
{
new[, ]
{
{ -0.80537379400399278, 0.41910423130779234, -0.97124654040321312 },
{ 0.40789139000539582, -0.89706697984869466, 0.46692747025766013 },
{ -0.7938561825726973, -0.74708704168834117, -0.0907913437911862 },
{ 0.536301613044516, 0.1534224913289616, -0.76938811941200691 }
}
}
};
mockExecuter.Setup(m => m.Execute(It.IsAny <FullyConnectedNeuralNetworkModel>(), It.Is <double[]>(it => it.Equals(batch[0].Inputs))))
.Returns <FullyConnectedNeuralNetworkModel, double[]>((arg1, arg2) => new double[][] { arg2, new[] { 0.23878872335077425, 0.41971501950211154, 0.66660062117031826 } });
mockExecuter.Setup(m => m.Execute(It.IsAny <FullyConnectedNeuralNetworkModel>(), It.Is <double[]>(it => it.Equals(batch[1].Inputs))))
.Returns <FullyConnectedNeuralNetworkModel, double[]>((arg1, arg2) => new double[][] { arg2, new[] { 0.51112549346515468, 0.53561880887592039, 0.61608786703901575 } });
mockSigmoidActivationFunction.Setup(m => m.CalculateDerivative(It.IsAny <double>()))
.Returns <double>(arg1 => 0.5 * arg1);
sut.Initialize(model);
// Act
sut.Learn(batch);
// Assert
mockExecuter.Verify(m => m.Execute(It.IsAny <FullyConnectedNeuralNetworkModel>(), It.Is <double[]>(it => it.Equals(batch[0].Inputs))), Times.Once);
mockExecuter.Verify(m => m.Execute(It.IsAny <FullyConnectedNeuralNetworkModel>(), It.Is <double[]>(it => it.Equals(batch[1].Inputs))), Times.Once);
mockSigmoidActivationFunction.Verify(m => m.CalculateDerivative(It.IsAny <double>()), Times.Exactly(model.BiasLayers.Sum(l => l.Length) * batch.Length));
sut.Model.Should().BeEquivalentTo(expectedResult);
}
public void SingleIterationOfLearnWithoutActivationFunctionReturnCorrectResult()
{
// Arrange
var activationFunction = ActivationFunction.None;
var model = CreateFullyConnectedNeuralNetworkModel(activationFunction);
var batch = CreateBatch();
var expectedResult = new FullyConnectedNeuralNetworkModel
{
ActivationCountsPerLayer = new[] { 4, 3 },
ActivationFunction = activationFunction,
BiasLayers = new[]
{
new[] { -1.7469237523315382, 0.87181075544508735, 0.035472836914632211 }
},
WeightLayers = new[]
{
new[, ]
{
{ -1.0562265202161245, 0.46810951163623871, -1.1486458525020145 },
{ 0.20928221605096742, -0.841987459370628, 0.2539848744314131 },
{ -1.2309194153735166, -0.67912216088816624, -0.32232313672256552 },
{ 0.46326568779018962, 0.19574613013194636, -0.9459582954704272 }
}
}
};
mockExecuter.Setup(m => m.Execute(It.IsAny <FullyConnectedNeuralNetworkModel>(), It.Is <double[]>(it => it.Equals(batch[0].Inputs))))
.Returns <FullyConnectedNeuralNetworkModel, double[]>((arg1, arg2) => new double[][] { arg2, new[] { 0.23878872335077425, 0.41971501950211154, 0.66660062117031826 } });
mockExecuter.Setup(m => m.Execute(It.IsAny <FullyConnectedNeuralNetworkModel>(), It.Is <double[]>(it => it.Equals(batch[1].Inputs))))
.Returns <FullyConnectedNeuralNetworkModel, double[]>((arg1, arg2) => new double[][] { arg2, new[] { 0.51112549346515468, 0.53561880887592039, 0.61608786703901575 } });
sut.Initialize(model);
// Act
sut.Learn(batch);
// Assert
mockExecuter.Verify(m => m.Execute(It.IsAny <FullyConnectedNeuralNetworkModel>(), It.Is <double[]>(it => it.Equals(batch[0].Inputs))), Times.Once);
mockExecuter.Verify(m => m.Execute(It.IsAny <FullyConnectedNeuralNetworkModel>(), It.Is <double[]>(it => it.Equals(batch[1].Inputs))), Times.Once);
sut.Model.Should().BeEquivalentTo(expectedResult);
}
private double[] PropagateLayerBackwards(int layerIndex, double[] inputActivations, FullyConnectedNeuralNetworkModel model, double[] deltaOutputActivations)
{
var outputActivationCount = deltaOutputActivations.Length;
var sigmoidDerivatives = new double[outputActivationCount];
for (var j = 0; j < outputActivationCount; j++)
{
sigmoidDerivatives[j] = Activation(inputActivations, model.WeightLayers[layerIndex], model.BiasLayers[layerIndex][j], model.ActivationFunction, j);
}
DeltaWeights(layerIndex, inputActivations, sigmoidDerivatives, deltaOutputActivations);
DeltaBiases(layerIndex, sigmoidDerivatives, deltaOutputActivations);
return(DeltaActivations(model.WeightLayers[layerIndex], sigmoidDerivatives, deltaOutputActivations));
}
public void Initialize(FullyConnectedNeuralNetworkModel model)
{
Model = model;
}
