public void Update(Network network, TemporaryNetwork tempNet)
{
UpdateWeightsForHiddenLayer(network, tempNet);
UpdateBiasForHiddenLayer(network, tempNet);
UpdateWeightsForOutputLayer(network, tempNet);
UpdateBiasForOutputLayer(network, tempNet);
}
private List <Matrix <double> > CreateEmptyContenerForBiases(TemporaryNetwork tempNet)
{
return(new List <Matrix <double> >
{
_matrixBuilder.Dense(tempNet.ActiveNeuronInHiddenLayerCount(), 1),
_matrixBuilder.Dense(tempNet.OutputLayerCount(), 1)
});
}
private void UpdateBiasForOutputLayer(Network network, TemporaryNetwork tempNet)
{
for (var i = 0; i < tempNet.ActiveNeuronInHiddenLayerCount(); i++)
{
var biases = tempNet.GetBiasOnOutputLayer();
network.SetBiasesOnOutputLayer(biases);
}
}
private void UpdateWeightsForOutputLayer(Network network, TemporaryNetwork tempNet)
{
for (var i = 0; i < tempNet.ActiveNeuronInHiddenLayerCount(); i++)
{
var neuronLabel = tempNet.GetActiveNeuronLabel(i);
var weights = tempNet.GetWeightsForInputFromHiddenLayer(i);
network.SetWeightsForInputsFromHiddenLayer(neuronLabel, weights);
}
}
private void UpdateBiasForHiddenLayer(Network network, TemporaryNetwork tempNet)
{
for (var i = 0; i < tempNet.ActiveNeuronInHiddenLayerCount(); i++)
{
var neuronLabel = tempNet.GetActiveNeuronLabel(i);
var bias = tempNet.GetBiasForNeuronFromHiddenLayer(i);
network.SetBiasOnHiddenLayer(neuronLabel, bias);
}
}
private Delta Backprop(TemporaryNetwork tempNet, DigitImageForStuding singleBatch)
{
var nablaB = CreateEmptyContenerForBiases(tempNet);
var nablaW = CreateEmptyContenerForWeights(tempNet);
var activision = singleBatch.Pixels;
var activisions = new List <Matrix <double> > {
activision
};
//list to store all z vectors, layer by layer
var zs = new List <Matrix <double> >();
for (var i = 0; i < 2; i++)
{
var layerNumber = i + 2;
var z = tempNet.GetWeightsForNLayer(layerNumber).Multiply(activision) + tempNet.GetBiasOnNLayer(layerNumber);
zs.Add(z);
activision = z.Map(_activationFunction.Function);
activisions.Add(activision);
}
var part1 = CostDerivate(activisions[activisions.Count - 1], singleBatch.Label);
var part2 = zs[zs.Count - 1].Map(_activationFunction.Calcucus);
var delta = part1.PointwiseMultiply(part2);
nablaB[nablaB.Count - 1] = delta;
nablaW[nablaW.Count - 1] = delta.Multiply(activisions[activisions.Count - 2].Transpose());
for (var i = 2; i < _numLayers; i++)
{
var z = zs[zs.Count - i];
var sp = z.Map(_activationFunction.Calcucus);
var layerLabel = _weights.Count - i + 1 + 2;
delta = tempNet.GetWeightsForNLayer(layerLabel).Transpose().Multiply(delta).PointwiseMultiply(sp);
nablaB[nablaB.Count - i] = delta;
nablaW[nablaW.Count - i] = delta.Multiply(activisions[activisions.Count - i - 1].Transpose());
}
return(new Delta(nablaW, nablaB));
}
private void UpdateMiniBatch(TemporaryNetwork tempNet, List <DigitImageForStuding> batch, Parametrs param)
{
var nablaB = CreateEmptyContenerForBiases(tempNet);
var nablaW = CreateEmptyContenerForWeights(tempNet);
foreach (var singleBatch in batch)
{
var delta = Backprop(tempNet, singleBatch);
for (var i = 0; i < nablaB.Count; i++)
{
nablaB[i] = nablaB[i].Add(delta.DeltaBiases[i]);
}
for (var i = 0; i < nablaW.Count; i++)
{
nablaW[i] = nablaW[i].Add(delta.DeltaWeights[i]);
}
}
for (var i = 0; i < _weights.Count; i++)
{
var layerNumber = i + 2;
//var first = _weights[i].Subtract((eta/batch.Count)*nablaW[i]); dodano L2
var first = tempNet.GetWeightsForNLayer(layerNumber).Multiply(1 - param.Eta * (param.Lambda / param.NumberOfNeuronsInHiddenLayer)).Subtract((param.Eta / batch.Count) * nablaW[i]);
//var momentum = param.Beta*(_weights[i].Subtract(_previousWeights[i])); Momentum na razie idzie do kosza
tempNet.SetWeightsForNLayer(layerNumber, first);
}
for (var i = 0; i < _biases.Count; i++)
{
var layerNumber = i + 2;
var newBias = tempNet.GetBiasOnNLayer(layerNumber).Subtract((param.Eta / batch.Count) * nablaB[i]);
tempNet.SetBiasOnNLayer(layerNumber, newBias);
}
}
public TemporaryNetwork Build(Network mainNetwork)
{
var activeNeurons = _builder.BuildList(mainNetwork.HiddenLayerSize());
var tempNet = new TemporaryNetwork(activeNeurons);
var weightForHiddenLayer =
_adapter.CreateNewWeightsMatrixBeetwenInputAndHiddenLayers(
mainNetwork.GetWeightsBeetwenInputAndHiddenLayer(), activeNeurons);
var biasOnHiddenLayer =
_adapter.CreateNewBiasOnHiddenLayer(mainNetwork.GetBiasOnHiddenLayer(), activeNeurons);
var weightForOutputLayer =
_adapter.CreateNewWeightsMatrixBeetwenHiddenAndOutputLayer(
mainNetwork.GetWeightsBeetwenHiddenAndOutputLayer(), activeNeurons);
var biasOnOutputLayer = mainNetwork.GetBiasOnOutputLayer();
tempNet.SetWeightsBeetwenInputAndHiddenLayer(weightForHiddenLayer);
tempNet.SetBiasOnHiddenLayer(biasOnHiddenLayer);
tempNet.SetWeightsBeetwenHiddenAndOutputLayer(weightForOutputLayer);
tempNet.SetBiasOnOutputLayer(biasOnOutputLayer);
return(tempNet);
}
