public void processError(FeedForwardNeuralNetwork network, Vector error)
{
// TODO calculate total error somewhere
// create Sensitivity Matrices
outputSensitivity.sensitivityMatrixFromErrorMatrix(error);
hiddenSensitivity
.sensitivityMatrixFromSucceedingLayer(outputSensitivity);
// calculate weight Updates
calculateWeightUpdates(outputSensitivity, hiddenLayer
.getLastActivationValues(), learningRate, momentum);
calculateWeightUpdates(hiddenSensitivity, hiddenLayer
.getLastInputValues(), learningRate, momentum);
// calculate Bias Updates
calculateBiasUpdates(outputSensitivity, learningRate, momentum);
calculateBiasUpdates(hiddenSensitivity, learningRate, momentum);
// update weightsAndBiases
outputLayer.updateWeights();
outputLayer.updateBiases();
hiddenLayer.updateWeights();
hiddenLayer.updateBiases();
}
public Vector processInput(FeedForwardNeuralNetwork network, Vector input)
{
hiddenLayer.feedForward(input);
outputLayer.feedForward(hiddenLayer.getLastActivationValues());
return outputLayer.getLastActivationValues();
}
public Matrix sensitivityMatrixFromErrorMatrix(Vector errorVector) {
Matrix derivativeMatrix = createDerivativeMatrix(layer
.getLastInducedField());
Matrix calculatedSensitivityMatrix = derivativeMatrix
.times(errorVector).times(-2.0);
sensitivityMatrix = calculatedSensitivityMatrix.copy();
return calculatedSensitivityMatrix;
}
//
// PRIVATE METHODS
//
private Matrix createDerivativeMatrix(Vector lastInducedField) {
List<Double> lst = new List<Double>();
for (int i = 0; i < lastInducedField.size(); i++) {
lst.Add(layer.getActivationFunction().deriv(
lastInducedField.getValue(i)));
}
return Matrix.createDiagonalMatrix(lst);
}
public bool isCorrect(Vector prediction)
{
/*
* compares the index having greatest value in target to indec having
* greatest value in prediction. Ifidentical, correct
*/
return getTarget().indexHavingMaxValue() == prediction
.indexHavingMaxValue();
}
public void processError(Vector error)
{
Matrix weightUpdate = error.times(lastInput.transpose());
layer.acceptNewWeightUpdate(weightUpdate);
Vector biasUpdate = layer.getBiasVector().plus(error);
layer.acceptNewBiasUpdate(biasUpdate);
}
public Vector plus(Vector v)
{
Vector result = new Vector(size());
for (int i = 0; i < size(); i++)
{
result.setValue(i, getValue(i) + v.getValue(i));
}
return result;
}
public Vector copyVector()
{
Vector result = new Vector(getRowDimension());
for (int i = 0; i < getRowDimension(); i++)
{
result.setValue(i, getValue(i));
}
return result;
}
/*
* ONLY for testing to set up a network with known weights in future use to
* deserialize networks after adding variables for pen weightupdate,
* lastnput etc
*/
public FeedForwardNeuralNetwork(Matrix hiddenLayerWeights,
Vector hiddenLayerBias, Matrix outputLayerWeights,
Vector outputLayerBias) {
hiddenLayer = new Layer(hiddenLayerWeights, hiddenLayerBias,
new LogSigActivationFunction());
outputLayer = new Layer(outputLayerWeights, outputLayerBias,
new PureLinearActivationFunction());
}
public Matrix calculateWeightUpdates(LayerSensitivity layerSensitivity,
Vector previousLayerActivationOrInput, double alpha, double momentum)
{
Layer layer = layerSensitivity.getLayer();
Matrix activationTranspose = previousLayerActivationOrInput.transpose();
Matrix momentumLessUpdate = layerSensitivity.getSensitivityMatrix()
.times(activationTranspose).times(alpha).times(-1.0);
Matrix updateWithMomentum = layer.getLastWeightUpdateMatrix().times(
momentum).plus(momentumLessUpdate.times(1.0 - momentum));
layer.acceptNewWeightUpdate(updateWithMomentum.copy());
return updateWithMomentum;
}
public Layer(Matrix weightMatrix, Vector biasVector, ActivationFunction af) {
activationFunction = af;
this.weightMatrix = weightMatrix;
lastWeightUpdateMatrix = new Matrix(weightMatrix.getRowDimension(),
weightMatrix.getColumnDimension());
penultimateWeightUpdateMatrix = new Matrix(weightMatrix
.getRowDimension(), weightMatrix.getColumnDimension());
this.biasVector = biasVector;
lastBiasUpdateVector = new Vector(biasVector.getRowDimension());
penultimateBiasUpdateVector = new Vector(biasVector.getRowDimension());
}
public void testFeedForwardAndBAckLoopWorksWithMomentum()
{
// example 11.14 of Neural Network Design by Hagan, Demuth and Beale
Matrix hiddenLayerWeightMatrix = new Matrix(2, 1);
hiddenLayerWeightMatrix.set(0, 0, -0.27);
hiddenLayerWeightMatrix.set(1, 0, -0.41);
Vector hiddenLayerBiasVector = new Vector(2);
hiddenLayerBiasVector.setValue(0, -0.48);
hiddenLayerBiasVector.setValue(1, -0.13);
Vector input = new Vector(1);
input.setValue(0, 1);
Matrix outputLayerWeightMatrix = new Matrix(1, 2);
outputLayerWeightMatrix.set(0, 0, 0.09);
outputLayerWeightMatrix.set(0, 1, -0.17);
Vector outputLayerBiasVector = new Vector(1);
outputLayerBiasVector.setValue(0, 0.48);
Vector error = new Vector(1);
error.setValue(0, 1.261);
double learningRate = 0.1;
double momentumFactor = 0.5;
FeedForwardNeuralNetwork ffnn = new FeedForwardNeuralNetwork(
hiddenLayerWeightMatrix, hiddenLayerBiasVector,
outputLayerWeightMatrix, outputLayerBiasVector);
ffnn.setTrainingScheme(new BackPropLearning(learningRate,
momentumFactor));
ffnn.processInput(input);
ffnn.processError(error);
Matrix finalHiddenLayerWeights = ffnn.getHiddenLayerWeights();
Assert.AreEqual(-0.2675, finalHiddenLayerWeights.get(0, 0), 0.001);
Assert.AreEqual(-0.4149, finalHiddenLayerWeights.get(1, 0), 0.001);
Vector hiddenLayerBias = ffnn.getHiddenLayerBias();
Assert.AreEqual(-0.4775, hiddenLayerBias.getValue(0), 0.001);
Assert.AreEqual(-0.1349, hiddenLayerBias.getValue(1), 0.001);
Matrix finalOutputLayerWeights = ffnn.getOutputLayerWeights();
Assert.AreEqual(0.1304, finalOutputLayerWeights.get(0, 0), 0.001);
Assert.AreEqual(-0.1235, finalOutputLayerWeights.get(0, 1), 0.001);
Vector outputLayerBias = ffnn.getOutputLayerBias();
Assert.AreEqual(0.6061, outputLayerBias.getValue(0), 0.001);
}
public Layer(int numberOfNeurons, int numberOfInputs,
double lowerLimitForWeights, double upperLimitForWeights,
ActivationFunction af) {
activationFunction = af;
this.weightMatrix = new Matrix(numberOfNeurons, numberOfInputs);
lastWeightUpdateMatrix = new Matrix(weightMatrix.getRowDimension(),
weightMatrix.getColumnDimension());
penultimateWeightUpdateMatrix = new Matrix(weightMatrix
.getRowDimension(), weightMatrix.getColumnDimension());
this.biasVector = new Vector(numberOfNeurons);
lastBiasUpdateVector = new Vector(biasVector.getRowDimension());
penultimateBiasUpdateVector = new Vector(biasVector.getRowDimension());
initializeMatrix(weightMatrix, lowerLimitForWeights,
upperLimitForWeights);
initializeVector(biasVector, lowerLimitForWeights, upperLimitForWeights);
}
public Vector feedForward(Vector inputVector) {
lastInput = inputVector;
Matrix inducedField = weightMatrix.times(inputVector).plus(biasVector);
Vector inducedFieldVector = new Vector(numberOfNeurons());
for (int i = 0; i < numberOfNeurons(); i++) {
inducedFieldVector.setValue(i, inducedField.get(i, 0));
}
lastInducedField = inducedFieldVector.copyVector();
Vector resultVector = new Vector(numberOfNeurons());
for (int i = 0; i < numberOfNeurons(); i++) {
resultVector.setValue(i, activationFunction
.activation(inducedFieldVector.getValue(i)));
}
// set the result as the last activation value
lastActivationValues = resultVector.copyVector();
return resultVector;
}
public void testFeedForward()
{
// example 11.14 of Neural Network Design by Hagan, Demuth and Beale
// lots of tedious tests necessary to ensure nn is fundamentally correct
Matrix weightMatrix1 = new Matrix(2, 1);
weightMatrix1.set(0, 0, -0.27);
weightMatrix1.set(1, 0, -0.41);
Vector biasVector1 = new Vector(2);
biasVector1.setValue(0, -0.48);
biasVector1.setValue(1, -0.13);
Layer layer1 = new Layer(weightMatrix1, biasVector1,
new LogSigActivationFunction());
Vector inputVector1 = new Vector(1);
inputVector1.setValue(0, 1);
Vector expected = new Vector(2);
expected.setValue(0, 0.321);
expected.setValue(1, 0.368);
Vector result1 = layer1.feedForward(inputVector1);
Assert.AreEqual(expected.getValue(0), result1.getValue(0), 0.001);
Assert.AreEqual(expected.getValue(1), result1.getValue(1), 0.001);
Matrix weightMatrix2 = new Matrix(1, 2);
weightMatrix2.set(0, 0, 0.09);
weightMatrix2.set(0, 1, -0.17);
Vector biasVector2 = new Vector(1);
biasVector2.setValue(0, 0.48);
Layer layer2 = new Layer(weightMatrix2, biasVector2,
new PureLinearActivationFunction());
Vector inputVector2 = layer1.getLastActivationValues();
Vector result2 = layer2.feedForward(inputVector2);
Assert.AreEqual(0.446, result2.getValue(0), 0.001);
}
public void testSensitivityMatrixCalculationFromErrorVector()
{
Matrix weightMatrix1 = new Matrix(2, 1);
weightMatrix1.set(0, 0, -0.27);
weightMatrix1.set(1, 0, -0.41);
Vector biasVector1 = new Vector(2);
biasVector1.setValue(0, -0.48);
biasVector1.setValue(1, -0.13);
Layer layer1 = new Layer(weightMatrix1, biasVector1,
new LogSigActivationFunction());
Vector inputVector1 = new Vector(1);
inputVector1.setValue(0, 1);
layer1.feedForward(inputVector1);
Matrix weightMatrix2 = new Matrix(1, 2);
weightMatrix2.set(0, 0, 0.09);
weightMatrix2.set(0, 1, -0.17);
Vector biasVector2 = new Vector(1);
biasVector2.setValue(0, 0.48);
Layer layer2 = new Layer(weightMatrix2, biasVector2,
new PureLinearActivationFunction());
Vector inputVector2 = layer1.getLastActivationValues();
layer2.feedForward(inputVector2);
Vector errorVector = new Vector(1);
errorVector.setValue(0, 1.261);
LayerSensitivity layer2Sensitivity = new LayerSensitivity(layer2);
layer2Sensitivity.sensitivityMatrixFromErrorMatrix(errorVector);
Matrix sensitivityMatrix = layer2Sensitivity.getSensitivityMatrix();
Assert.AreEqual(-2.522, sensitivityMatrix.get(0, 0), 0.0001);
}
public void acceptNewBiasUpdate(Vector biasUpdate) {
setPenultimateBiasUpdateVector(getLastBiasUpdateVector());
setLastBiasUpdateVector(biasUpdate);
}
private static void initializeVector(Vector aVector, double lowerLimit,
double upperLimit) {
for (int i = 0; i < aVector.size(); i++) {
double random = Util.generateRandomDoubleBetween(lowerLimit,
upperLimit);
aVector.setValue(i, random);
}
}
public Vector processInput(Vector input)
{
lastInput = input;
return layer.feedForward(input);
}
public void processError(Vector error) {
trainingScheme.processError(this, error);
}
public Vector calculateBiasUpdates(LayerSensitivity layerSensitivity,
double alpha, double momentum)
{
Layer layer = layerSensitivity.getLayer();
Matrix biasUpdateMatrixWithoutMomentum = layerSensitivity
.getSensitivityMatrix().times(alpha).times(-1.0);
Matrix biasUpdateMatrixWithMomentum = layer.getLastBiasUpdateVector()
.times(momentum).plus(
biasUpdateMatrixWithoutMomentum.times(1.0 - momentum));
Vector result = new Vector(biasUpdateMatrixWithMomentum
.getRowDimension());
for (int i = 0; i < biasUpdateMatrixWithMomentum.getRowDimension(); i++)
{
result.setValue(i, biasUpdateMatrixWithMomentum.get(i, 0));
}
layer.acceptNewBiasUpdate(result.copyVector());
return result;
}
public void setLastBiasUpdateVector(Vector v) {
lastBiasUpdateVector = v;
}
public void testWeightsAndBiasesUpdatedCorrectly()
{
Matrix weightMatrix1 = new Matrix(2, 1);
weightMatrix1.set(0, 0, -0.27);
weightMatrix1.set(1, 0, -0.41);
Vector biasVector1 = new Vector(2);
biasVector1.setValue(0, -0.48);
biasVector1.setValue(1, -0.13);
Layer layer1 = new Layer(weightMatrix1, biasVector1,
new LogSigActivationFunction());
LayerSensitivity layer1Sensitivity = new LayerSensitivity(layer1);
Vector inputVector1 = new Vector(1);
inputVector1.setValue(0, 1);
layer1.feedForward(inputVector1);
Matrix weightMatrix2 = new Matrix(1, 2);
weightMatrix2.set(0, 0, 0.09);
weightMatrix2.set(0, 1, -0.17);
Vector biasVector2 = new Vector(1);
biasVector2.setValue(0, 0.48);
Layer layer2 = new Layer(weightMatrix2, biasVector2,
new PureLinearActivationFunction());
Vector inputVector2 = layer1.getLastActivationValues();
layer2.feedForward(inputVector2);
Vector errorVector = new Vector(1);
errorVector.setValue(0, 1.261);
LayerSensitivity layer2Sensitivity = new LayerSensitivity(layer2);
layer2Sensitivity.sensitivityMatrixFromErrorMatrix(errorVector);
layer1Sensitivity
.sensitivityMatrixFromSucceedingLayer(layer2Sensitivity);
BackPropLearning.calculateWeightUpdates(layer2Sensitivity, layer1
.getLastActivationValues(), 0.1);
BackPropLearning.calculateBiasUpdates(layer2Sensitivity, 0.1);
BackPropLearning.calculateWeightUpdates(layer1Sensitivity,
inputVector1, 0.1);
BackPropLearning.calculateBiasUpdates(layer1Sensitivity, 0.1);
layer2.updateWeights();
Matrix newWeightMatrix2 = layer2.getWeightMatrix();
Assert.AreEqual(0.171, newWeightMatrix2.get(0, 0), 0.001);
Assert.AreEqual(-0.0772, newWeightMatrix2.get(0, 1), 0.001);
layer2.updateBiases();
Vector newBiasVector2 = layer2.getBiasVector();
Assert.AreEqual(0.7322, newBiasVector2.getValue(0), 0.00001);
layer1.updateWeights();
Matrix newWeightMatrix1 = layer1.getWeightMatrix();
Assert.AreEqual(-0.265, newWeightMatrix1.get(0, 0), 0.001);
Assert.AreEqual(-0.419, newWeightMatrix1.get(1, 0), 0.001);
layer1.updateBiases();
Vector newBiasVector1 = layer1.getBiasVector();
Assert.AreEqual(-0.475, newBiasVector1.getValue(0), 0.001);
Assert.AreEqual(-0.139, newBiasVector1.getValue(1), 0.001);
}
public void testBiasUpdateMatrixesFormedCorrectly()
{
Matrix weightMatrix1 = new Matrix(2, 1);
weightMatrix1.set(0, 0, -0.27);
weightMatrix1.set(1, 0, -0.41);
Vector biasVector1 = new Vector(2);
biasVector1.setValue(0, -0.48);
biasVector1.setValue(1, -0.13);
Layer layer1 = new Layer(weightMatrix1, biasVector1,
new LogSigActivationFunction());
LayerSensitivity layer1Sensitivity = new LayerSensitivity(layer1);
Vector inputVector1 = new Vector(1);
inputVector1.setValue(0, 1);
layer1.feedForward(inputVector1);
Matrix weightMatrix2 = new Matrix(1, 2);
weightMatrix2.set(0, 0, 0.09);
weightMatrix2.set(0, 1, -0.17);
Vector biasVector2 = new Vector(1);
biasVector2.setValue(0, 0.48);
Layer layer2 = new Layer(weightMatrix2, biasVector2,
new PureLinearActivationFunction());
LayerSensitivity layer2Sensitivity = new LayerSensitivity(layer2);
Vector inputVector2 = layer1.getLastActivationValues();
layer2.feedForward(inputVector2);
Vector errorVector = new Vector(1);
errorVector.setValue(0, 1.261);
layer2Sensitivity.sensitivityMatrixFromErrorMatrix(errorVector);
layer1Sensitivity
.sensitivityMatrixFromSucceedingLayer(layer2Sensitivity);
Vector biasUpdateVector2 = BackPropLearning.calculateBiasUpdates(
layer2Sensitivity, 0.1);
Assert.AreEqual(0.2522, biasUpdateVector2.getValue(0), 0.001);
Vector lastBiasUpdateVector2 = layer2.getLastBiasUpdateVector();
Assert.AreEqual(0.2522, lastBiasUpdateVector2.getValue(0), 0.001);
Vector penultimateBiasUpdateVector2 = layer2
.getPenultimateBiasUpdateVector();
Assert.AreEqual(0.0, penultimateBiasUpdateVector2.getValue(0),
0.001);
Vector biasUpdateVector1 = BackPropLearning.calculateBiasUpdates(
layer1Sensitivity, 0.1);
Assert.AreEqual(0.00495, biasUpdateVector1.getValue(0), 0.001);
Assert.AreEqual(-0.00997, biasUpdateVector1.getValue(1), 0.001);
Vector lastBiasUpdateVector1 = layer1.getLastBiasUpdateVector();
Assert.AreEqual(0.00495, lastBiasUpdateVector1.getValue(0), 0.001);
Assert.AreEqual(-0.00997, lastBiasUpdateVector1.getValue(1), 0.001);
Vector penultimateBiasUpdateVector1 = layer1
.getPenultimateBiasUpdateVector();
Assert.AreEqual(0.0, penultimateBiasUpdateVector1.getValue(0),
0.001);
Assert.AreEqual(0.0, penultimateBiasUpdateVector1.getValue(1),
0.001);
}
public Vector errorVectorFrom(Vector target) {
return target.minus(getLastActivationValues());
}
public void setPenultimateBiasUpdateVector(Vector v) {
penultimateBiasUpdateVector = v;
}
public Vector getTarget()
{
Vector v = new Vector(normalizedTarget);
return v;
}
public void updateBiases() {
Matrix biasMatrix = biasVector.plusEquals(lastBiasUpdateVector);
Vector result = new Vector(biasMatrix.getRowDimension());
for (int i = 0; i < biasMatrix.getRowDimension(); i++) {
result.setValue(i, biasMatrix.get(i, 0));
}
biasVector = result;
}
public Vector processInput(Vector input) {
return trainingScheme.processInput(this, input);
}
public Vector getInput()
{
Vector v = new Vector(normalizedInput);
return v;
}
