public void processError(FeedForwardNeuralNetwork network, Vector error) { // TODO calculate total error somewhere // create Sensitivity Matrices outputSensitivity.sensitivityMatrixFromErrorMatrix(error); hiddenSensitivity .sensitivityMatrixFromSucceedingLayer(outputSensitivity); // calculate weight Updates calculateWeightUpdates(outputSensitivity, hiddenLayer .getLastActivationValues(), learningRate, momentum); calculateWeightUpdates(hiddenSensitivity, hiddenLayer .getLastInputValues(), learningRate, momentum); // calculate Bias Updates calculateBiasUpdates(outputSensitivity, learningRate, momentum); calculateBiasUpdates(hiddenSensitivity, learningRate, momentum); // update weightsAndBiases outputLayer.updateWeights(); outputLayer.updateBiases(); hiddenLayer.updateWeights(); hiddenLayer.updateBiases(); }
public Vector processInput(FeedForwardNeuralNetwork network, Vector input) { hiddenLayer.feedForward(input); outputLayer.feedForward(hiddenLayer.getLastActivationValues()); return outputLayer.getLastActivationValues(); }
public Matrix sensitivityMatrixFromErrorMatrix(Vector errorVector) { Matrix derivativeMatrix = createDerivativeMatrix(layer .getLastInducedField()); Matrix calculatedSensitivityMatrix = derivativeMatrix .times(errorVector).times(-2.0); sensitivityMatrix = calculatedSensitivityMatrix.copy(); return calculatedSensitivityMatrix; }
// // PRIVATE METHODS // private Matrix createDerivativeMatrix(Vector lastInducedField) { List<Double> lst = new List<Double>(); for (int i = 0; i < lastInducedField.size(); i++) { lst.Add(layer.getActivationFunction().deriv( lastInducedField.getValue(i))); } return Matrix.createDiagonalMatrix(lst); }
public bool isCorrect(Vector prediction) { /* * compares the index having greatest value in target to indec having * greatest value in prediction. Ifidentical, correct */ return getTarget().indexHavingMaxValue() == prediction .indexHavingMaxValue(); }
public void processError(Vector error) { Matrix weightUpdate = error.times(lastInput.transpose()); layer.acceptNewWeightUpdate(weightUpdate); Vector biasUpdate = layer.getBiasVector().plus(error); layer.acceptNewBiasUpdate(biasUpdate); }
public Vector plus(Vector v) { Vector result = new Vector(size()); for (int i = 0; i < size(); i++) { result.setValue(i, getValue(i) + v.getValue(i)); } return result; }
public Vector copyVector() { Vector result = new Vector(getRowDimension()); for (int i = 0; i < getRowDimension(); i++) { result.setValue(i, getValue(i)); } return result; }
/* * ONLY for testing to set up a network with known weights in future use to * deserialize networks after adding variables for pen weightupdate, * lastnput etc */ public FeedForwardNeuralNetwork(Matrix hiddenLayerWeights, Vector hiddenLayerBias, Matrix outputLayerWeights, Vector outputLayerBias) { hiddenLayer = new Layer(hiddenLayerWeights, hiddenLayerBias, new LogSigActivationFunction()); outputLayer = new Layer(outputLayerWeights, outputLayerBias, new PureLinearActivationFunction()); }
public Matrix calculateWeightUpdates(LayerSensitivity layerSensitivity, Vector previousLayerActivationOrInput, double alpha, double momentum) { Layer layer = layerSensitivity.getLayer(); Matrix activationTranspose = previousLayerActivationOrInput.transpose(); Matrix momentumLessUpdate = layerSensitivity.getSensitivityMatrix() .times(activationTranspose).times(alpha).times(-1.0); Matrix updateWithMomentum = layer.getLastWeightUpdateMatrix().times( momentum).plus(momentumLessUpdate.times(1.0 - momentum)); layer.acceptNewWeightUpdate(updateWithMomentum.copy()); return updateWithMomentum; }
public Layer(Matrix weightMatrix, Vector biasVector, ActivationFunction af) { activationFunction = af; this.weightMatrix = weightMatrix; lastWeightUpdateMatrix = new Matrix(weightMatrix.getRowDimension(), weightMatrix.getColumnDimension()); penultimateWeightUpdateMatrix = new Matrix(weightMatrix .getRowDimension(), weightMatrix.getColumnDimension()); this.biasVector = biasVector; lastBiasUpdateVector = new Vector(biasVector.getRowDimension()); penultimateBiasUpdateVector = new Vector(biasVector.getRowDimension()); }
public void testFeedForwardAndBAckLoopWorksWithMomentum() { // example 11.14 of Neural Network Design by Hagan, Demuth and Beale Matrix hiddenLayerWeightMatrix = new Matrix(2, 1); hiddenLayerWeightMatrix.set(0, 0, -0.27); hiddenLayerWeightMatrix.set(1, 0, -0.41); Vector hiddenLayerBiasVector = new Vector(2); hiddenLayerBiasVector.setValue(0, -0.48); hiddenLayerBiasVector.setValue(1, -0.13); Vector input = new Vector(1); input.setValue(0, 1); Matrix outputLayerWeightMatrix = new Matrix(1, 2); outputLayerWeightMatrix.set(0, 0, 0.09); outputLayerWeightMatrix.set(0, 1, -0.17); Vector outputLayerBiasVector = new Vector(1); outputLayerBiasVector.setValue(0, 0.48); Vector error = new Vector(1); error.setValue(0, 1.261); double learningRate = 0.1; double momentumFactor = 0.5; FeedForwardNeuralNetwork ffnn = new FeedForwardNeuralNetwork( hiddenLayerWeightMatrix, hiddenLayerBiasVector, outputLayerWeightMatrix, outputLayerBiasVector); ffnn.setTrainingScheme(new BackPropLearning(learningRate, momentumFactor)); ffnn.processInput(input); ffnn.processError(error); Matrix finalHiddenLayerWeights = ffnn.getHiddenLayerWeights(); Assert.AreEqual(-0.2675, finalHiddenLayerWeights.get(0, 0), 0.001); Assert.AreEqual(-0.4149, finalHiddenLayerWeights.get(1, 0), 0.001); Vector hiddenLayerBias = ffnn.getHiddenLayerBias(); Assert.AreEqual(-0.4775, hiddenLayerBias.getValue(0), 0.001); Assert.AreEqual(-0.1349, hiddenLayerBias.getValue(1), 0.001); Matrix finalOutputLayerWeights = ffnn.getOutputLayerWeights(); Assert.AreEqual(0.1304, finalOutputLayerWeights.get(0, 0), 0.001); Assert.AreEqual(-0.1235, finalOutputLayerWeights.get(0, 1), 0.001); Vector outputLayerBias = ffnn.getOutputLayerBias(); Assert.AreEqual(0.6061, outputLayerBias.getValue(0), 0.001); }
public Layer(int numberOfNeurons, int numberOfInputs, double lowerLimitForWeights, double upperLimitForWeights, ActivationFunction af) { activationFunction = af; this.weightMatrix = new Matrix(numberOfNeurons, numberOfInputs); lastWeightUpdateMatrix = new Matrix(weightMatrix.getRowDimension(), weightMatrix.getColumnDimension()); penultimateWeightUpdateMatrix = new Matrix(weightMatrix .getRowDimension(), weightMatrix.getColumnDimension()); this.biasVector = new Vector(numberOfNeurons); lastBiasUpdateVector = new Vector(biasVector.getRowDimension()); penultimateBiasUpdateVector = new Vector(biasVector.getRowDimension()); initializeMatrix(weightMatrix, lowerLimitForWeights, upperLimitForWeights); initializeVector(biasVector, lowerLimitForWeights, upperLimitForWeights); }
public Vector feedForward(Vector inputVector) { lastInput = inputVector; Matrix inducedField = weightMatrix.times(inputVector).plus(biasVector); Vector inducedFieldVector = new Vector(numberOfNeurons()); for (int i = 0; i < numberOfNeurons(); i++) { inducedFieldVector.setValue(i, inducedField.get(i, 0)); } lastInducedField = inducedFieldVector.copyVector(); Vector resultVector = new Vector(numberOfNeurons()); for (int i = 0; i < numberOfNeurons(); i++) { resultVector.setValue(i, activationFunction .activation(inducedFieldVector.getValue(i))); } // set the result as the last activation value lastActivationValues = resultVector.copyVector(); return resultVector; }
public void testFeedForward() { // example 11.14 of Neural Network Design by Hagan, Demuth and Beale // lots of tedious tests necessary to ensure nn is fundamentally correct Matrix weightMatrix1 = new Matrix(2, 1); weightMatrix1.set(0, 0, -0.27); weightMatrix1.set(1, 0, -0.41); Vector biasVector1 = new Vector(2); biasVector1.setValue(0, -0.48); biasVector1.setValue(1, -0.13); Layer layer1 = new Layer(weightMatrix1, biasVector1, new LogSigActivationFunction()); Vector inputVector1 = new Vector(1); inputVector1.setValue(0, 1); Vector expected = new Vector(2); expected.setValue(0, 0.321); expected.setValue(1, 0.368); Vector result1 = layer1.feedForward(inputVector1); Assert.AreEqual(expected.getValue(0), result1.getValue(0), 0.001); Assert.AreEqual(expected.getValue(1), result1.getValue(1), 0.001); Matrix weightMatrix2 = new Matrix(1, 2); weightMatrix2.set(0, 0, 0.09); weightMatrix2.set(0, 1, -0.17); Vector biasVector2 = new Vector(1); biasVector2.setValue(0, 0.48); Layer layer2 = new Layer(weightMatrix2, biasVector2, new PureLinearActivationFunction()); Vector inputVector2 = layer1.getLastActivationValues(); Vector result2 = layer2.feedForward(inputVector2); Assert.AreEqual(0.446, result2.getValue(0), 0.001); }
public void testSensitivityMatrixCalculationFromErrorVector() { Matrix weightMatrix1 = new Matrix(2, 1); weightMatrix1.set(0, 0, -0.27); weightMatrix1.set(1, 0, -0.41); Vector biasVector1 = new Vector(2); biasVector1.setValue(0, -0.48); biasVector1.setValue(1, -0.13); Layer layer1 = new Layer(weightMatrix1, biasVector1, new LogSigActivationFunction()); Vector inputVector1 = new Vector(1); inputVector1.setValue(0, 1); layer1.feedForward(inputVector1); Matrix weightMatrix2 = new Matrix(1, 2); weightMatrix2.set(0, 0, 0.09); weightMatrix2.set(0, 1, -0.17); Vector biasVector2 = new Vector(1); biasVector2.setValue(0, 0.48); Layer layer2 = new Layer(weightMatrix2, biasVector2, new PureLinearActivationFunction()); Vector inputVector2 = layer1.getLastActivationValues(); layer2.feedForward(inputVector2); Vector errorVector = new Vector(1); errorVector.setValue(0, 1.261); LayerSensitivity layer2Sensitivity = new LayerSensitivity(layer2); layer2Sensitivity.sensitivityMatrixFromErrorMatrix(errorVector); Matrix sensitivityMatrix = layer2Sensitivity.getSensitivityMatrix(); Assert.AreEqual(-2.522, sensitivityMatrix.get(0, 0), 0.0001); }
public void acceptNewBiasUpdate(Vector biasUpdate) { setPenultimateBiasUpdateVector(getLastBiasUpdateVector()); setLastBiasUpdateVector(biasUpdate); }
private static void initializeVector(Vector aVector, double lowerLimit, double upperLimit) { for (int i = 0; i < aVector.size(); i++) { double random = Util.generateRandomDoubleBetween(lowerLimit, upperLimit); aVector.setValue(i, random); } }
public Vector processInput(Vector input) { lastInput = input; return layer.feedForward(input); }
public void processError(Vector error) { trainingScheme.processError(this, error); }
public Vector calculateBiasUpdates(LayerSensitivity layerSensitivity, double alpha, double momentum) { Layer layer = layerSensitivity.getLayer(); Matrix biasUpdateMatrixWithoutMomentum = layerSensitivity .getSensitivityMatrix().times(alpha).times(-1.0); Matrix biasUpdateMatrixWithMomentum = layer.getLastBiasUpdateVector() .times(momentum).plus( biasUpdateMatrixWithoutMomentum.times(1.0 - momentum)); Vector result = new Vector(biasUpdateMatrixWithMomentum .getRowDimension()); for (int i = 0; i < biasUpdateMatrixWithMomentum.getRowDimension(); i++) { result.setValue(i, biasUpdateMatrixWithMomentum.get(i, 0)); } layer.acceptNewBiasUpdate(result.copyVector()); return result; }
public void setLastBiasUpdateVector(Vector v) { lastBiasUpdateVector = v; }
public void testWeightsAndBiasesUpdatedCorrectly() { Matrix weightMatrix1 = new Matrix(2, 1); weightMatrix1.set(0, 0, -0.27); weightMatrix1.set(1, 0, -0.41); Vector biasVector1 = new Vector(2); biasVector1.setValue(0, -0.48); biasVector1.setValue(1, -0.13); Layer layer1 = new Layer(weightMatrix1, biasVector1, new LogSigActivationFunction()); LayerSensitivity layer1Sensitivity = new LayerSensitivity(layer1); Vector inputVector1 = new Vector(1); inputVector1.setValue(0, 1); layer1.feedForward(inputVector1); Matrix weightMatrix2 = new Matrix(1, 2); weightMatrix2.set(0, 0, 0.09); weightMatrix2.set(0, 1, -0.17); Vector biasVector2 = new Vector(1); biasVector2.setValue(0, 0.48); Layer layer2 = new Layer(weightMatrix2, biasVector2, new PureLinearActivationFunction()); Vector inputVector2 = layer1.getLastActivationValues(); layer2.feedForward(inputVector2); Vector errorVector = new Vector(1); errorVector.setValue(0, 1.261); LayerSensitivity layer2Sensitivity = new LayerSensitivity(layer2); layer2Sensitivity.sensitivityMatrixFromErrorMatrix(errorVector); layer1Sensitivity .sensitivityMatrixFromSucceedingLayer(layer2Sensitivity); BackPropLearning.calculateWeightUpdates(layer2Sensitivity, layer1 .getLastActivationValues(), 0.1); BackPropLearning.calculateBiasUpdates(layer2Sensitivity, 0.1); BackPropLearning.calculateWeightUpdates(layer1Sensitivity, inputVector1, 0.1); BackPropLearning.calculateBiasUpdates(layer1Sensitivity, 0.1); layer2.updateWeights(); Matrix newWeightMatrix2 = layer2.getWeightMatrix(); Assert.AreEqual(0.171, newWeightMatrix2.get(0, 0), 0.001); Assert.AreEqual(-0.0772, newWeightMatrix2.get(0, 1), 0.001); layer2.updateBiases(); Vector newBiasVector2 = layer2.getBiasVector(); Assert.AreEqual(0.7322, newBiasVector2.getValue(0), 0.00001); layer1.updateWeights(); Matrix newWeightMatrix1 = layer1.getWeightMatrix(); Assert.AreEqual(-0.265, newWeightMatrix1.get(0, 0), 0.001); Assert.AreEqual(-0.419, newWeightMatrix1.get(1, 0), 0.001); layer1.updateBiases(); Vector newBiasVector1 = layer1.getBiasVector(); Assert.AreEqual(-0.475, newBiasVector1.getValue(0), 0.001); Assert.AreEqual(-0.139, newBiasVector1.getValue(1), 0.001); }
public void testBiasUpdateMatrixesFormedCorrectly() { Matrix weightMatrix1 = new Matrix(2, 1); weightMatrix1.set(0, 0, -0.27); weightMatrix1.set(1, 0, -0.41); Vector biasVector1 = new Vector(2); biasVector1.setValue(0, -0.48); biasVector1.setValue(1, -0.13); Layer layer1 = new Layer(weightMatrix1, biasVector1, new LogSigActivationFunction()); LayerSensitivity layer1Sensitivity = new LayerSensitivity(layer1); Vector inputVector1 = new Vector(1); inputVector1.setValue(0, 1); layer1.feedForward(inputVector1); Matrix weightMatrix2 = new Matrix(1, 2); weightMatrix2.set(0, 0, 0.09); weightMatrix2.set(0, 1, -0.17); Vector biasVector2 = new Vector(1); biasVector2.setValue(0, 0.48); Layer layer2 = new Layer(weightMatrix2, biasVector2, new PureLinearActivationFunction()); LayerSensitivity layer2Sensitivity = new LayerSensitivity(layer2); Vector inputVector2 = layer1.getLastActivationValues(); layer2.feedForward(inputVector2); Vector errorVector = new Vector(1); errorVector.setValue(0, 1.261); layer2Sensitivity.sensitivityMatrixFromErrorMatrix(errorVector); layer1Sensitivity .sensitivityMatrixFromSucceedingLayer(layer2Sensitivity); Vector biasUpdateVector2 = BackPropLearning.calculateBiasUpdates( layer2Sensitivity, 0.1); Assert.AreEqual(0.2522, biasUpdateVector2.getValue(0), 0.001); Vector lastBiasUpdateVector2 = layer2.getLastBiasUpdateVector(); Assert.AreEqual(0.2522, lastBiasUpdateVector2.getValue(0), 0.001); Vector penultimateBiasUpdateVector2 = layer2 .getPenultimateBiasUpdateVector(); Assert.AreEqual(0.0, penultimateBiasUpdateVector2.getValue(0), 0.001); Vector biasUpdateVector1 = BackPropLearning.calculateBiasUpdates( layer1Sensitivity, 0.1); Assert.AreEqual(0.00495, biasUpdateVector1.getValue(0), 0.001); Assert.AreEqual(-0.00997, biasUpdateVector1.getValue(1), 0.001); Vector lastBiasUpdateVector1 = layer1.getLastBiasUpdateVector(); Assert.AreEqual(0.00495, lastBiasUpdateVector1.getValue(0), 0.001); Assert.AreEqual(-0.00997, lastBiasUpdateVector1.getValue(1), 0.001); Vector penultimateBiasUpdateVector1 = layer1 .getPenultimateBiasUpdateVector(); Assert.AreEqual(0.0, penultimateBiasUpdateVector1.getValue(0), 0.001); Assert.AreEqual(0.0, penultimateBiasUpdateVector1.getValue(1), 0.001); }
public Vector errorVectorFrom(Vector target) { return target.minus(getLastActivationValues()); }
public void setPenultimateBiasUpdateVector(Vector v) { penultimateBiasUpdateVector = v; }
public Vector getTarget() { Vector v = new Vector(normalizedTarget); return v; }
public void updateBiases() { Matrix biasMatrix = biasVector.plusEquals(lastBiasUpdateVector); Vector result = new Vector(biasMatrix.getRowDimension()); for (int i = 0; i < biasMatrix.getRowDimension(); i++) { result.setValue(i, biasMatrix.get(i, 0)); } biasVector = result; }
public Vector processInput(Vector input) { return trainingScheme.processInput(this, input); }
public Vector getInput() { Vector v = new Vector(normalizedInput); return v; }