public LayerSensitivity(Layer layer) { Matrix weightMatrix = layer.getWeightMatrix(); this.sensitivityMatrix = new Matrix(weightMatrix.getRowDimension(), weightMatrix.getColumnDimension()); this.layer = layer; }
public Perceptron(int numberOfNeurons, int numberOfInputs) { this.layer = new Layer(numberOfNeurons, numberOfInputs, 2.0, -2.0, new HardLimitActivationFunction()); }
public void setNeuralNetwork(FunctionApproximator fapp) { FeedForwardNeuralNetwork ffnn = (FeedForwardNeuralNetwork)fapp; this.hiddenLayer = ffnn.getHiddenLayer(); this.outputLayer = ffnn.getOutputLayer(); this.hiddenSensitivity = new LayerSensitivity(hiddenLayer); this.outputSensitivity = new LayerSensitivity(outputLayer); }
/* * ONLY for testing to set up a network with known weights in future use to * deserialize networks after adding variables for pen weightupdate, * lastnput etc */ public FeedForwardNeuralNetwork(Matrix hiddenLayerWeights, Vector hiddenLayerBias, Matrix outputLayerWeights, Vector outputLayerBias) { hiddenLayer = new Layer(hiddenLayerWeights, hiddenLayerBias, new LogSigActivationFunction()); outputLayer = new Layer(outputLayerWeights, outputLayerBias, new PureLinearActivationFunction()); }
public void testFeedForward() { // example 11.14 of Neural Network Design by Hagan, Demuth and Beale // lots of tedious tests necessary to ensure nn is fundamentally correct Matrix weightMatrix1 = new Matrix(2, 1); weightMatrix1.set(0, 0, -0.27); weightMatrix1.set(1, 0, -0.41); Vector biasVector1 = new Vector(2); biasVector1.setValue(0, -0.48); biasVector1.setValue(1, -0.13); Layer layer1 = new Layer(weightMatrix1, biasVector1, new LogSigActivationFunction()); Vector inputVector1 = new Vector(1); inputVector1.setValue(0, 1); Vector expected = new Vector(2); expected.setValue(0, 0.321); expected.setValue(1, 0.368); Vector result1 = layer1.feedForward(inputVector1); Assert.AreEqual(expected.getValue(0), result1.getValue(0), 0.001); Assert.AreEqual(expected.getValue(1), result1.getValue(1), 0.001); Matrix weightMatrix2 = new Matrix(1, 2); weightMatrix2.set(0, 0, 0.09); weightMatrix2.set(0, 1, -0.17); Vector biasVector2 = new Vector(1); biasVector2.setValue(0, 0.48); Layer layer2 = new Layer(weightMatrix2, biasVector2, new PureLinearActivationFunction()); Vector inputVector2 = layer1.getLastActivationValues(); Vector result2 = layer2.feedForward(inputVector2); Assert.AreEqual(0.446, result2.getValue(0), 0.001); }
public void testSensitivityMatrixCalculationFromErrorVector() { Matrix weightMatrix1 = new Matrix(2, 1); weightMatrix1.set(0, 0, -0.27); weightMatrix1.set(1, 0, -0.41); Vector biasVector1 = new Vector(2); biasVector1.setValue(0, -0.48); biasVector1.setValue(1, -0.13); Layer layer1 = new Layer(weightMatrix1, biasVector1, new LogSigActivationFunction()); Vector inputVector1 = new Vector(1); inputVector1.setValue(0, 1); layer1.feedForward(inputVector1); Matrix weightMatrix2 = new Matrix(1, 2); weightMatrix2.set(0, 0, 0.09); weightMatrix2.set(0, 1, -0.17); Vector biasVector2 = new Vector(1); biasVector2.setValue(0, 0.48); Layer layer2 = new Layer(weightMatrix2, biasVector2, new PureLinearActivationFunction()); Vector inputVector2 = layer1.getLastActivationValues(); layer2.feedForward(inputVector2); Vector errorVector = new Vector(1); errorVector.setValue(0, 1.261); LayerSensitivity layer2Sensitivity = new LayerSensitivity(layer2); layer2Sensitivity.sensitivityMatrixFromErrorMatrix(errorVector); Matrix sensitivityMatrix = layer2Sensitivity.getSensitivityMatrix(); Assert.AreEqual(-2.522, sensitivityMatrix.get(0, 0), 0.0001); }
/* * constructor to be used for non testing code. */ public FeedForwardNeuralNetwork(NNConfig config) { int numberOfInputNeurons = config .getParameterAsint(NUMBER_OF_INPUTS); int numberOfHiddenNeurons = config .getParameterAsint(NUMBER_OF_HIDDEN_NEURONS); int numberOfOutputNeurons = config .getParameterAsint(NUMBER_OF_OUTPUTS); double lowerLimitForWeights = config .getParameterAsDouble(LOWER_LIMIT_WEIGHTS); double upperLimitForWeights = config .getParameterAsDouble(UPPER_LIMIT_WEIGHTS); hiddenLayer = new Layer(numberOfHiddenNeurons, numberOfInputNeurons, lowerLimitForWeights, upperLimitForWeights, new LogSigActivationFunction()); outputLayer = new Layer(numberOfOutputNeurons, numberOfHiddenNeurons, lowerLimitForWeights, upperLimitForWeights, new PureLinearActivationFunction()); }
public void testWeightsAndBiasesUpdatedCorrectly() { Matrix weightMatrix1 = new Matrix(2, 1); weightMatrix1.set(0, 0, -0.27); weightMatrix1.set(1, 0, -0.41); Vector biasVector1 = new Vector(2); biasVector1.setValue(0, -0.48); biasVector1.setValue(1, -0.13); Layer layer1 = new Layer(weightMatrix1, biasVector1, new LogSigActivationFunction()); LayerSensitivity layer1Sensitivity = new LayerSensitivity(layer1); Vector inputVector1 = new Vector(1); inputVector1.setValue(0, 1); layer1.feedForward(inputVector1); Matrix weightMatrix2 = new Matrix(1, 2); weightMatrix2.set(0, 0, 0.09); weightMatrix2.set(0, 1, -0.17); Vector biasVector2 = new Vector(1); biasVector2.setValue(0, 0.48); Layer layer2 = new Layer(weightMatrix2, biasVector2, new PureLinearActivationFunction()); Vector inputVector2 = layer1.getLastActivationValues(); layer2.feedForward(inputVector2); Vector errorVector = new Vector(1); errorVector.setValue(0, 1.261); LayerSensitivity layer2Sensitivity = new LayerSensitivity(layer2); layer2Sensitivity.sensitivityMatrixFromErrorMatrix(errorVector); layer1Sensitivity .sensitivityMatrixFromSucceedingLayer(layer2Sensitivity); BackPropLearning.calculateWeightUpdates(layer2Sensitivity, layer1 .getLastActivationValues(), 0.1); BackPropLearning.calculateBiasUpdates(layer2Sensitivity, 0.1); BackPropLearning.calculateWeightUpdates(layer1Sensitivity, inputVector1, 0.1); BackPropLearning.calculateBiasUpdates(layer1Sensitivity, 0.1); layer2.updateWeights(); Matrix newWeightMatrix2 = layer2.getWeightMatrix(); Assert.AreEqual(0.171, newWeightMatrix2.get(0, 0), 0.001); Assert.AreEqual(-0.0772, newWeightMatrix2.get(0, 1), 0.001); layer2.updateBiases(); Vector newBiasVector2 = layer2.getBiasVector(); Assert.AreEqual(0.7322, newBiasVector2.getValue(0), 0.00001); layer1.updateWeights(); Matrix newWeightMatrix1 = layer1.getWeightMatrix(); Assert.AreEqual(-0.265, newWeightMatrix1.get(0, 0), 0.001); Assert.AreEqual(-0.419, newWeightMatrix1.get(1, 0), 0.001); layer1.updateBiases(); Vector newBiasVector1 = layer1.getBiasVector(); Assert.AreEqual(-0.475, newBiasVector1.getValue(0), 0.001); Assert.AreEqual(-0.139, newBiasVector1.getValue(1), 0.001); }
public void testBiasUpdateMatrixesFormedCorrectly() { Matrix weightMatrix1 = new Matrix(2, 1); weightMatrix1.set(0, 0, -0.27); weightMatrix1.set(1, 0, -0.41); Vector biasVector1 = new Vector(2); biasVector1.setValue(0, -0.48); biasVector1.setValue(1, -0.13); Layer layer1 = new Layer(weightMatrix1, biasVector1, new LogSigActivationFunction()); LayerSensitivity layer1Sensitivity = new LayerSensitivity(layer1); Vector inputVector1 = new Vector(1); inputVector1.setValue(0, 1); layer1.feedForward(inputVector1); Matrix weightMatrix2 = new Matrix(1, 2); weightMatrix2.set(0, 0, 0.09); weightMatrix2.set(0, 1, -0.17); Vector biasVector2 = new Vector(1); biasVector2.setValue(0, 0.48); Layer layer2 = new Layer(weightMatrix2, biasVector2, new PureLinearActivationFunction()); LayerSensitivity layer2Sensitivity = new LayerSensitivity(layer2); Vector inputVector2 = layer1.getLastActivationValues(); layer2.feedForward(inputVector2); Vector errorVector = new Vector(1); errorVector.setValue(0, 1.261); layer2Sensitivity.sensitivityMatrixFromErrorMatrix(errorVector); layer1Sensitivity .sensitivityMatrixFromSucceedingLayer(layer2Sensitivity); Vector biasUpdateVector2 = BackPropLearning.calculateBiasUpdates( layer2Sensitivity, 0.1); Assert.AreEqual(0.2522, biasUpdateVector2.getValue(0), 0.001); Vector lastBiasUpdateVector2 = layer2.getLastBiasUpdateVector(); Assert.AreEqual(0.2522, lastBiasUpdateVector2.getValue(0), 0.001); Vector penultimateBiasUpdateVector2 = layer2 .getPenultimateBiasUpdateVector(); Assert.AreEqual(0.0, penultimateBiasUpdateVector2.getValue(0), 0.001); Vector biasUpdateVector1 = BackPropLearning.calculateBiasUpdates( layer1Sensitivity, 0.1); Assert.AreEqual(0.00495, biasUpdateVector1.getValue(0), 0.001); Assert.AreEqual(-0.00997, biasUpdateVector1.getValue(1), 0.001); Vector lastBiasUpdateVector1 = layer1.getLastBiasUpdateVector(); Assert.AreEqual(0.00495, lastBiasUpdateVector1.getValue(0), 0.001); Assert.AreEqual(-0.00997, lastBiasUpdateVector1.getValue(1), 0.001); Vector penultimateBiasUpdateVector1 = layer1 .getPenultimateBiasUpdateVector(); Assert.AreEqual(0.0, penultimateBiasUpdateVector1.getValue(0), 0.001); Assert.AreEqual(0.0, penultimateBiasUpdateVector1.getValue(1), 0.001); }