public Matrix calculateWeightUpdates(LayerSensitivity layerSensitivity, Vector previousLayerActivationOrInput, double alpha, double momentum) { Layer layer = layerSensitivity.getLayer(); Matrix activationTranspose = previousLayerActivationOrInput.transpose(); Matrix momentumLessUpdate = layerSensitivity.getSensitivityMatrix() .times(activationTranspose).times(alpha).times(-1.0); Matrix updateWithMomentum = layer.getLastWeightUpdateMatrix().times( momentum).plus(momentumLessUpdate.times(1.0 - momentum)); layer.acceptNewWeightUpdate(updateWithMomentum.copy()); return(updateWithMomentum); }
public void testWeightUpdateMatrixesFormedCorrectly() { Matrix weightMatrix1 = new Matrix(2, 1); weightMatrix1.set(0, 0, -0.27); weightMatrix1.set(1, 0, -0.41); Vector biasVector1 = new Vector(2); biasVector1.setValue(0, -0.48); biasVector1.setValue(1, -0.13); Layer layer1 = new Layer(weightMatrix1, biasVector1, new LogSigActivationFunction()); LayerSensitivity layer1Sensitivity = new LayerSensitivity(layer1); Vector inputVector1 = new Vector(1); inputVector1.setValue(0, 1); layer1.feedForward(inputVector1); Matrix weightMatrix2 = new Matrix(1, 2); weightMatrix2.set(0, 0, 0.09); weightMatrix2.set(0, 1, -0.17); Vector biasVector2 = new Vector(1); biasVector2.setValue(0, 0.48); Layer layer2 = new Layer(weightMatrix2, biasVector2, new PureLinearActivationFunction()); Vector inputVector2 = layer1.getLastActivationValues(); layer2.feedForward(inputVector2); Vector errorVector = new Vector(1); errorVector.setValue(0, 1.261); LayerSensitivity layer2Sensitivity = new LayerSensitivity(layer2); layer2Sensitivity.sensitivityMatrixFromErrorMatrix(errorVector); layer1Sensitivity .sensitivityMatrixFromSucceedingLayer(layer2Sensitivity); Matrix weightUpdateMatrix2 = BackPropLearning.calculateWeightUpdates( layer2Sensitivity, layer1.getLastActivationValues(), 0.1); Assert.AreEqual(0.0809, weightUpdateMatrix2.get(0, 0), 0.001); Assert.AreEqual(0.0928, weightUpdateMatrix2.get(0, 1), 0.001); Matrix lastWeightUpdateMatrix2 = layer2.getLastWeightUpdateMatrix(); Assert.AreEqual(0.0809, lastWeightUpdateMatrix2.get(0, 0), 0.001); Assert.AreEqual(0.0928, lastWeightUpdateMatrix2.get(0, 1), 0.001); Matrix penultimateWeightUpdatematrix2 = layer2 .getPenultimateWeightUpdateMatrix(); Assert.AreEqual(0.0, penultimateWeightUpdatematrix2.get(0, 0), 0.001); Assert.AreEqual(0.0, penultimateWeightUpdatematrix2.get(0, 1), 0.001); Matrix weightUpdateMatrix1 = BackPropLearning.calculateWeightUpdates( layer1Sensitivity, inputVector1, 0.1); Assert.AreEqual(0.0049, weightUpdateMatrix1.get(0, 0), 0.001); Assert.AreEqual(-0.00997, weightUpdateMatrix1.get(1, 0), 0.001); Matrix lastWeightUpdateMatrix1 = layer1.getLastWeightUpdateMatrix(); Assert.AreEqual(0.0049, lastWeightUpdateMatrix1.get(0, 0), 0.001); Assert.AreEqual(-0.00997, lastWeightUpdateMatrix1.get(1, 0), 0.001); Matrix penultimateWeightUpdatematrix1 = layer1 .getPenultimateWeightUpdateMatrix(); Assert.AreEqual(0.0, penultimateWeightUpdatematrix1.get(0, 0), 0.001); Assert.AreEqual(0.0, penultimateWeightUpdatematrix1.get(1, 0), 0.001); }