public static void LUDecomposition() { SquareMatrix A = new SquareMatrix(new double[, ] { { 1, -2, 3 }, { 2, -5, 12 }, { 0, 2, -10 } }); ColumnVector b = new ColumnVector(2, 8, -4); LUDecomposition lud = A.LUDecomposition(); ColumnVector x = lud.Solve(b); PrintMatrix("x", x); PrintMatrix("Ax", A * x); SquareMatrix L = lud.LMatrix(); SquareMatrix U = lud.UMatrix(); SquareMatrix P = lud.PMatrix(); PrintMatrix("LU", L * U); PrintMatrix("PA", P * A); SquareMatrix AI = lud.Inverse(); PrintMatrix("A * AI", A * AI); Console.WriteLine($"det(a) = {lud.Determinant()}"); }
public void SquareRandomMatrixLUDecomposition() { for (int d = 1; d <= 256; d += 11) { SquareMatrix M = CreateSquareRandomMatrix(d); // LU decompose the matrix //Stopwatch sw = Stopwatch.StartNew(); LUDecomposition LU = M.LUDecomposition(); //sw.Stop(); //Console.WriteLine(sw.ElapsedMilliseconds); Assert.IsTrue(LU.Dimension == d); // test that the decomposition works SquareMatrix P = LU.PMatrix(); SquareMatrix L = LU.LMatrix(); SquareMatrix U = LU.UMatrix(); Assert.IsTrue(TestUtilities.IsNearlyEqual(P * M, L * U)); // check that the inverse works SquareMatrix MI = LU.Inverse(); Assert.IsTrue(TestUtilities.IsNearlyEqual(M * MI, UnitMatrix.OfDimension(d))); // test that a solution works ColumnVector t = new ColumnVector(d); for (int i = 0; i < d; i++) { t[i] = i; } ColumnVector s = LU.Solve(t); Assert.IsTrue(TestUtilities.IsNearlyEqual(M * s, t)); } }
public void SquareUnitMatrixLUDecomposition() { for (int d = 1; d <= 10; d++) { SquareMatrix I = UnitMatrix.OfDimension(d).ToSquareMatrix(); Assert.IsTrue(I.Trace() == d); LUDecomposition LU = I.LUDecomposition(); Assert.IsTrue(LU.Determinant() == 1.0); SquareMatrix II = LU.Inverse(); Assert.IsTrue(TestUtilities.IsNearlyEqual(II, I)); } }
public void SquareVandermondeMatrixLUDecomposition() { // fails now for d = 8 because determinant slightly off for (int d = 1; d <= 7; d++) { Console.WriteLine("d={0}", d); double[] x = new double[d]; for (int i = 0; i < d; i++) { x[i] = i; } double det = 1.0; for (int i = 0; i < d; i++) { for (int j = 0; j < i; j++) { det = det * (x[i] - x[j]); } } // LU decompose the matrix SquareMatrix V = CreateVandermondeMatrix(d); LUDecomposition LU = V.LUDecomposition(); // test that the decomposition works SquareMatrix P = LU.PMatrix(); SquareMatrix L = LU.LMatrix(); SquareMatrix U = LU.UMatrix(); Assert.IsTrue(TestUtilities.IsNearlyEqual(P * V, L * U)); // check that the determinant agrees with the analytic expression Console.WriteLine("det {0} {1}", LU.Determinant(), det); Assert.IsTrue(TestUtilities.IsNearlyEqual(LU.Determinant(), det)); // check that the inverse works SquareMatrix VI = LU.Inverse(); //PrintMatrix(VI); //PrintMatrix(V * VI); SquareMatrix I = TestUtilities.CreateSquareUnitMatrix(d); Assert.IsTrue(TestUtilities.IsNearlyEqual(V * VI, I)); // test that a solution works ColumnVector t = new ColumnVector(d); for (int i = 0; i < d; i++) { t[i] = 1.0; } ColumnVector s = LU.Solve(t); Assert.IsTrue(TestUtilities.IsNearlyEqual(V * s, t)); } }
/// <summary> /// Perform one iteration. /// </summary> public override void Iteration() { LUDecomposition decomposition = null; double trace = 0; PreIteration(); this.weights = NetworkCODEC.NetworkToArray(this.network); IComputeJacobian j = new JacobianChainRule(this.network, this.indexableTraining); double sumOfSquaredErrors = j.Calculate(this.weights); double sumOfSquaredWeights = CalculateSumOfSquaredWeights(); // this.setError(j.getError()); CalculateHessian(j.Jacobian, j.RowErrors); // Define the objective function // bayesian regularization objective function double objective = this.beta * sumOfSquaredErrors + this.alpha * sumOfSquaredWeights; double current = objective + 1.0; // Start the main Levenberg-Macquardt method this.lambda /= LevenbergMarquardtTraining.SCALE_LAMBDA; // We'll try to find a direction with less error // (or where the objective function is smaller) while ((current >= objective) && (this.lambda < LevenbergMarquardtTraining.LAMBDA_MAX)) { this.lambda *= LevenbergMarquardtTraining.SCALE_LAMBDA; // Update diagonal (Levenberg-Marquardt formula) for (int i = 0; i < this.parametersLength; i++) { this.hessian[i][i] = this.diagonal[i] + (this.lambda + this.alpha); } // Decompose to solve the linear system decomposition = new LUDecomposition( this.hessianMatrix); // Check if the Jacobian has become non-invertible if (!decomposition.IsNonsingular) { continue; } // Solve using LU (or SVD) decomposition this.deltas = decomposition.Solve(this.gradient); // Update weights using the calculated deltas sumOfSquaredWeights = UpdateWeights(); // Calculate the new error sumOfSquaredErrors = 0.0; for (int i = 0; i < this.trainingLength; i++) { this.indexableTraining.GetRecord(i, this.pair); INeuralData actual = this.network.Compute(this.pair .Input); double e = this.pair.Ideal[0] - actual[0]; sumOfSquaredErrors += e * e; } sumOfSquaredErrors /= 2.0; // Update the objective function current = this.beta * sumOfSquaredErrors + this.alpha * sumOfSquaredWeights; // If the object function is bigger than before, the method // is tried again using a greater dumping factor. } // If this iteration caused a error drop, then next iteration // will use a smaller damping factor. this.lambda /= LevenbergMarquardtTraining.SCALE_LAMBDA; if (useBayesianRegularization && decomposition != null) { // Compute the trace for the inverse Hessian trace = Trace(decomposition.Inverse()); // Poland update's formula: gamma = this.parametersLength - (alpha * trace); alpha = this.parametersLength / (2.0 * sumOfSquaredWeights + trace); beta = Math.Abs((this.trainingLength - gamma) / (2.0 * sumOfSquaredErrors)); } this.Error = sumOfSquaredErrors; PostIteration(); }
/// <summary> /// Perform one iteration. /// </summary> /// public override void Iteration() { LUDecomposition decomposition = null; PreIteration(); _weights = NetworkCODEC.NetworkToArray(_network); IComputeJacobian j = new JacobianChainRule(_network, _indexableTraining); double sumOfSquaredErrors = j.Calculate(_weights); double sumOfSquaredWeights = CalculateSumOfSquaredWeights(); // this.setError(j.getError()); CalculateHessian(j.Jacobian, j.RowErrors); // Define the objective function // bayesian regularization objective function double objective = _beta * sumOfSquaredErrors + _alpha * sumOfSquaredWeights; double current = objective + 1.0d; // Start the main Levenberg-Macquardt method _lambda /= ScaleLambda; // We'll try to find a direction with less error // (or where the objective function is smaller) while ((current >= objective) && (_lambda < LambdaMax)) { _lambda *= ScaleLambda; // Update diagonal (Levenberg-Marquardt formula) for (int i = 0; i < _parametersLength; i++) { _hessian[i][i] = _diagonal[i] + (_lambda + _alpha); } // Decompose to solve the linear system decomposition = new LUDecomposition(_hessianMatrix); // Check if the Jacobian has become non-invertible if (!decomposition.IsNonsingular) { continue; } // Solve using LU (or SVD) decomposition _deltas = decomposition.Solve(_gradient); // Update weights using the calculated deltas sumOfSquaredWeights = UpdateWeights(); // Calculate the new error sumOfSquaredErrors = 0.0d; for (int i = 0; i < _trainingLength; i++) { _indexableTraining.GetRecord(i, _pair); IMLData actual = _network .Compute(_pair.Input); double e = _pair.Ideal[0] - actual[0]; sumOfSquaredErrors += e * e; } sumOfSquaredErrors /= 2.0d; // Update the objective function current = _beta * sumOfSquaredErrors + _alpha * sumOfSquaredWeights; // If the object function is bigger than before, the method // is tried again using a greater dumping factor. } // If this iteration caused a error drop, then next iteration // will use a smaller damping factor. _lambda /= ScaleLambda; if (_useBayesianRegularization && (decomposition != null)) { // Compute the trace for the inverse Hessian double trace = Trace(decomposition.Inverse()); // Poland update's formula: _gamma = _parametersLength - (_alpha * trace); _alpha = _parametersLength / (2.0d * sumOfSquaredWeights + trace); _beta = Math.Abs((_trainingLength - _gamma) / (2.0d * sumOfSquaredErrors)); } Error = sumOfSquaredErrors; PostIteration(); }