public static void LUDecomposition()
{
SquareMatrix A = new SquareMatrix(new double[, ] {
{ 1, -2, 3 },
{ 2, -5, 12 },
{ 0, 2, -10 }
});
ColumnVector b = new ColumnVector(2, 8, -4);
LUDecomposition lud = A.LUDecomposition();
ColumnVector x = lud.Solve(b);
PrintMatrix("x", x);
PrintMatrix("Ax", A * x);
SquareMatrix L = lud.LMatrix();
SquareMatrix U = lud.UMatrix();
SquareMatrix P = lud.PMatrix();
PrintMatrix("LU", L * U);
PrintMatrix("PA", P * A);
SquareMatrix AI = lud.Inverse();
PrintMatrix("A * AI", A * AI);
Console.WriteLine($"det(a) = {lud.Determinant()}");
}
public void SquareRandomMatrixLUDecomposition()
{
for (int d = 1; d <= 256; d += 11)
{
SquareMatrix M = CreateSquareRandomMatrix(d);
// LU decompose the matrix
//Stopwatch sw = Stopwatch.StartNew();
LUDecomposition LU = M.LUDecomposition();
//sw.Stop();
//Console.WriteLine(sw.ElapsedMilliseconds);
Assert.IsTrue(LU.Dimension == d);
// test that the decomposition works
SquareMatrix P = LU.PMatrix();
SquareMatrix L = LU.LMatrix();
SquareMatrix U = LU.UMatrix();
Assert.IsTrue(TestUtilities.IsNearlyEqual(P * M, L * U));
// check that the inverse works
SquareMatrix MI = LU.Inverse();
Assert.IsTrue(TestUtilities.IsNearlyEqual(M * MI, UnitMatrix.OfDimension(d)));
// test that a solution works
ColumnVector t = new ColumnVector(d);
for (int i = 0; i < d; i++)
{
t[i] = i;
}
ColumnVector s = LU.Solve(t);
Assert.IsTrue(TestUtilities.IsNearlyEqual(M * s, t));
}
}
public void SquareUnitMatrixLUDecomposition()
{
for (int d = 1; d <= 10; d++)
{
SquareMatrix I = UnitMatrix.OfDimension(d).ToSquareMatrix();
Assert.IsTrue(I.Trace() == d);
LUDecomposition LU = I.LUDecomposition();
Assert.IsTrue(LU.Determinant() == 1.0);
SquareMatrix II = LU.Inverse();
Assert.IsTrue(TestUtilities.IsNearlyEqual(II, I));
}
}
public void SquareVandermondeMatrixLUDecomposition()
{
// fails now for d = 8 because determinant slightly off
for (int d = 1; d <= 7; d++)
{
Console.WriteLine("d={0}", d);
double[] x = new double[d];
for (int i = 0; i < d; i++)
{
x[i] = i;
}
double det = 1.0;
for (int i = 0; i < d; i++)
{
for (int j = 0; j < i; j++)
{
det = det * (x[i] - x[j]);
}
}
// LU decompose the matrix
SquareMatrix V = CreateVandermondeMatrix(d);
LUDecomposition LU = V.LUDecomposition();
// test that the decomposition works
SquareMatrix P = LU.PMatrix();
SquareMatrix L = LU.LMatrix();
SquareMatrix U = LU.UMatrix();
Assert.IsTrue(TestUtilities.IsNearlyEqual(P * V, L * U));
// check that the determinant agrees with the analytic expression
Console.WriteLine("det {0} {1}", LU.Determinant(), det);
Assert.IsTrue(TestUtilities.IsNearlyEqual(LU.Determinant(), det));
// check that the inverse works
SquareMatrix VI = LU.Inverse();
//PrintMatrix(VI);
//PrintMatrix(V * VI);
SquareMatrix I = TestUtilities.CreateSquareUnitMatrix(d);
Assert.IsTrue(TestUtilities.IsNearlyEqual(V * VI, I));
// test that a solution works
ColumnVector t = new ColumnVector(d);
for (int i = 0; i < d; i++)
{
t[i] = 1.0;
}
ColumnVector s = LU.Solve(t);
Assert.IsTrue(TestUtilities.IsNearlyEqual(V * s, t));
}
}
/// <summary>
/// Perform one iteration.
/// </summary>
public override void Iteration()
{
LUDecomposition decomposition = null;
double trace = 0;
PreIteration();
this.weights = NetworkCODEC.NetworkToArray(this.network);
IComputeJacobian j = new JacobianChainRule(this.network,
this.indexableTraining);
double sumOfSquaredErrors = j.Calculate(this.weights);
double sumOfSquaredWeights = CalculateSumOfSquaredWeights();
// this.setError(j.getError());
CalculateHessian(j.Jacobian, j.RowErrors);
// Define the objective function
// bayesian regularization objective function
double objective = this.beta * sumOfSquaredErrors + this.alpha
* sumOfSquaredWeights;
double current = objective + 1.0;
// Start the main Levenberg-Macquardt method
this.lambda /= LevenbergMarquardtTraining.SCALE_LAMBDA;
// We'll try to find a direction with less error
// (or where the objective function is smaller)
while ((current >= objective) &&
(this.lambda < LevenbergMarquardtTraining.LAMBDA_MAX))
{
this.lambda *= LevenbergMarquardtTraining.SCALE_LAMBDA;
// Update diagonal (Levenberg-Marquardt formula)
for (int i = 0; i < this.parametersLength; i++)
{
this.hessian[i][i] = this.diagonal[i]
+ (this.lambda + this.alpha);
}
// Decompose to solve the linear system
decomposition = new LUDecomposition(
this.hessianMatrix);
// Check if the Jacobian has become non-invertible
if (!decomposition.IsNonsingular)
{
continue;
}
// Solve using LU (or SVD) decomposition
this.deltas = decomposition.Solve(this.gradient);
// Update weights using the calculated deltas
sumOfSquaredWeights = UpdateWeights();
// Calculate the new error
sumOfSquaredErrors = 0.0;
for (int i = 0; i < this.trainingLength; i++)
{
this.indexableTraining.GetRecord(i, this.pair);
INeuralData actual = this.network.Compute(this.pair
.Input);
double e = this.pair.Ideal[0]
- actual[0];
sumOfSquaredErrors += e * e;
}
sumOfSquaredErrors /= 2.0;
// Update the objective function
current = this.beta * sumOfSquaredErrors + this.alpha
* sumOfSquaredWeights;
// If the object function is bigger than before, the method
// is tried again using a greater dumping factor.
}
// If this iteration caused a error drop, then next iteration
// will use a smaller damping factor.
this.lambda /= LevenbergMarquardtTraining.SCALE_LAMBDA;
if (useBayesianRegularization && decomposition != null)
{
// Compute the trace for the inverse Hessian
trace = Trace(decomposition.Inverse());
// Poland update's formula:
gamma = this.parametersLength - (alpha * trace);
alpha = this.parametersLength / (2.0 * sumOfSquaredWeights + trace);
beta = Math.Abs((this.trainingLength - gamma) / (2.0 * sumOfSquaredErrors));
}
this.Error = sumOfSquaredErrors;
PostIteration();
}
/// <summary>
/// Perform one iteration.
/// </summary>
///
public override void Iteration()
{
LUDecomposition decomposition = null;
PreIteration();
_weights = NetworkCODEC.NetworkToArray(_network);
IComputeJacobian j = new JacobianChainRule(_network,
_indexableTraining);
double sumOfSquaredErrors = j.Calculate(_weights);
double sumOfSquaredWeights = CalculateSumOfSquaredWeights();
// this.setError(j.getError());
CalculateHessian(j.Jacobian, j.RowErrors);
// Define the objective function
// bayesian regularization objective function
double objective = _beta * sumOfSquaredErrors + _alpha
* sumOfSquaredWeights;
double current = objective + 1.0d;
// Start the main Levenberg-Macquardt method
_lambda /= ScaleLambda;
// We'll try to find a direction with less error
// (or where the objective function is smaller)
while ((current >= objective) &&
(_lambda < LambdaMax))
{
_lambda *= ScaleLambda;
// Update diagonal (Levenberg-Marquardt formula)
for (int i = 0; i < _parametersLength; i++)
{
_hessian[i][i] = _diagonal[i]
+ (_lambda + _alpha);
}
// Decompose to solve the linear system
decomposition = new LUDecomposition(_hessianMatrix);
// Check if the Jacobian has become non-invertible
if (!decomposition.IsNonsingular)
{
continue;
}
// Solve using LU (or SVD) decomposition
_deltas = decomposition.Solve(_gradient);
// Update weights using the calculated deltas
sumOfSquaredWeights = UpdateWeights();
// Calculate the new error
sumOfSquaredErrors = 0.0d;
for (int i = 0; i < _trainingLength; i++)
{
_indexableTraining.GetRecord(i, _pair);
IMLData actual = _network
.Compute(_pair.Input);
double e = _pair.Ideal[0]
- actual[0];
sumOfSquaredErrors += e * e;
}
sumOfSquaredErrors /= 2.0d;
// Update the objective function
current = _beta * sumOfSquaredErrors + _alpha
* sumOfSquaredWeights;
// If the object function is bigger than before, the method
// is tried again using a greater dumping factor.
}
// If this iteration caused a error drop, then next iteration
// will use a smaller damping factor.
_lambda /= ScaleLambda;
if (_useBayesianRegularization && (decomposition != null))
{
// Compute the trace for the inverse Hessian
double trace = Trace(decomposition.Inverse());
// Poland update's formula:
_gamma = _parametersLength - (_alpha * trace);
_alpha = _parametersLength
/ (2.0d * sumOfSquaredWeights + trace);
_beta = Math.Abs((_trainingLength - _gamma)
/ (2.0d * sumOfSquaredErrors));
}
Error = sumOfSquaredErrors;
PostIteration();
}
