public void Simple()
{
var matrixA = new Matrix(2, 2);
matrixA[0, 0] = 0;
matrixA[0, 1] = 1;
matrixA[1, 0] = 2;
matrixA[1, 1] = 0;
var matrixB = new Matrix(2, 2);
matrixB[0, 0] = 1;
matrixB[0, 1] = 0;
matrixB[1, 0] = 0;
matrixB[1, 1] = -1;
var decomposition = new LUDecomposition(matrixA);
var solveMatrix = decomposition.Solve(matrixB);
Assert.AreEqual(solveMatrix.Rows, 2);
Assert.AreEqual(solveMatrix.Columns, 2);
Assert.AreEqual(solveMatrix[0, 0], 0);
Assert.AreEqual(solveMatrix[0, 1], -0.5);
Assert.AreEqual(solveMatrix[1, 0], 1);
Assert.AreEqual(solveMatrix[1, 1], 0);
}
public static void LUDecomposition()
{
SquareMatrix A = new SquareMatrix(new double[, ] {
{ 1, -2, 3 },
{ 2, -5, 12 },
{ 0, 2, -10 }
});
ColumnVector b = new ColumnVector(2, 8, -4);
LUDecomposition lud = A.LUDecomposition();
ColumnVector x = lud.Solve(b);
PrintMatrix("x", x);
PrintMatrix("Ax", A * x);
SquareMatrix L = lud.LMatrix();
SquareMatrix U = lud.UMatrix();
SquareMatrix P = lud.PMatrix();
PrintMatrix("LU", L * U);
PrintMatrix("PA", P * A);
SquareMatrix AI = lud.Inverse();
PrintMatrix("A * AI", A * AI);
Console.WriteLine($"det(a) = {lud.Determinant()}");
}
public void SquareRandomMatrixLUDecomposition()
{
for (int d = 1; d <= 256; d += 11)
{
SquareMatrix M = CreateSquareRandomMatrix(d);
// LU decompose the matrix
//Stopwatch sw = Stopwatch.StartNew();
LUDecomposition LU = M.LUDecomposition();
//sw.Stop();
//Console.WriteLine(sw.ElapsedMilliseconds);
Assert.IsTrue(LU.Dimension == d);
// test that the decomposition works
SquareMatrix P = LU.PMatrix();
SquareMatrix L = LU.LMatrix();
SquareMatrix U = LU.UMatrix();
Assert.IsTrue(TestUtilities.IsNearlyEqual(P * M, L * U));
// check that the inverse works
SquareMatrix MI = LU.Inverse();
Assert.IsTrue(TestUtilities.IsNearlyEqual(M * MI, UnitMatrix.OfDimension(d)));
// test that a solution works
ColumnVector t = new ColumnVector(d);
for (int i = 0; i < d; i++)
{
t[i] = i;
}
ColumnVector s = LU.Solve(t);
Assert.IsTrue(TestUtilities.IsNearlyEqual(M * s, t));
}
}
public void ExceptionDifferentRowCounts()
{
var matrixA = new Matrix(2, 2);
matrixA[0, 0] = 0;
matrixA[0, 1] = 1;
matrixA[1, 0] = 2;
matrixA[1, 1] = 0;
var matrixB = new Matrix(3, 2);
matrixB[0, 0] = 1;
matrixB[0, 1] = 0;
matrixB[1, 0] = 0;
matrixB[1, 1] = -1;
matrixB[2, 0] = 1;
matrixB[2, 1] = 3;
var decomposition = new LUDecomposition(matrixA);
Assert.Throws <ArgumentException>(() => decomposition.Solve(matrixB));
}
/// <inheritdoc />
public override void Iteration()
{
if (!_initComplete)
{
_hessian.Init(_network, Training);
_initComplete = true;
}
PreIteration();
_hessian.Clear();
_weights = NetworkCODEC.NetworkToArray(_network);
_hessian.Compute();
double currentError = _hessian.SSE;
SaveDiagonal();
double startingError = currentError;
bool done = false;
bool singular;
while (!done)
{
ApplyLambda();
var decomposition = new LUDecomposition(_hessian.HessianMatrix);
singular = decomposition.IsNonsingular;
if (singular)
{
_deltas = decomposition.Solve(_hessian.Gradients);
UpdateWeights();
currentError = CalculateError();
}
if (!singular || currentError >= startingError)
{
_lambda *= ScaleLambda;
if (_lambda > LambdaMax)
{
_lambda = LambdaMax;
done = true;
}
}
else
{
_lambda /= ScaleLambda;
done = true;
}
}
Error = currentError;
PostIteration();
}
public void SquareVandermondeMatrixLUDecomposition()
{
// fails now for d = 8 because determinant slightly off
for (int d = 1; d <= 7; d++)
{
Console.WriteLine("d={0}", d);
double[] x = new double[d];
for (int i = 0; i < d; i++)
{
x[i] = i;
}
double det = 1.0;
for (int i = 0; i < d; i++)
{
for (int j = 0; j < i; j++)
{
det = det * (x[i] - x[j]);
}
}
// LU decompose the matrix
SquareMatrix V = CreateVandermondeMatrix(d);
LUDecomposition LU = V.LUDecomposition();
// test that the decomposition works
SquareMatrix P = LU.PMatrix();
SquareMatrix L = LU.LMatrix();
SquareMatrix U = LU.UMatrix();
Assert.IsTrue(TestUtilities.IsNearlyEqual(P * V, L * U));
// check that the determinant agrees with the analytic expression
Console.WriteLine("det {0} {1}", LU.Determinant(), det);
Assert.IsTrue(TestUtilities.IsNearlyEqual(LU.Determinant(), det));
// check that the inverse works
SquareMatrix VI = LU.Inverse();
//PrintMatrix(VI);
//PrintMatrix(V * VI);
SquareMatrix I = TestUtilities.CreateSquareUnitMatrix(d);
Assert.IsTrue(TestUtilities.IsNearlyEqual(V * VI, I));
// test that a solution works
ColumnVector t = new ColumnVector(d);
for (int i = 0; i < d; i++)
{
t[i] = 1.0;
}
ColumnVector s = LU.Solve(t);
Assert.IsTrue(TestUtilities.IsNearlyEqual(V * s, t));
}
}
/// <summary>
/// Perform one iteration.
/// </summary>
public override void Iteration()
{
LUDecomposition decomposition;
PreIteration();
_hessian.Clear();
_weights = NetworkCODEC.NetworkToArray(_network);
_hessian.Compute();
double currentError = _hessian.SSE;
SaveDiagonal();
double startingError = currentError;
bool done = false;
while (!done)
{
ApplyLambda();
decomposition = new LUDecomposition(_hessian.HessianMatrix);
if (decomposition.IsNonsingular)
{
_deltas = decomposition.Solve(_hessian.Gradients);
UpdateWeights();
currentError = CalculateError();
if (currentError < startingError)
{
_lambda /= LevenbergMarquardtTraining.ScaleLambda;
done = true;
}
}
if (!done)
{
_lambda *= LevenbergMarquardtTraining.ScaleLambda;
if (_lambda > LevenbergMarquardtTraining.LambdaMax)
{
_lambda = LevenbergMarquardtTraining.LambdaMax;
done = true;
}
}
}
Error = currentError;
PostIteration();
}
public static Double[,] Inverse(Double[,] matrix)
{
var rows = matrix.GetLength(0);
var cols = matrix.GetLength(1);
var target = Helpers.One(cols);
if (cols < 24)
{
var lu = new LUDecomposition(matrix);
return(lu.Solve(target));
}
else if (Helpers.IsSymmetric(matrix))
{
var cho = new CholeskyDecomposition(matrix);
return(cho.Solve(target));
}
else
{
var qr = QRDecomposition.Create(matrix);
return(qr.Solve(target));
}
}
public void ExceptionDifferentRowCounts()
{
var matrixA = new Matrix(2, 2);
matrixA[0, 0] = 0;
matrixA[0, 1] = 1;
matrixA[1, 0] = 2;
matrixA[1, 1] = 0;
var matrixB = new Matrix(3, 2);
matrixB[0, 0] = 1;
matrixB[0, 1] = 0;
matrixB[1, 0] = 0;
matrixB[1, 1] = -1;
matrixB[2, 0] = 1;
matrixB[2, 1] = 3;
var decomposition = new LUDecomposition(matrixA);
var solveMatrix = decomposition.Solve(matrixB);
}
/// <summary>
/// Finds a vector argument which makes a vector function zero.
/// </summary>
/// <param name="f">The vector function.</param>
/// <param name="x0">The vector argument.</param>
/// <returns>The vector argument which makes all components of the vector function zero.</returns>
/// <exception cref="ArgumentNullException"><paramref name="f"/> or <paramref name="x0"/> is null.</exception>
/// <exception cref="DimensionMismatchException">The dimension of <paramref name="f"/> is not equal to the
/// dimension of <paramref name="x0"/>.</exception>
public static ColumnVector FindZero(Func <IList <double>, IList <double> > f, IList <double> x0)
{
if (f == null)
{
throw new ArgumentNullException("f");
}
if (x0 == null)
{
throw new ArgumentNullException("x0");
}
// we will use Broyden's method, which is a generalization of the secant method to the multi-dimensional problem
// just as the secant method is essentially Newton's method with a crude, numerical value for the slope,
// Broyden's method is a multi-dimensional Newton's method with a crude, numerical value for the Jacobian matrix
int d = x0.Count;
// we should re-engineer this code to work directly on vector/matrix storage rather than go back and forth
// between vectors and arrays, but for the moment this works; implementing Blas2 rank-1 update would help
// starting values
ColumnVector x = new ColumnVector(x0);
//double[] x = new double[d]; Blas1.dCopy(x0, 0, 1, x, 0, 1, d);
ColumnVector F = new ColumnVector(f(x.ToArray()));
//double[] F = f(x);
if (F.Dimension != d)
{
throw new DimensionMismatchException();
}
SquareMatrix B = ApproximateJacobian(f, x0);
double g = MoreMath.Pow2(F.Norm());
//double g = Blas1.dDot(F, 0, 1, F, 0, 1, d);
for (int n = 0; n < Global.SeriesMax; n++)
{
// determine the Newton step
LUDecomposition LU = B.LUDecomposition();
ColumnVector dx = -LU.Solve(F);
//for (int i = 0; i < d; i++) {
// Console.WriteLine("F[{0}]={1} x[{0}]={2} dx[{0}]={3}", i, F[i], x[i], dx[i]);
//}
// determine how far we will move along the Newton step
ColumnVector x1 = x + dx;
ColumnVector F1 = new ColumnVector(f(x1));
double g1 = MoreMath.Pow2(F1.Norm());
// check whether the Newton step decreases the function vector magnitude
// NR suggest that it's necessary to ensure that it decrease by a certain amount, but I have yet to see that make a diference
double gm = g - 0.0;
if (g1 > gm)
{
// the Newton step did not reduce the function vector magnitude, so we won't step that far
// determine how far along the descent direction we will step by parabolic interpolation
double z = g / (g + g1);
//Console.WriteLine("z={0}", z);
// take at least a small step in the descent direction
if (z < (1.0 / 16.0))
{
z = 1.0 / 16.0;
}
dx = z * dx;
x1 = x + dx;
F1 = new ColumnVector(f(x1.ToArray()));
g1 = MoreMath.Pow2(F1.Norm());
// NR suggest that this be repeated, but I have yet to see that make a difference
}
// take the step
x = x + dx;
// check for convergence
if ((F1.InfinityNorm() < Global.Accuracy) || (dx.InfinityNorm() < Global.Accuracy))
{
//if ((g1 < Global.Accuracy) || (dx.Norm() < Global.Accuracy)) {
return(x);
}
// update B
ColumnVector dF = F1 - F;
RectangularMatrix dB = (dF - B * dx) * (dx / (dx.Transpose() * dx)).Transpose();
//RectangularMatrix dB = F1 * ( dx / MoreMath.Pow(dx.Norm(), 2) ).Transpose();
for (int i = 0; i < d; i++)
{
for (int j = 0; j < d; j++)
{
B[i, j] += dB[i, j];
}
}
// prepare for the next iteration
F = F1;
g = g1;
}
throw new NonconvergenceException();
}
public override IVector Apply(IMatrix A, IVector b, IVector x)
{
double[] xs = (double[])decomp.Solve(
new Matrix(Blas.Default.GetArrayCopy(b), b.Length));
return(Blas.Default.SetVector(xs, x));
}
public void T01_BasicLinearEquations()
{
const double Accuracy = 2.2205e-15, SvdAccuracy = 4.46e-15, DeterminantAccuracy = 1.2e-13;
// given w=-1, x=2, y=-3, and z=4...
// w - 2x + 3y - 5z = -1 - 4 - 9 - 20 = -34
// w + 2x + 3y + 5z = -1 + 4 - 9 + 20 = 14
// 5w - 3x + 2y - z = -5 - 6 - 6 - 4 = -21
// 5w + 3x + 2y + z = -5 + 6 - 6 + 4 = -1
double[] coefficientArray = new double[16]
{
1, -2, 3, -5,
1, 2, 3, 5,
5, -3, 2, -1,
5, 3, 2, 1,
};
double[] valueArray = new double[4] {
-34, 14, -21, -1
};
// first solve using Gauss-Jordan elimination
Matrix4 coefficients = new Matrix4(coefficientArray);
Matrix gjInverse, values = new Matrix(valueArray, 1);
Matrix solution = GaussJordan.Solve(coefficients.ToMatrix(), values, out gjInverse);
CheckSolution(solution, Accuracy);
CheckSolution(gjInverse * values, Accuracy); // make sure multiplying by the matrix inverse also gives the right answer
// then solve using LU decomposition
LUDecomposition lud = new LUDecomposition(coefficients.ToMatrix());
solution = lud.Solve(values);
CheckSolution(solution, Accuracy);
CheckSolution(lud.GetInverse() * values, Accuracy); // check that the inverse can be multiplied to produce a good solution
solution.Multiply(1.1); // mess up the solution
lud.RefineSolution(values, solution); // test that refinement can fix it
CheckSolution(solution, Accuracy);
// check that the computed determinants are what we expect
bool negative;
Assert.AreEqual(coefficients.GetDeterminant(), lud.GetDeterminant(), DeterminantAccuracy);
Assert.AreEqual(Math.Log(Math.Abs(coefficients.GetDeterminant())), lud.GetLogDeterminant(out negative), Accuracy);
Assert.AreEqual(coefficients.GetDeterminant() < 0, negative);
// check that the inverses are about the same from both methods
Assert.IsTrue(gjInverse.Equals(lud.GetInverse(), Accuracy));
// then solve using QR decomposition
QRDecomposition qrd = new QRDecomposition(coefficients.ToMatrix());
solution = qrd.Solve(values);
CheckSolution(solution, Accuracy);
CheckSolution(qrd.GetInverse() * values, Accuracy); // check that the inverse can be multiplied to produce a good solution
// TODO: test qrd.Update()
// finally, try solving using singular value decomposition
SVDecomposition svd = new SVDecomposition(coefficients.ToMatrix());
solution = svd.Solve(values);
CheckSolution(solution, SvdAccuracy);
CheckSolution(svd.GetInverse() * values, SvdAccuracy);
Assert.IsTrue(gjInverse.Equals(svd.GetInverse(), Accuracy));
Assert.AreEqual(4, svd.GetRank());
Assert.AreEqual(0, svd.GetNullity());
}
/// <summary>
/// Perform one iteration.
/// </summary>
public override void Iteration()
{
LUDecomposition decomposition = null;
double trace = 0;
PreIteration();
this.weights = NetworkCODEC.NetworkToArray(this.network);
IComputeJacobian j = new JacobianChainRule(this.network,
this.indexableTraining);
double sumOfSquaredErrors = j.Calculate(this.weights);
double sumOfSquaredWeights = CalculateSumOfSquaredWeights();
// this.setError(j.getError());
CalculateHessian(j.Jacobian, j.RowErrors);
// Define the objective function
// bayesian regularization objective function
double objective = this.beta * sumOfSquaredErrors + this.alpha
* sumOfSquaredWeights;
double current = objective + 1.0;
// Start the main Levenberg-Macquardt method
this.lambda /= LevenbergMarquardtTraining.SCALE_LAMBDA;
// We'll try to find a direction with less error
// (or where the objective function is smaller)
while ((current >= objective) &&
(this.lambda < LevenbergMarquardtTraining.LAMBDA_MAX))
{
this.lambda *= LevenbergMarquardtTraining.SCALE_LAMBDA;
// Update diagonal (Levenberg-Marquardt formula)
for (int i = 0; i < this.parametersLength; i++)
{
this.hessian[i][i] = this.diagonal[i]
+ (this.lambda + this.alpha);
}
// Decompose to solve the linear system
decomposition = new LUDecomposition(
this.hessianMatrix);
// Check if the Jacobian has become non-invertible
if (!decomposition.IsNonsingular)
{
continue;
}
// Solve using LU (or SVD) decomposition
this.deltas = decomposition.Solve(this.gradient);
// Update weights using the calculated deltas
sumOfSquaredWeights = UpdateWeights();
// Calculate the new error
sumOfSquaredErrors = 0.0;
for (int i = 0; i < this.trainingLength; i++)
{
this.indexableTraining.GetRecord(i, this.pair);
INeuralData actual = this.network.Compute(this.pair
.Input);
double e = this.pair.Ideal[0]
- actual[0];
sumOfSquaredErrors += e * e;
}
sumOfSquaredErrors /= 2.0;
// Update the objective function
current = this.beta * sumOfSquaredErrors + this.alpha
* sumOfSquaredWeights;
// If the object function is bigger than before, the method
// is tried again using a greater dumping factor.
}
// If this iteration caused a error drop, then next iteration
// will use a smaller damping factor.
this.lambda /= LevenbergMarquardtTraining.SCALE_LAMBDA;
if (useBayesianRegularization && decomposition != null)
{
// Compute the trace for the inverse Hessian
trace = Trace(decomposition.Inverse());
// Poland update's formula:
gamma = this.parametersLength - (alpha * trace);
alpha = this.parametersLength / (2.0 * sumOfSquaredWeights + trace);
beta = Math.Abs((this.trainingLength - gamma) / (2.0 * sumOfSquaredErrors));
}
this.Error = sumOfSquaredErrors;
PostIteration();
}
/// <summary>
/// Perform one iteration.
/// </summary>
///
public override void Iteration()
{
LUDecomposition decomposition = null;
PreIteration();
_weights = NetworkCODEC.NetworkToArray(_network);
IComputeJacobian j = new JacobianChainRule(_network,
_indexableTraining);
double sumOfSquaredErrors = j.Calculate(_weights);
double sumOfSquaredWeights = CalculateSumOfSquaredWeights();
// this.setError(j.getError());
CalculateHessian(j.Jacobian, j.RowErrors);
// Define the objective function
// bayesian regularization objective function
double objective = _beta * sumOfSquaredErrors + _alpha
* sumOfSquaredWeights;
double current = objective + 1.0d;
// Start the main Levenberg-Macquardt method
_lambda /= ScaleLambda;
// We'll try to find a direction with less error
// (or where the objective function is smaller)
while ((current >= objective) &&
(_lambda < LambdaMax))
{
_lambda *= ScaleLambda;
// Update diagonal (Levenberg-Marquardt formula)
for (int i = 0; i < _parametersLength; i++)
{
_hessian[i][i] = _diagonal[i]
+ (_lambda + _alpha);
}
// Decompose to solve the linear system
decomposition = new LUDecomposition(_hessianMatrix);
// Check if the Jacobian has become non-invertible
if (!decomposition.IsNonsingular)
{
continue;
}
// Solve using LU (or SVD) decomposition
_deltas = decomposition.Solve(_gradient);
// Update weights using the calculated deltas
sumOfSquaredWeights = UpdateWeights();
// Calculate the new error
sumOfSquaredErrors = 0.0d;
for (int i = 0; i < _trainingLength; i++)
{
_indexableTraining.GetRecord(i, _pair);
IMLData actual = _network
.Compute(_pair.Input);
double e = _pair.Ideal[0]
- actual[0];
sumOfSquaredErrors += e * e;
}
sumOfSquaredErrors /= 2.0d;
// Update the objective function
current = _beta * sumOfSquaredErrors + _alpha
* sumOfSquaredWeights;
// If the object function is bigger than before, the method
// is tried again using a greater dumping factor.
}
// If this iteration caused a error drop, then next iteration
// will use a smaller damping factor.
_lambda /= ScaleLambda;
if (_useBayesianRegularization && (decomposition != null))
{
// Compute the trace for the inverse Hessian
double trace = Trace(decomposition.Inverse());
// Poland update's formula:
_gamma = _parametersLength - (_alpha * trace);
_alpha = _parametersLength
/ (2.0d * sumOfSquaredWeights + trace);
_beta = Math.Abs((_trainingLength - _gamma)
/ (2.0d * sumOfSquaredErrors));
}
Error = sumOfSquaredErrors;
PostIteration();
}
