public void SolveTest1()
{
double[,] value =
{
{ 2, 3, 0 },
{ -1, 2, 1 },
{ 0, -1, 3 }
};
double[] rhs = { 5, 0, 1 };
double[] expected =
{
1.6522,
0.5652,
0.5217,
};
var target = new LuDecomposition(value);
double[] actual = target.Solve(rhs);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-3));
Assert.IsTrue(Matrix.IsEqual(value, target.Reverse()));
var target2 = new JaggedLuDecomposition(value.ToJagged());
actual = target2.Solve(rhs);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-3));
Assert.IsTrue(Matrix.IsEqual(value, target2.Reverse()));
}
/// <summary>
/// Creates a new object that is a copy of the current instance.
/// </summary>
/// <returns>
/// A new object that is a copy of this instance.
/// </returns>
///
public object Clone()
{
LuDecomposition lud = new LuDecomposition();
lud.rows = this.rows;
lud.cols = this.cols;
lud.lu = (Double[, ]) this.lu.Clone();
lud.pivotSign = this.pivotSign;
lud.pivotVector = (int[])this.pivotVector;
return(lud);
}
public void InverseTestNaN()
{
int n = 5;
var I = Matrix.Identity(n);
for (int i = 0; i < n; i++)
{
for (int j = 0; j < n; j++)
{
double[,] value = Matrix.Magic(n);
value[i, j] = double.NaN;
var target = new LuDecomposition(value);
Assert.IsTrue(Matrix.IsEqual(target.Solve(I), target.Inverse()));
var target2 = new JaggedLuDecomposition(value.ToJagged());
Assert.IsTrue(Matrix.IsEqual(target2.Solve(I.ToJagged()), target2.Inverse()));
}
}
}
public void InverseTestNaN()
{
int n = 5;
var I = Matrix.Identity(n);
for (int i = 0; i < n; i++)
{
for (int j = 0; j < n; j++)
{
double[,] value = Matrix.Magic(n);
value[i, j] = double.NaN;
var target = new LuDecomposition(value);
var solution = target.Solve(I);
var inverse = target.Inverse();
Assert.IsTrue(Matrix.IsEqual(solution, inverse));
}
}
}
public void SolveTest1()
{
double[,] value =
{
{ 2, 3, 0 },
{ -1, 2, 1 },
{ 0, -1, 3 }
};
double[] rhs = { 5, 0, 1 };
double[] expected =
{
1.6522,
0.5652,
0.5217,
};
LuDecomposition target = new LuDecomposition(value);
double[] actual = target.Solve(rhs);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001));
}
public void SolveTest()
{
double[,] value =
{
{ 2, 3, 0 },
{ -1, 2, 1 },
{ 0, -1, 3 }
};
double[,] rhs =
{
{ 1, 2, 3 },
{ 3, 2, 1 },
{ 5, 0, 1 },
};
double[,] expected =
{
{ -0.2174, -0.1739, 0.6522 },
{ 0.4783, 0.7826, 0.5652 },
{ 1.8261, 0.2609, 0.5217 },
};
LuDecomposition target = new LuDecomposition(value);
double[,] actual = target.Solve(rhs);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001));
}
public void DeterminantTest()
{
double[,] value =
{
{ 2, 3, 0 },
{ -1, 2, 1 },
{ 0, -1, 3 }
};
LuDecomposition lu = new LuDecomposition(value);
Assert.AreEqual(23, lu.Determinant);
Assert.IsTrue(lu.Nonsingular);
}
public void LogDeterminantTest2()
{
double[,] value =
{
{ 2, 3, 0 },
{ -1, 2, 1 },
{ 0, -1, 3 }
};
LuDecomposition lu = new LuDecomposition(value);
Assert.AreEqual(23, lu.Determinant);
double expected = System.Math.Log(23);
double actual = lu.LogDeterminant;
Assert.AreEqual(expected, actual);
}
public void LuDecompositionConstructorTest()
{
double[,] value =
{
{ 2, -1, 0 },
{ -1, 2, -1 },
{ 0, -1, 2 }
};
double[,] expectedL =
{
{ 1.0000, 0, 0 },
{ -0.5000, 1.0000, 0 },
{ 0, -0.6667, 1.0000 },
};
double[,] expectedU =
{
{ 2.0000, -1.0000, 0 },
{ 0, 1.5000, -1.0000 },
{ 0, 0, 1.3333 },
};
LuDecomposition target = new LuDecomposition(value);
double[,] actualL = target.LowerTriangularFactor;
double[,] actualU = target.UpperTriangularFactor;
Assert.IsTrue(Matrix.IsEqual(expectedL, actualL, 0.001));
Assert.IsTrue(Matrix.IsEqual(expectedU, actualU, 0.001));
target = new LuDecomposition(value.Transpose(), true);
actualL = target.LowerTriangularFactor;
actualU = target.UpperTriangularFactor;
Assert.IsTrue(Matrix.IsEqual(expectedL, actualL, 0.001));
Assert.IsTrue(Matrix.IsEqual(expectedU, actualU, 0.001));
}
public void LogDeterminantTest()
{
LuDecomposition lu = new LuDecomposition(CholeskyDecompositionTest.bigmatrix);
Assert.AreEqual(0, lu.Determinant);
Assert.AreEqual(-2224.8931093738875, lu.LogDeterminant, 1e-12);
Assert.IsTrue(lu.Nonsingular);
}
public void SolveTest5()
{
double[,] value =
{
{ 2.1, 3.1 },
{ 1.6, 4.2 },
{ 2.1, 5.1 },
};
double[] rhs = { 6.1, 4.3, 2.1 };
double[] expected = { 3.1839, -0.1891 };
LuDecomposition target = new LuDecomposition(value);
bool thrown = false;
try
{
double[] actual = target.Solve(rhs);
}
catch (InvalidOperationException)
{
thrown = true;
}
Assert.IsTrue(thrown);
}
public void SolveTransposeTest()
{
double[,] a =
{
{ 2, 1, 4 },
{ 6, 2, 2 },
{ 0, 1, 6 },
};
double[,] b =
{
{ 1, 0, 7 },
{ 5, 2, 1 },
{ 1, 5, 2 },
};
double[,] expected =
{
{ 0.5062, 0.2813, 0.0875 },
{ 0.1375, 1.1875, -0.0750 },
{ 0.8063, -0.2188, 0.2875 },
};
double[,] actual = new LuDecomposition(b, true).SolveTranspose(a);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001));
}
public void SolveTest4()
{
double[,] value =
{
{ 2.1, 3.1 },
{ 1.6, 4.2 },
};
double[] rhs = { 6.1, 4.3 };
double[] expected = { 3.1839, -0.1891 };
var target1 = new LuDecomposition(value);
var target2 = new JaggedLuDecomposition(value.ToJagged());
Assert.IsTrue(Matrix.IsEqual(expected, target1.Solve(rhs), 1e-3));
Assert.IsTrue(Matrix.IsEqual(expected, target2.Solve(rhs), 1e-3));
}
public void SolveTest4()
{
double[,] value =
{
{ 2.1, 3.1 },
{ 1.6, 4.2 },
};
double[] rhs = { 6.1, 4.3 };
double[] expected = { 3.1839, -0.1891 };
LuDecomposition target = new LuDecomposition(value);
double[] actual = target.Solve(rhs);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001));
}
public void InverseTest()
{
double[,] value =
{
{ 2, 3, 0 },
{ -1, 2, 1 },
{ 0, -1, 3 }
};
double[,] expectedInverse =
{
{ 0.3043, -0.3913, 0.1304 },
{ 0.1304, 0.2609, -0.0870 },
{ 0.0435, 0.0870, 0.3043 },
};
LuDecomposition target = new LuDecomposition(value);
double[,] actualInverse = target.Inverse();
Assert.IsTrue(Matrix.IsEqual(expectedInverse, actualInverse, 0.001));
}
static void Main(string[] args)
{
Console.WriteLine("Please take a look at the source for examples!");
Console.ReadKey();
#region 1. Declaring matrices
// 1.1 Using standard .NET declaration
double[,] A =
{
{1, 2, 3},
{6, 2, 0},
{0, 0, 1}
};
double[,] B =
{
{2, 0, 0},
{0, 2, 0},
{0, 0, 2}
};
{
// 1.2 Using Accord extension methods
double[,] Bi = Matrix.Identity(3).Multiply(2);
double[,] Bj = Matrix.Diagonal(3, 2.0); // both are equal to B
// 1.2 Using Accord extension methods with implicit typing
var I = Matrix.Identity(3);
}
#endregion
#region 2. Matrix Operations
{
// 2.1 Addition
var C = A.Add(B);
// 2.2 Subtraction
var D = A.Subtract(B);
// 2.3 Multiplication
{
// 2.3.1 By a scalar
var halfM = A.Multiply(0.5);
// 2.3.2 By a vector
double[] m = A.Multiply(new double[] { 1, 2, 3 });
// 2.3.3 By a matrix
var M = A.Multiply(B);
// 2.4 Transposing
var At = A.Transpose();
}
}
// 2.5 Elementwise operations
// 2.5.1 Elementwise multiplication
A.ElementwiseMultiply(B); // A.*B
// 2.5.1 Elementwise division
A.ElementwiseDivide(B); // A./B
#endregion
#region 3. Matrix characteristics
{
// 3.1 Calculating the determinant
double det = A.Determinant();
// 3.2 Calculating the trace
double tr = A.Trace();
// 3.3 Computing the sum vector
{
double[] sumVector = A.Sum();
// 3.3.1 Computing the total sum of elements
double sum = sumVector.Sum();
// 3.3.2 Computing the sum along the rows
sumVector = A.Sum(0); // Equivalent to Octave's sum(A, 1)
// 3.3.2 Computing the sum along the columns
sumVector = A.Sum(1); // Equivalent to Octave's sum(A, 2)
}
}
#endregion
#region 4. Linear Algebra
{
// 4.1 Computing the inverse
var invA = A.Inverse();
// 4.2 Computing the pseudo-inverse
var pinvA = A.PseudoInverse();
// 4.3 Solving a linear system (Ax = B)
var x = A.Solve(B);
}
#endregion
#region 5. Special operators
{
// 5.1 Finding the indices of elements
double[] v = { 5, 2, 2, 7, 1, 0 };
int[] idx = v.Find(e => e > 2); // finding the index of every element in v higher than 2.
// 5.2 Selecting elements by index
double[] u = v.Submatrix(idx); // u is { 5, 7 }
// 5.3 Converting between different matrix representations
double[][] jaggedA = A.ToArray(); // from multidimensional to jagged array
// 5.4 Extracting a column or row from the matrix
double[] a = A.GetColumn(0); // retrieves the first column
double[] b = B.GetRow(1); // retrieves the second row
// 5.5 Taking the absolute of a matrix
var absA = A.Abs();
// 5.6 Applying some function to every element
var newv = v.Apply(e => e + 1);
}
#endregion
#region 7. Vector operations
{
double[] u = { 1, 2, 3 };
double[] v = { 4, 5, 6 };
var w1 = u.InnerProduct(v);
var w2 = u.OuterProduct(v);
var w3 = u.CartesianProduct(v);
double[] m = { 1, 2, 3, 4 };
double[,] M = Matrix.Reshape(m, 2, 2);
}
#endregion
#region Decompositions
{
// Singular value decomposition
{
SingularValueDecomposition svd = new SingularValueDecomposition(A);
var U = svd.LeftSingularVectors;
var S = svd.Diagonal;
var V = svd.RightSingularVectors;
}
// or (please see documentation for details)
{
SingularValueDecomposition svd = new SingularValueDecomposition(A.Transpose());
var U = svd.RightSingularVectors;
var S = svd.Diagonal;
var V = svd.LeftSingularVectors;
}
// Eigenvalue decomposition
{
EigenvalueDecomposition eig = new EigenvalueDecomposition(A);
var V = eig.Eigenvectors;
var D = eig.DiagonalMatrix;
}
// QR decomposition
{
QrDecomposition qr = new QrDecomposition(A);
var Q = qr.OrthogonalFactor;
var R = qr.UpperTriangularFactor;
}
// Cholesky decomposition
{
CholeskyDecomposition chol = new CholeskyDecomposition(A);
var R = chol.LeftTriangularFactor;
}
// LU decomposition
{
LuDecomposition lu = new LuDecomposition(A);
var L = lu.LowerTriangularFactor;
var U = lu.UpperTriangularFactor;
}
}
#endregion
}
public void SolveTransposeTest()
{
double[,] a =
{
{ 2, 1, 4 },
{ 6, 2, 2 },
{ 0, 1, 6 },
};
double[,] b =
{
{ 1, 0, 7 },
{ 5, 2, 1 },
{ 1, 5, 2 },
};
double[,] expected =
{
{ 0.5062, 0.2813, 0.0875 },
{ 0.1375, 1.1875, -0.0750 },
{ 0.8063, -0.2188, 0.2875 },
};
Assert.IsTrue(Matrix.IsEqual(expected, new LuDecomposition(b, true).SolveTranspose(a), 1e-3));
Assert.IsTrue(Matrix.IsEqual(expected, new JaggedLuDecomposition(b.ToJagged(), true).SolveTranspose(a.ToJagged()), 1e-3));
var target = new LuDecomposition(b, true);
var p = target.PivotPermutationVector;
int[] idx = p.ArgSort();
var r = target.LowerTriangularFactor.Dot(target.UpperTriangularFactor)
.Submatrix(idx, null).Transpose();
Assert.IsTrue(Matrix.IsEqual(b, r, 1e-3));
Assert.IsTrue(Matrix.IsEqual(b.Transpose(), target.Reverse(), 1e-3));
Assert.IsTrue(Matrix.IsEqual(b.Transpose(), new JaggedLuDecomposition(b.ToJagged(), true).Reverse(), 1e-3));
}
private double run(double[][] inputs, double[][] outputs)
{
// Regress using Lower-Bound Newton-Raphson estimation
//
// The main idea is to replace the Hessian matrix with a
// suitable lower bound. Indeed, the Hessian is lower
// bounded by a negative definite matrix that does not
// even depend on w [Krishnapuram et al].
//
// - http://www.lx.it.pt/~mtf/Krishnapuram_Carin_Figueiredo_Hartemink_2005.pdf
//
// Initial definitions and memory allocations
int N = inputs.Length;
double[][] design = new double[N][];
double[][] coefficients = this.regression.Coefficients;
// Compute the regression matrix
for (int i = 0; i < inputs.Length; i++)
{
double[] row = design[i] = new double[M];
row[0] = 1; // for intercept
for (int j = 0; j < inputs[i].Length; j++)
row[j + 1] = inputs[i][j];
}
// Reset Hessian matrix and gradient
for (int i = 0; i < gradient.Length; i++)
gradient[i] = 0;
if (UpdateLowerBound)
{
for (int i = 0; i < gradient.Length; i++)
for (int j = 0; j < gradient.Length; j++)
lowerBound[i, j] = 0;
}
// In the multinomial logistic regression, the objective
// function is the log-likelihood function l(w). As given
// by Krishnapuram et al and Böhning, this is a concave
// function with Hessian given by:
//
// H(w) = -sum(P(w) - p(w)p(w)') (x) xx'
// (see referenced paper for proper indices)
//
// In which (x) denotes the Kronocker product. By using
// the lower bound principle, Krishnapuram has shown that
// we can replace H(w) with a lower bound approximation B
// which does not depend on w (eq. 8 on aforementined paper):
//
// B = -(1/2) [I - 11/M] (x) sum(xx')
//
// Thus we can compute and invert this matrix only once.
//
// For each input sample in the dataset
for (int i = 0; i < inputs.Length; i++)
{
// Grab variables related to the sample
double[] x = design[i];
double[] y = outputs[i];
// Compute and estimate outputs
this.compute(inputs[i], output);
// Compute errors for the sample
for (int j = 0; j < errors.Length; j++)
errors[j] = y[j + 1] - output[j];
// Compute current gradient and Hessian
// We can take advantage of the block structure of the
// Hessian matrix and gradient vector by employing the
// Kronocker product. See [Böhning, 1992]
// (Re-) Compute error gradient
double[] g = Matrix.KroneckerProduct(errors, x);
for (int j = 0; j < g.Length; j++)
gradient[j] += g[j];
if (UpdateLowerBound)
{
// Compute xxt matrix
for (int k = 0; k < x.Length; k++)
for (int j = 0; j < x.Length; j++)
xxt[k, j] = x[k] * x[j];
// (Re-) Compute weighted "Hessian" matrix
double[,] h = Matrix.KroneckerProduct(weights, xxt);
for (int j = 0; j < parameterCount; j++)
for (int k = 0; k < parameterCount; k++)
lowerBound[j, k] += h[j, k];
}
}
if (UpdateLowerBound)
{
UpdateLowerBound = false;
// Decompose to solve the linear system. Usually the hessian will
// be invertible and LU will succeed. However, sometimes the hessian
// may be singular and a Singular Value Decomposition may be needed.
LuDecomposition lu = new LuDecomposition(lowerBound);
// The SVD is very stable, but is quite expensive, being on average
// about 10-15 times more expensive than LU decomposition. There are
// other ways to avoid a singular Hessian. For a very interesting
// reading on the subject, please see:
//
// - Jeff Gill & Gary King, "What to Do When Your Hessian Is Not Invertible",
// Sociological Methods & Research, Vol 33, No. 1, August 2004, 54-87.
// Available in: http://gking.harvard.edu/files/help.pdf
//
// Moreover, the computation of the inverse is optional, as it will
// be used only to compute the standard errors of the regression.
if (lu.Nonsingular)
{
// Solve using LU decomposition
deltas = lu.Solve(gradient);
decomposition = lu;
}
else
{
// Hessian Matrix is singular, try pseudo-inverse solution
decomposition = new SingularValueDecomposition(lowerBound);
deltas = decomposition.Solve(gradient);
}
}
else
{
deltas = decomposition.Solve(gradient);
}
previous = coefficients.Reshape(1);
// Update coefficients using the calculated deltas
for (int i = 0, k = 0; i < coefficients.Length; i++)
for (int j = 0; j < coefficients[i].Length; j++)
coefficients[i][j] -= deltas[k++];
solution = coefficients.Reshape(1);
if (computeStandardErrors)
{
// Grab the regression information matrix
double[,] inverse = decomposition.Inverse();
// Calculate coefficients' standard errors
double[][] standardErrors = regression.StandardErrors;
for (int i = 0, k = 0; i < standardErrors.Length; i++)
for (int j = 0; j < standardErrors[i].Length; j++, k++)
standardErrors[i][j] = Math.Sqrt(Math.Abs(inverse[k, k]));
}
// Return the relative maximum parameter change
for (int i = 0; i < deltas.Length; i++)
deltas[i] = Math.Abs(deltas[i]) / Math.Abs(previous[i]);
return Matrix.Max(deltas);
}
/// <summary>
/// Runs one iteration of the Reweighted Least Squares algorithm.
/// </summary>
/// <param name="inputs">The input data.</param>
/// <param name="outputs">The outputs associated with each input vector.</param>
/// <returns>The maximum relative change in the parameters after the iteration.</returns>
///
public double Run(double[][] inputs, double[] outputs)
{
// Regress using Iteratively Reweighted Least Squares estimation.
// References:
// - Bishop, Christopher M.; Pattern Recognition
// and Machine Learning. Springer; 1st ed. 2006.
// Initial definitions and memory allocations
int N = inputs.Length;
double[][] design = new double[N][];
double[] errors = new double[N];
double[] weights = new double[N];
double[] coefficients = this.regression.Coefficients;
double[] deltas;
// Compute the regression matrix
for (int i = 0; i < inputs.Length; i++)
{
double[] row = design[i] = new double[parameterCount];
row[0] = 1; // for intercept
for (int j = 0; j < inputs[i].Length; j++)
row[j + 1] = inputs[i][j];
}
// Compute errors and weighing matrix
for (int i = 0; i < inputs.Length; i++)
{
double y = regression.Compute(inputs[i]);
// Calculate error vector
errors[i] = y - outputs[i];
// Calculate weighting matrix
weights[i] = y * (1.0 - y);
}
// Reset Hessian matrix and gradient
for (int i = 0; i < gradient.Length; i++)
{
gradient[i] = 0;
for (int j = 0; j < gradient.Length; j++)
hessian[i, j] = 0;
}
// (Re-) Compute error gradient
for (int j = 0; j < design.Length; j++)
for (int i = 0; i < gradient.Length; i++)
gradient[i] += design[j][i] * errors[j];
// (Re-) Compute weighted "Hessian" matrix
for (int k = 0; k < weights.Length; k++)
{
double[] rk = design[k];
for (int j = 0; j < rk.Length; j++)
for (int i = 0; i < rk.Length; i++)
hessian[j, i] += rk[i] * rk[j] * weights[k];
}
// Decompose to solve the linear system. Usually the hessian will
// be invertible and LU will succeed. However, sometimes the hessian
// may be singular and a Singular Value Decomposition may be needed.
LuDecomposition lu = new LuDecomposition(hessian);
// The SVD is very stable, but is quite expensive, being on average
// about 10-15 times more expensive than LU decomposition. There are
// other ways to avoid a singular Hessian. For a very interesting
// reading on the subject, please see:
//
// - Jeff Gill & Gary King, "What to Do When Your Hessian Is Not Invertible",
// Sociological Methods & Research, Vol 33, No. 1, August 2004, 54-87.
// Available in: http://gking.harvard.edu/files/help.pdf
//
// Moreover, the computation of the inverse is optional, as it will
// be used only to compute the standard errors of the regression.
if (lu.Nonsingular)
{
// Solve using LU decomposition
deltas = lu.Solve(gradient);
decomposition = lu;
}
else
{
// Hessian Matrix is singular, try pseudo-inverse solution
decomposition = new SingularValueDecomposition(hessian);
deltas = decomposition.Solve(gradient);
}
previous = (double[])coefficients.Clone();
// Update coefficients using the calculated deltas
for (int i = 0; i < coefficients.Length; i++)
coefficients[i] -= deltas[i];
if (computeStandardErrors)
{
// Grab the regression information matrix
double[,] inverse = decomposition.Inverse();
// Calculate coefficients' standard errors
double[] standardErrors = regression.StandardErrors;
for (int i = 0; i < standardErrors.Length; i++)
standardErrors[i] = Math.Sqrt(inverse[i, i]);
}
// Return the relative maximum parameter change
for (int i = 0; i < deltas.Length; i++)
deltas[i] = Math.Abs(deltas[i]) / Math.Abs(previous[i]);
return Matrix.Max(deltas);
}
public void SolveTest3()
{
double[,] value =
{
{ 2.000, 3.000, 0.000 },
{ -1.000, 2.000, 1.000 },
};
LuDecomposition target = new LuDecomposition(value);
double[,] L = target.LowerTriangularFactor;
double[,] U = target.UpperTriangularFactor;
double[,] expectedL =
{
{ 1.000, 0.000 },
{ -0.500, 1.000 },
};
double[,] expectedU =
{
{ 2.000, 3.000, 0.000 },
{ 0.000, 3.500, 1.000 },
};
Assert.IsTrue(Matrix.IsEqual(expectedL, L, 0.001));
Assert.IsTrue(Matrix.IsEqual(expectedU, U, 0.001));
}
/// <summary>
/// Creates a new object that is a copy of the current instance.
/// </summary>
/// <returns>
/// A new object that is a copy of this instance.
/// </returns>
///
public object Clone()
{
LuDecomposition lud = new LuDecomposition();
lud.rows = this.rows;
lud.cols = this.cols;
lud.lu = (double[,])this.lu.Clone();
lud.pivotSign = this.pivotSign;
lud.pivotVector = (int[])this.pivotVector;
return lud;
}
