public void SolveTest1() { double[,] value = { { 2, 3, 0 }, { -1, 2, 1 }, { 0, -1, 3 } }; double[] rhs = { 5, 0, 1 }; double[] expected = { 1.6522, 0.5652, 0.5217, }; var target = new LuDecomposition(value); double[] actual = target.Solve(rhs); Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-3)); Assert.IsTrue(Matrix.IsEqual(value, target.Reverse())); var target2 = new JaggedLuDecomposition(value.ToJagged()); actual = target2.Solve(rhs); Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-3)); Assert.IsTrue(Matrix.IsEqual(value, target2.Reverse())); }
/// <summary> /// Creates a new object that is a copy of the current instance. /// </summary> /// <returns> /// A new object that is a copy of this instance. /// </returns> /// public object Clone() { LuDecomposition lud = new LuDecomposition(); lud.rows = this.rows; lud.cols = this.cols; lud.lu = (Double[, ]) this.lu.Clone(); lud.pivotSign = this.pivotSign; lud.pivotVector = (int[])this.pivotVector; return(lud); }
public void InverseTestNaN() { int n = 5; var I = Matrix.Identity(n); for (int i = 0; i < n; i++) { for (int j = 0; j < n; j++) { double[,] value = Matrix.Magic(n); value[i, j] = double.NaN; var target = new LuDecomposition(value); Assert.IsTrue(Matrix.IsEqual(target.Solve(I), target.Inverse())); var target2 = new JaggedLuDecomposition(value.ToJagged()); Assert.IsTrue(Matrix.IsEqual(target2.Solve(I.ToJagged()), target2.Inverse())); } } }
public void InverseTestNaN() { int n = 5; var I = Matrix.Identity(n); for (int i = 0; i < n; i++) { for (int j = 0; j < n; j++) { double[,] value = Matrix.Magic(n); value[i, j] = double.NaN; var target = new LuDecomposition(value); var solution = target.Solve(I); var inverse = target.Inverse(); Assert.IsTrue(Matrix.IsEqual(solution, inverse)); } } }
public void SolveTest1() { double[,] value = { { 2, 3, 0 }, { -1, 2, 1 }, { 0, -1, 3 } }; double[] rhs = { 5, 0, 1 }; double[] expected = { 1.6522, 0.5652, 0.5217, }; LuDecomposition target = new LuDecomposition(value); double[] actual = target.Solve(rhs); Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001)); }
public void SolveTest() { double[,] value = { { 2, 3, 0 }, { -1, 2, 1 }, { 0, -1, 3 } }; double[,] rhs = { { 1, 2, 3 }, { 3, 2, 1 }, { 5, 0, 1 }, }; double[,] expected = { { -0.2174, -0.1739, 0.6522 }, { 0.4783, 0.7826, 0.5652 }, { 1.8261, 0.2609, 0.5217 }, }; LuDecomposition target = new LuDecomposition(value); double[,] actual = target.Solve(rhs); Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001)); }
public void DeterminantTest() { double[,] value = { { 2, 3, 0 }, { -1, 2, 1 }, { 0, -1, 3 } }; LuDecomposition lu = new LuDecomposition(value); Assert.AreEqual(23, lu.Determinant); Assert.IsTrue(lu.Nonsingular); }
public void LogDeterminantTest2() { double[,] value = { { 2, 3, 0 }, { -1, 2, 1 }, { 0, -1, 3 } }; LuDecomposition lu = new LuDecomposition(value); Assert.AreEqual(23, lu.Determinant); double expected = System.Math.Log(23); double actual = lu.LogDeterminant; Assert.AreEqual(expected, actual); }
public void LuDecompositionConstructorTest() { double[,] value = { { 2, -1, 0 }, { -1, 2, -1 }, { 0, -1, 2 } }; double[,] expectedL = { { 1.0000, 0, 0 }, { -0.5000, 1.0000, 0 }, { 0, -0.6667, 1.0000 }, }; double[,] expectedU = { { 2.0000, -1.0000, 0 }, { 0, 1.5000, -1.0000 }, { 0, 0, 1.3333 }, }; LuDecomposition target = new LuDecomposition(value); double[,] actualL = target.LowerTriangularFactor; double[,] actualU = target.UpperTriangularFactor; Assert.IsTrue(Matrix.IsEqual(expectedL, actualL, 0.001)); Assert.IsTrue(Matrix.IsEqual(expectedU, actualU, 0.001)); target = new LuDecomposition(value.Transpose(), true); actualL = target.LowerTriangularFactor; actualU = target.UpperTriangularFactor; Assert.IsTrue(Matrix.IsEqual(expectedL, actualL, 0.001)); Assert.IsTrue(Matrix.IsEqual(expectedU, actualU, 0.001)); }
public void LogDeterminantTest() { LuDecomposition lu = new LuDecomposition(CholeskyDecompositionTest.bigmatrix); Assert.AreEqual(0, lu.Determinant); Assert.AreEqual(-2224.8931093738875, lu.LogDeterminant, 1e-12); Assert.IsTrue(lu.Nonsingular); }
public void SolveTest5() { double[,] value = { { 2.1, 3.1 }, { 1.6, 4.2 }, { 2.1, 5.1 }, }; double[] rhs = { 6.1, 4.3, 2.1 }; double[] expected = { 3.1839, -0.1891 }; LuDecomposition target = new LuDecomposition(value); bool thrown = false; try { double[] actual = target.Solve(rhs); } catch (InvalidOperationException) { thrown = true; } Assert.IsTrue(thrown); }
public void SolveTransposeTest() { double[,] a = { { 2, 1, 4 }, { 6, 2, 2 }, { 0, 1, 6 }, }; double[,] b = { { 1, 0, 7 }, { 5, 2, 1 }, { 1, 5, 2 }, }; double[,] expected = { { 0.5062, 0.2813, 0.0875 }, { 0.1375, 1.1875, -0.0750 }, { 0.8063, -0.2188, 0.2875 }, }; double[,] actual = new LuDecomposition(b, true).SolveTranspose(a); Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001)); }
public void SolveTest4() { double[,] value = { { 2.1, 3.1 }, { 1.6, 4.2 }, }; double[] rhs = { 6.1, 4.3 }; double[] expected = { 3.1839, -0.1891 }; var target1 = new LuDecomposition(value); var target2 = new JaggedLuDecomposition(value.ToJagged()); Assert.IsTrue(Matrix.IsEqual(expected, target1.Solve(rhs), 1e-3)); Assert.IsTrue(Matrix.IsEqual(expected, target2.Solve(rhs), 1e-3)); }
public void SolveTest4() { double[,] value = { { 2.1, 3.1 }, { 1.6, 4.2 }, }; double[] rhs = { 6.1, 4.3 }; double[] expected = { 3.1839, -0.1891 }; LuDecomposition target = new LuDecomposition(value); double[] actual = target.Solve(rhs); Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001)); }
public void InverseTest() { double[,] value = { { 2, 3, 0 }, { -1, 2, 1 }, { 0, -1, 3 } }; double[,] expectedInverse = { { 0.3043, -0.3913, 0.1304 }, { 0.1304, 0.2609, -0.0870 }, { 0.0435, 0.0870, 0.3043 }, }; LuDecomposition target = new LuDecomposition(value); double[,] actualInverse = target.Inverse(); Assert.IsTrue(Matrix.IsEqual(expectedInverse, actualInverse, 0.001)); }
static void Main(string[] args) { Console.WriteLine("Please take a look at the source for examples!"); Console.ReadKey(); #region 1. Declaring matrices // 1.1 Using standard .NET declaration double[,] A = { {1, 2, 3}, {6, 2, 0}, {0, 0, 1} }; double[,] B = { {2, 0, 0}, {0, 2, 0}, {0, 0, 2} }; { // 1.2 Using Accord extension methods double[,] Bi = Matrix.Identity(3).Multiply(2); double[,] Bj = Matrix.Diagonal(3, 2.0); // both are equal to B // 1.2 Using Accord extension methods with implicit typing var I = Matrix.Identity(3); } #endregion #region 2. Matrix Operations { // 2.1 Addition var C = A.Add(B); // 2.2 Subtraction var D = A.Subtract(B); // 2.3 Multiplication { // 2.3.1 By a scalar var halfM = A.Multiply(0.5); // 2.3.2 By a vector double[] m = A.Multiply(new double[] { 1, 2, 3 }); // 2.3.3 By a matrix var M = A.Multiply(B); // 2.4 Transposing var At = A.Transpose(); } } // 2.5 Elementwise operations // 2.5.1 Elementwise multiplication A.ElementwiseMultiply(B); // A.*B // 2.5.1 Elementwise division A.ElementwiseDivide(B); // A./B #endregion #region 3. Matrix characteristics { // 3.1 Calculating the determinant double det = A.Determinant(); // 3.2 Calculating the trace double tr = A.Trace(); // 3.3 Computing the sum vector { double[] sumVector = A.Sum(); // 3.3.1 Computing the total sum of elements double sum = sumVector.Sum(); // 3.3.2 Computing the sum along the rows sumVector = A.Sum(0); // Equivalent to Octave's sum(A, 1) // 3.3.2 Computing the sum along the columns sumVector = A.Sum(1); // Equivalent to Octave's sum(A, 2) } } #endregion #region 4. Linear Algebra { // 4.1 Computing the inverse var invA = A.Inverse(); // 4.2 Computing the pseudo-inverse var pinvA = A.PseudoInverse(); // 4.3 Solving a linear system (Ax = B) var x = A.Solve(B); } #endregion #region 5. Special operators { // 5.1 Finding the indices of elements double[] v = { 5, 2, 2, 7, 1, 0 }; int[] idx = v.Find(e => e > 2); // finding the index of every element in v higher than 2. // 5.2 Selecting elements by index double[] u = v.Submatrix(idx); // u is { 5, 7 } // 5.3 Converting between different matrix representations double[][] jaggedA = A.ToArray(); // from multidimensional to jagged array // 5.4 Extracting a column or row from the matrix double[] a = A.GetColumn(0); // retrieves the first column double[] b = B.GetRow(1); // retrieves the second row // 5.5 Taking the absolute of a matrix var absA = A.Abs(); // 5.6 Applying some function to every element var newv = v.Apply(e => e + 1); } #endregion #region 7. Vector operations { double[] u = { 1, 2, 3 }; double[] v = { 4, 5, 6 }; var w1 = u.InnerProduct(v); var w2 = u.OuterProduct(v); var w3 = u.CartesianProduct(v); double[] m = { 1, 2, 3, 4 }; double[,] M = Matrix.Reshape(m, 2, 2); } #endregion #region Decompositions { // Singular value decomposition { SingularValueDecomposition svd = new SingularValueDecomposition(A); var U = svd.LeftSingularVectors; var S = svd.Diagonal; var V = svd.RightSingularVectors; } // or (please see documentation for details) { SingularValueDecomposition svd = new SingularValueDecomposition(A.Transpose()); var U = svd.RightSingularVectors; var S = svd.Diagonal; var V = svd.LeftSingularVectors; } // Eigenvalue decomposition { EigenvalueDecomposition eig = new EigenvalueDecomposition(A); var V = eig.Eigenvectors; var D = eig.DiagonalMatrix; } // QR decomposition { QrDecomposition qr = new QrDecomposition(A); var Q = qr.OrthogonalFactor; var R = qr.UpperTriangularFactor; } // Cholesky decomposition { CholeskyDecomposition chol = new CholeskyDecomposition(A); var R = chol.LeftTriangularFactor; } // LU decomposition { LuDecomposition lu = new LuDecomposition(A); var L = lu.LowerTriangularFactor; var U = lu.UpperTriangularFactor; } } #endregion }
public void SolveTransposeTest() { double[,] a = { { 2, 1, 4 }, { 6, 2, 2 }, { 0, 1, 6 }, }; double[,] b = { { 1, 0, 7 }, { 5, 2, 1 }, { 1, 5, 2 }, }; double[,] expected = { { 0.5062, 0.2813, 0.0875 }, { 0.1375, 1.1875, -0.0750 }, { 0.8063, -0.2188, 0.2875 }, }; Assert.IsTrue(Matrix.IsEqual(expected, new LuDecomposition(b, true).SolveTranspose(a), 1e-3)); Assert.IsTrue(Matrix.IsEqual(expected, new JaggedLuDecomposition(b.ToJagged(), true).SolveTranspose(a.ToJagged()), 1e-3)); var target = new LuDecomposition(b, true); var p = target.PivotPermutationVector; int[] idx = p.ArgSort(); var r = target.LowerTriangularFactor.Dot(target.UpperTriangularFactor) .Submatrix(idx, null).Transpose(); Assert.IsTrue(Matrix.IsEqual(b, r, 1e-3)); Assert.IsTrue(Matrix.IsEqual(b.Transpose(), target.Reverse(), 1e-3)); Assert.IsTrue(Matrix.IsEqual(b.Transpose(), new JaggedLuDecomposition(b.ToJagged(), true).Reverse(), 1e-3)); }
private double run(double[][] inputs, double[][] outputs) { // Regress using Lower-Bound Newton-Raphson estimation // // The main idea is to replace the Hessian matrix with a // suitable lower bound. Indeed, the Hessian is lower // bounded by a negative definite matrix that does not // even depend on w [Krishnapuram et al]. // // - http://www.lx.it.pt/~mtf/Krishnapuram_Carin_Figueiredo_Hartemink_2005.pdf // // Initial definitions and memory allocations int N = inputs.Length; double[][] design = new double[N][]; double[][] coefficients = this.regression.Coefficients; // Compute the regression matrix for (int i = 0; i < inputs.Length; i++) { double[] row = design[i] = new double[M]; row[0] = 1; // for intercept for (int j = 0; j < inputs[i].Length; j++) row[j + 1] = inputs[i][j]; } // Reset Hessian matrix and gradient for (int i = 0; i < gradient.Length; i++) gradient[i] = 0; if (UpdateLowerBound) { for (int i = 0; i < gradient.Length; i++) for (int j = 0; j < gradient.Length; j++) lowerBound[i, j] = 0; } // In the multinomial logistic regression, the objective // function is the log-likelihood function l(w). As given // by Krishnapuram et al and Böhning, this is a concave // function with Hessian given by: // // H(w) = -sum(P(w) - p(w)p(w)') (x) xx' // (see referenced paper for proper indices) // // In which (x) denotes the Kronocker product. By using // the lower bound principle, Krishnapuram has shown that // we can replace H(w) with a lower bound approximation B // which does not depend on w (eq. 8 on aforementined paper): // // B = -(1/2) [I - 11/M] (x) sum(xx') // // Thus we can compute and invert this matrix only once. // // For each input sample in the dataset for (int i = 0; i < inputs.Length; i++) { // Grab variables related to the sample double[] x = design[i]; double[] y = outputs[i]; // Compute and estimate outputs this.compute(inputs[i], output); // Compute errors for the sample for (int j = 0; j < errors.Length; j++) errors[j] = y[j + 1] - output[j]; // Compute current gradient and Hessian // We can take advantage of the block structure of the // Hessian matrix and gradient vector by employing the // Kronocker product. See [Böhning, 1992] // (Re-) Compute error gradient double[] g = Matrix.KroneckerProduct(errors, x); for (int j = 0; j < g.Length; j++) gradient[j] += g[j]; if (UpdateLowerBound) { // Compute xxt matrix for (int k = 0; k < x.Length; k++) for (int j = 0; j < x.Length; j++) xxt[k, j] = x[k] * x[j]; // (Re-) Compute weighted "Hessian" matrix double[,] h = Matrix.KroneckerProduct(weights, xxt); for (int j = 0; j < parameterCount; j++) for (int k = 0; k < parameterCount; k++) lowerBound[j, k] += h[j, k]; } } if (UpdateLowerBound) { UpdateLowerBound = false; // Decompose to solve the linear system. Usually the hessian will // be invertible and LU will succeed. However, sometimes the hessian // may be singular and a Singular Value Decomposition may be needed. LuDecomposition lu = new LuDecomposition(lowerBound); // The SVD is very stable, but is quite expensive, being on average // about 10-15 times more expensive than LU decomposition. There are // other ways to avoid a singular Hessian. For a very interesting // reading on the subject, please see: // // - Jeff Gill & Gary King, "What to Do When Your Hessian Is Not Invertible", // Sociological Methods & Research, Vol 33, No. 1, August 2004, 54-87. // Available in: http://gking.harvard.edu/files/help.pdf // // Moreover, the computation of the inverse is optional, as it will // be used only to compute the standard errors of the regression. if (lu.Nonsingular) { // Solve using LU decomposition deltas = lu.Solve(gradient); decomposition = lu; } else { // Hessian Matrix is singular, try pseudo-inverse solution decomposition = new SingularValueDecomposition(lowerBound); deltas = decomposition.Solve(gradient); } } else { deltas = decomposition.Solve(gradient); } previous = coefficients.Reshape(1); // Update coefficients using the calculated deltas for (int i = 0, k = 0; i < coefficients.Length; i++) for (int j = 0; j < coefficients[i].Length; j++) coefficients[i][j] -= deltas[k++]; solution = coefficients.Reshape(1); if (computeStandardErrors) { // Grab the regression information matrix double[,] inverse = decomposition.Inverse(); // Calculate coefficients' standard errors double[][] standardErrors = regression.StandardErrors; for (int i = 0, k = 0; i < standardErrors.Length; i++) for (int j = 0; j < standardErrors[i].Length; j++, k++) standardErrors[i][j] = Math.Sqrt(Math.Abs(inverse[k, k])); } // Return the relative maximum parameter change for (int i = 0; i < deltas.Length; i++) deltas[i] = Math.Abs(deltas[i]) / Math.Abs(previous[i]); return Matrix.Max(deltas); }
/// <summary> /// Runs one iteration of the Reweighted Least Squares algorithm. /// </summary> /// <param name="inputs">The input data.</param> /// <param name="outputs">The outputs associated with each input vector.</param> /// <returns>The maximum relative change in the parameters after the iteration.</returns> /// public double Run(double[][] inputs, double[] outputs) { // Regress using Iteratively Reweighted Least Squares estimation. // References: // - Bishop, Christopher M.; Pattern Recognition // and Machine Learning. Springer; 1st ed. 2006. // Initial definitions and memory allocations int N = inputs.Length; double[][] design = new double[N][]; double[] errors = new double[N]; double[] weights = new double[N]; double[] coefficients = this.regression.Coefficients; double[] deltas; // Compute the regression matrix for (int i = 0; i < inputs.Length; i++) { double[] row = design[i] = new double[parameterCount]; row[0] = 1; // for intercept for (int j = 0; j < inputs[i].Length; j++) row[j + 1] = inputs[i][j]; } // Compute errors and weighing matrix for (int i = 0; i < inputs.Length; i++) { double y = regression.Compute(inputs[i]); // Calculate error vector errors[i] = y - outputs[i]; // Calculate weighting matrix weights[i] = y * (1.0 - y); } // Reset Hessian matrix and gradient for (int i = 0; i < gradient.Length; i++) { gradient[i] = 0; for (int j = 0; j < gradient.Length; j++) hessian[i, j] = 0; } // (Re-) Compute error gradient for (int j = 0; j < design.Length; j++) for (int i = 0; i < gradient.Length; i++) gradient[i] += design[j][i] * errors[j]; // (Re-) Compute weighted "Hessian" matrix for (int k = 0; k < weights.Length; k++) { double[] rk = design[k]; for (int j = 0; j < rk.Length; j++) for (int i = 0; i < rk.Length; i++) hessian[j, i] += rk[i] * rk[j] * weights[k]; } // Decompose to solve the linear system. Usually the hessian will // be invertible and LU will succeed. However, sometimes the hessian // may be singular and a Singular Value Decomposition may be needed. LuDecomposition lu = new LuDecomposition(hessian); // The SVD is very stable, but is quite expensive, being on average // about 10-15 times more expensive than LU decomposition. There are // other ways to avoid a singular Hessian. For a very interesting // reading on the subject, please see: // // - Jeff Gill & Gary King, "What to Do When Your Hessian Is Not Invertible", // Sociological Methods & Research, Vol 33, No. 1, August 2004, 54-87. // Available in: http://gking.harvard.edu/files/help.pdf // // Moreover, the computation of the inverse is optional, as it will // be used only to compute the standard errors of the regression. if (lu.Nonsingular) { // Solve using LU decomposition deltas = lu.Solve(gradient); decomposition = lu; } else { // Hessian Matrix is singular, try pseudo-inverse solution decomposition = new SingularValueDecomposition(hessian); deltas = decomposition.Solve(gradient); } previous = (double[])coefficients.Clone(); // Update coefficients using the calculated deltas for (int i = 0; i < coefficients.Length; i++) coefficients[i] -= deltas[i]; if (computeStandardErrors) { // Grab the regression information matrix double[,] inverse = decomposition.Inverse(); // Calculate coefficients' standard errors double[] standardErrors = regression.StandardErrors; for (int i = 0; i < standardErrors.Length; i++) standardErrors[i] = Math.Sqrt(inverse[i, i]); } // Return the relative maximum parameter change for (int i = 0; i < deltas.Length; i++) deltas[i] = Math.Abs(deltas[i]) / Math.Abs(previous[i]); return Matrix.Max(deltas); }
public void SolveTest3() { double[,] value = { { 2.000, 3.000, 0.000 }, { -1.000, 2.000, 1.000 }, }; LuDecomposition target = new LuDecomposition(value); double[,] L = target.LowerTriangularFactor; double[,] U = target.UpperTriangularFactor; double[,] expectedL = { { 1.000, 0.000 }, { -0.500, 1.000 }, }; double[,] expectedU = { { 2.000, 3.000, 0.000 }, { 0.000, 3.500, 1.000 }, }; Assert.IsTrue(Matrix.IsEqual(expectedL, L, 0.001)); Assert.IsTrue(Matrix.IsEqual(expectedU, U, 0.001)); }
/// <summary> /// Creates a new object that is a copy of the current instance. /// </summary> /// <returns> /// A new object that is a copy of this instance. /// </returns> /// public object Clone() { LuDecomposition lud = new LuDecomposition(); lud.rows = this.rows; lud.cols = this.cols; lud.lu = (double[,])this.lu.Clone(); lud.pivotSign = this.pivotSign; lud.pivotVector = (int[])this.pivotVector; return lud; }