public void InverseTestNaN() { int n = 5; var I = Jagged.Identity(n); for (int i = 0; i < n; i++) { for (int j = 0; j < n; j++) { double[][] value = Jagged.Magic(n); // Make symmetric value = Matrix.Multiply(value, value.Transpose()); value[i][j] = double.NaN; value[j][i] = double.NaN; Assert.IsTrue(value.IsSymmetric()); bool thrown = false; var target = new JaggedCholeskyDecomposition(value); try { target.Solve(I); } catch (Exception) { thrown = true; } Assert.IsTrue(thrown); } } }
/// <summary> /// Attempts to find the best values for the parameter vector /// minimizing the discrepancy between the generated outputs /// and the expected outputs for a given set of input data. /// </summary> /// /// <param name="inputs">A set of input data.</param> /// <param name="outputs">The values associated with each /// vector in the <paramref name="inputs"/> data.</param> /// public double Minimize(double[][] inputs, double[] outputs) { double sumOfSquaredErrors = 0.0; // Set upper triangular Hessian to zero for (int i = 0; i < hessian.Length; i++) { Array.Clear(hessian[i], i, hessian.Length - i); } // Set Gradient vector to zero Array.Clear(gradient, 0, gradient.Length); // Divide the problem into blocks. Instead of computing // a single Jacobian and a single error vector, we will // be computing multiple Jacobians for smaller problems // and then sum all blocks into the final Hessian matrix // and gradient vector. int blockSize = inputs.Length / Blocks; int finalBlock = inputs.Length % Blocks; int jacobianSize = blockSize * outputCount; // Re-allocate the partial Jacobian matrix only if needed if (jacobian[0] == null || jacobian[0].Length < jacobianSize) { for (int i = 0; i < jacobian.Length; i++) { this.jacobian[i] = new double[jacobianSize]; } } // Re-allocate error vector only if needed if (errors == null || errors.Length < jacobianSize) { errors = new double[jacobianSize]; } // For each block for (int s = 0; s <= Blocks; s++) { if (s == Blocks && finalBlock == 0) { continue; } int B = (s == Blocks) ? finalBlock : blockSize; int[] block = Vector.Range(s * blockSize, s * blockSize + B); // Compute the partial residuals vector sumOfSquaredErrors += computeErrors(inputs, outputs, block); // Compute the partial Jacobian computeJacobian(inputs, block); if (Double.IsNaN(sumOfSquaredErrors)) { throw new ArithmeticException("Error calculation has produced a non-finite number." + " Please make sure that there are no constant columns in the input data."); } // Compute error gradient using Jacobian for (int i = 0; i < jacobian.Length; i++) { double sum = 0; for (int j = 0; j < jacobianSize; j++) { sum += jacobian[i][j] * errors[j]; } gradient[i] += sum; } // Compute Quasi-Hessian Matrix approximation // using the outer product Jacobian (H ~ J'J) // Parallel.For(0, jacobian.Length, i => { double[] ji = jacobian[i]; double[] hi = hessian[i]; for (int j = i; j < hi.Length; j++) { double[] jj = jacobian[j]; double sum = 0; for (int k = 0; k < jj.Length; k++) { sum += ji[k] * jj[k]; } // The Hessian need only be upper-triangular, since // it is symmetric. The Cholesky decomposition will // make use of this fact and use the lower-triangular // portion to hold the decomposition, conserving memory. hi[j] += 2 * sum; } }); } // Store the Hessian's diagonal for future computations. The // diagonal will be destroyed in the decomposition, so it can // still be updated on every iteration by restoring this copy. // for (int i = 0; i < hessian.Length; i++) { diagonal[i] = hessian[i][i]; } // Create the initial weights vector for (int i = 0; i < solution.Length; i++) { weights[i] = solution[i]; } // Define the objective function: double objective = sumOfSquaredErrors; double current = objective + 1.0; // Begin of the main Levenberg-Marquardt method lambda /= v; // We'll try to find a direction with less error // (or where the objective function is smaller) while (current >= objective && lambda < lambdaMax) { lambda *= v; // Update diagonal (Levenberg-Marquardt) for (int i = 0; i < diagonal.Length; i++) { hessian[i][i] = diagonal[i] + 2 * lambda; } // Decompose to solve the linear system. The Cholesky decomposition // is done in place, occupying the Hessian's lower-triangular part. decomposition = new JaggedCholeskyDecomposition(hessian, robust: true, inPlace: true); // Check if the decomposition exists if (decomposition.IsNotDefined) { // The Hessian is singular. Continue to the next // iteration until the diagonal update transforms // it back to non-singular. continue; } // Solve using Cholesky decomposition deltas = decomposition.Solve(gradient); // Update weights using the calculated deltas for (int i = 0; i < solution.Length; i++) { solution[i] = weights[i] + deltas[i]; } // Calculate the new error sumOfSquaredErrors = ComputeError(inputs, outputs); // Update the objective function current = sumOfSquaredErrors; // If the object function is bigger than before, the method // is tried again using a greater damping factor. } // If this iteration caused a error drop, then next iteration // will use a smaller damping factor. lambda /= v; return(sumOfSquaredErrors); }
/// <summary> /// Attempts to find the best values for the parameter vector /// minimizing the discrepancy between the generated outputs /// and the expected outputs for a given set of input data. /// </summary> /// /// <param name="inputs">A set of input data.</param> /// <param name="outputs">The values associated with each /// vector in the <paramref name="inputs"/> data.</param> /// public double Minimize(double[][] inputs, double[] outputs) { double sumOfSquaredErrors = 0.0; // Set upper triangular Hessian to zero for (int i = 0; i < hessian.Length; i++) Array.Clear(hessian[i], i, hessian.Length - i); // Set Gradient vector to zero Array.Clear(gradient, 0, gradient.Length); // Divide the problem into blocks. Instead of computing // a single Jacobian and a single error vector, we will // be computing multiple Jacobians for smaller problems // and then sum all blocks into the final Hessian matrix // and gradient vector. int blockSize = inputs.Length / Blocks; int finalBlock = inputs.Length % Blocks; int jacobianSize = blockSize * outputCount; // Re-allocate the partial Jacobian matrix only if needed if (jacobian[0] == null || jacobian[0].Length < jacobianSize) { for (int i = 0; i < jacobian.Length; i++) this.jacobian[i] = new double[jacobianSize]; } // Re-allocate error vector only if needed if (errors == null || errors.Length < jacobianSize) errors = new double[jacobianSize]; // For each block for (int s = 0; s <= Blocks; s++) { if (s == Blocks && finalBlock == 0) continue; int B = (s == Blocks) ? finalBlock : blockSize; int[] block = Matrix.Indices(s * blockSize, s * blockSize + B); // Compute the partial residuals vector sumOfSquaredErrors += computeErrors(inputs, outputs, block); // Compute the partial Jacobian computeJacobian(inputs, block); if (Double.IsNaN(sumOfSquaredErrors)) throw new ArithmeticException("Error calculation has produced a non-finite number."); // Compute error gradient using Jacobian for (int i = 0; i < jacobian.Length; i++) { double sum = 0; for (int j = 0; j < jacobianSize; j++) sum += jacobian[i][j] * errors[j]; gradient[i] += sum; } // Compute Quasi-Hessian Matrix approximation // using the outer product Jacobian (H ~ J'J) // Parallel.For(0, jacobian.Length, i => { double[] ji = jacobian[i]; double[] hi = hessian[i]; for (int j = i; j < hi.Length; j++) { double[] jj = jacobian[j]; double sum = 0; for (int k = 0; k < jj.Length; k++) sum += ji[k] * jj[k]; // The Hessian need only be upper-triangular, since // it is symmetric. The Cholesky decomposition will // make use of this fact and use the lower-triangular // portion to hold the decomposition, conserving memory. hi[j] += 2 * sum; } }); } // Store the Hessian's diagonal for future computations. The // diagonal will be destroyed in the decomposition, so it can // still be updated on every iteration by restoring this copy. // for (int i = 0; i < hessian.Length; i++) diagonal[i] = hessian[i][i]; // Create the initial weights vector for (int i = 0; i < solution.Length; i++) weights[i] = solution[i]; // Define the objective function: double objective = sumOfSquaredErrors; double current = objective + 1.0; // Begin of the main Levenberg-Marquardt method lambda /= v; // We'll try to find a direction with less error // (or where the objective function is smaller) while (current >= objective && lambda < lambdaMax) { lambda *= v; // Update diagonal (Levenberg-Marquardt) for (int i = 0; i < diagonal.Length; i++) hessian[i][i] = diagonal[i] + 2 * lambda; // Decompose to solve the linear system. The Cholesky decomposition // is done in place, occupying the Hessian's lower-triangular part. decomposition = new JaggedCholeskyDecomposition(hessian, robust: true, inPlace: true); // Check if the decomposition exists if (decomposition.IsNotDefined) { // The Hessian is singular. Continue to the next // iteration until the diagonal update transforms // it back to non-singular. continue; } // Solve using Cholesky decomposition deltas = decomposition.Solve(gradient); // Update weights using the calculated deltas for (int i = 0; i < solution.Length; i++) solution[i] = weights[i] + deltas[i]; // Calculate the new error sumOfSquaredErrors = ComputeError(inputs, outputs); // Update the objective function current = sumOfSquaredErrors; // If the object function is bigger than before, the method // is tried again using a greater damping factor. } // If this iteration caused a error drop, then next iteration // will use a smaller damping factor. lambda /= v; return sumOfSquaredErrors; }
private double iterate(double[][] inputs, double[] outputs, int blockSize, int finalBlock, int jacobianSize) { double sumOfSquaredErrors = 0; // Set upper triangular Hessian to zero for (var i = 0; i < Hessian.Length; i++) { Array.Clear(Hessian[i], i, Hessian.Length - i); } // Set Gradient vector to zero Array.Clear(gradient, 0, gradient.Length); // For each block for (var s = 0; s <= Blocks; s++) { if (s == Blocks && finalBlock == 0) { continue; } var B = s == Blocks ? finalBlock : blockSize; var block = Vector.Range(s * blockSize, s * blockSize + B); // Compute the partial residuals vector sumOfSquaredErrors += computeErrors(inputs, outputs, block); // Compute the partial Jacobian computeJacobian(inputs, block); if (double.IsNaN(sumOfSquaredErrors)) { throw new ArithmeticException("Error calculation has produced a non-finite number." + " Please make sure that there are no constant columns in the input data."); } // Compute error gradient using Jacobian for (var i = 0; i < jacobian.Length; i++) { double sum = 0; for (var j = 0; j < jacobianSize; j++) { sum += jacobian[i][j] * errors[j]; } gradient[i] += sum; } // Compute Quasi-Hessian Matrix approximation // using the outer product Jacobian (H ~ J'J) // Parallel.For(0, jacobian.Length, ParallelOptions, i => { var ji = jacobian[i]; var hi = Hessian[i]; for (var j = i; j < hi.Length; j++) { var jj = jacobian[j]; double sum = 0; for (var k = 0; k < jj.Length; k++) { sum += ji[k] * jj[k]; } // The Hessian need only be upper-triangular, since // it is symmetric. The Cholesky decomposition will // make use of this fact and use the lower-triangular // portion to hold the decomposition, conserving memory. hi[j] += 2 * sum; } }); } // Store the Hessian's diagonal for future computations. The // diagonal will be destroyed in the decomposition, so it can // still be updated on every iteration by restoring this copy. // for (var i = 0; i < Hessian.Length; i++) { diagonal[i] = Hessian[i][i] + Epsilon; } // Create the initial weights vector for (var i = 0; i < Solution.Length; i++) { weights[i] = Solution[i]; } // Define the objective function: var objective = sumOfSquaredErrors; var current = objective + 1.0; // Begin of the main Levenberg-Marquardt method LearningRate /= Adjustment; // We'll try to find a direction with less error // (or where the objective function is smaller) while (current >= objective && LearningRate < lambdaMax) { if (Token.IsCancellationRequested) { break; } LearningRate *= Adjustment; // Update diagonal (Levenberg-Marquardt) for (var i = 0; i < diagonal.Length; i++) { Hessian[i][i] = diagonal[i] * (1 + LearningRate); } // Decompose to solve the linear system. The Cholesky decomposition // is done in place, occupying the Hessian's lower-triangular part. decomposition = new JaggedCholeskyDecomposition(Hessian, true, true); // Check if the decomposition exists if (decomposition.IsUndefined) { // The Hessian is singular. Continue to the next // iteration until the diagonal update transforms // it back to non-singular. continue; } // Solve using Cholesky decomposition deltas = decomposition.Solve(gradient); // Update weights using the calculated deltas for (var i = 0; i < Solution.Length; i++) { Solution[i] = weights[i] + deltas[i]; } // Calculate the new error sumOfSquaredErrors = ComputeError(inputs, outputs); // Update the objective function current = sumOfSquaredErrors; // If the object function is bigger than before, the method // is tried again using a greater damping factor. } // If this iteration caused a error drop, then next iteration // will use a smaller damping factor. LearningRate /= Adjustment; return(sumOfSquaredErrors); }
public void JaggedCholeskyDecompositionConstructorTest() { // Based on tests by Ken Johnson double[,] value = // positive-definite { { 2, -1, 0 }, { -1, 2, -1 }, { 0, -1, 2 } }; double[,] expected = { { 1.4142, 0.0000, 0.0000 }, { -0.7071, 1.2247, 0.0000 }, { 0.0000, -0.8165, 1.1547 } }; var chol = new JaggedCholeskyDecomposition(value.ToJagged()); double[][] L = chol.LeftTriangularFactor; Assert.IsTrue(Matrix.IsEqual(L, expected, 0.0001)); // Decomposition Identity Assert.IsTrue(Matrix.IsEqual(Matrix.Multiply(L, L.Transpose()), value, 1e-3)); Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-3)); Assert.AreEqual(new LuDecomposition(value).Determinant, chol.Determinant, 1e-10); Assert.IsTrue(chol.IsPositiveDefinite); //Assert.AreEqual(true, chol.Symmetric); }
public void LogDeterminantTest2() { double[,] value = { { 6, -1, 2, 1 }, { -1, 9, -3, -2 }, { 2, -3, 8, 1 }, { 1, -2, 1, 7 }, }; var chol = new JaggedCholeskyDecomposition(value.ToJagged()); Assert.AreEqual(2232, chol.Determinant, 1e-12); double expected = System.Math.Log(2232); double actual = chol.LogDeterminant; Assert.AreEqual(expected, actual, 1e-10); }
public void DeterminantTest() { double[,] value = { { 6, -1, 2, 1 }, { -1, 9, -3, -2 }, { 2, -3, 8, 1 }, { 1, -2, 1, 7 }, }; var chol = new JaggedCholeskyDecomposition(value.ToJagged()); Assert.AreEqual(2232, chol.Determinant, 1e-12); Assert.IsTrue(chol.Nonsingular); //Assert.IsTrue(chol.Symmetric); }
public void LogDeterminantTest() { var chol = new JaggedCholeskyDecomposition(bigmatrix.ToJagged()); Assert.AreEqual(0.0, chol.Determinant); Assert.AreEqual(-2224.8931093738875, chol.LogDeterminant, 1e-10); Assert.IsTrue(chol.IsPositiveDefinite); Assert.IsTrue(chol.Nonsingular); Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), bigmatrix, 1e-6)); }
public void JaggedCholeskyDecompositionConstructorTest3() { double[,] value = // not positive-definite { { 6, -1, 2, 6 }, { -1, 3, -3, -2 }, { 2, -3, 2, 0 }, { 6, -2, 0, 0 }, }; double[,] expected = { { 1.0000, 0, 0, 0 }, { -0.1667, 1.0000, 0, 0 }, { 0.3333, -0.9412, 1.0000, 0 }, { 1.0000, -0.3529, 2.5000, 1.0000 }, }; double[,] diagonal = { { 6.0000, 0, 0, 0 }, { 0, 2.8333, 0, 0 }, { 0, 0, -1.1765, 0 }, { 0, 0, 0, 1.0000 }, }; var chol = new JaggedCholeskyDecomposition(value.ToJagged(), true); double[][] L = chol.LeftTriangularFactor; double[][] D = chol.DiagonalMatrix; Assert.IsTrue(Matrix.IsEqual(L, expected, 0.001)); Assert.IsTrue(Matrix.IsEqual(D, diagonal, 0.001)); // Decomposition Identity Assert.IsTrue(Matrix.IsEqual(L.Multiply(D).Multiply(L.Transpose()), value, 0.001)); Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-6)); Assert.AreEqual(new LuDecomposition(value).Determinant, chol.Determinant, 1e-10); Assert.IsFalse(chol.IsPositiveDefinite); //Assert.AreEqual(true, chol.Symmetric); }
public void JaggedCholeskyDecompositionConstructorTest4() { // Based on tests by Ken Johnson double[,] value = // positive-definite { { 2, 0, 0 }, { -1, 2, 0 }, { 0, -1, 2 } }; double[,] expected = { { 1.4142, 0.0000, 0.0000 }, { -0.7071, 1.2247, 0.0000 }, { 0.0000, -0.8165, 1.1547 } }; var chol = new JaggedCholeskyDecomposition(value.ToJagged(), robust: false, valueType: MatrixType.LowerTriangular); double[][] L = chol.LeftTriangularFactor; Assert.IsTrue(Matrix.IsEqual(L, expected, 1e-4)); Assert.AreEqual(4, chol.Determinant, 1e-10); Assert.IsTrue(chol.IsPositiveDefinite); //Assert.AreEqual(true, chol.Symmetric); Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value.GetSymmetric(type: MatrixType.LowerTriangular), 1e-4)); double[,] expected2 = { { 1.0000, 0.0000, 0.0000 }, { -0.5000, 1.0000, 0.0000 }, { 0.0000, -0.6667, 1.0000 } }; chol = new JaggedCholeskyDecomposition(value.ToJagged(), robust: true, valueType: MatrixType.LowerTriangular); L = chol.LeftTriangularFactor; Assert.IsTrue(Matrix.IsEqual(L, expected2, 1e-4)); Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value.GetSymmetric(type: MatrixType.LowerTriangular), 1e-4)); Assert.AreEqual(4, chol.Determinant, 1e-10); Assert.IsTrue(chol.IsPositiveDefinite); //Assert.AreEqual(true, chol.Symmetric); }
public void JaggedCholeskyDecompositionConstructorTest2() { double[,] value = // positive-definite { { 2, -1, 0 }, { -1, 2, -1 }, { 0, -1, 2 } }; double[,] expected = { { 1.0000, 0.0000, 0.0000 }, { -0.5000, 1.0000, 0.0000 }, { 0.0000, -0.6667, 1.0000 } }; double[,] diagonal = { { 2.0000, 0.0000, 0.0000 }, { 0.0000, 1.5000, 0.0000 }, { 0.0000, 0.0000, 1.3333 }, }; var chol = new JaggedCholeskyDecomposition(value.ToJagged(), true); double[][] L = chol.LeftTriangularFactor; double[][] D = chol.DiagonalMatrix; Assert.IsTrue(Matrix.IsEqual(L, expected, 0.001)); Assert.IsTrue(Matrix.IsEqual(D, diagonal, 0.001)); // Decomposition Identity Assert.IsTrue(Matrix.IsEqual(Matrix.Multiply(Matrix.Multiply(L, D), L.Transpose()), value, 0.001)); Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-6)); Assert.AreEqual(new LuDecomposition(value).Determinant, chol.Determinant, 1e-10); Assert.IsTrue(chol.IsPositiveDefinite); }
public void InverseTest() { double[,] value = // not positive-definite { { 6, -1, 2, 6 }, { -1, 3, -3, -2 }, { 2, -3, 2, 0 }, { 6, -2, 0, 0 }, }; var chol = new JaggedCholeskyDecomposition(value.ToJagged(), true); double[][] L = chol.LeftTriangularFactor; double[][] expected = Matrix.Inverse(value.ToJagged()); double[][] actual = chol.Inverse(); Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-10)); Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-6)); }
public void SolveTest4() { double[,] value = // not positive-definite { { 6, -1, 2, 6 }, { -1, 3, -3, -2 }, { 2, -3, 2, 0 }, { 6, -2, 0, 0 }, }; var chol = new JaggedCholeskyDecomposition(value.ToJagged(), true); double[][] L = chol.LeftTriangularFactor; double[] B = new double[] { 1, 2, 3, 4 }; double[] expected = { 5, 13, 16, -8 }; double[] actual = chol.Solve(B); Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.0000000000001)); }
public void SolveTest2() { double[,] value = // not positive-definite { { 6, -1, 2, 6 }, { -1, 3, -3, -2 }, { 2, -3, 2, 0 }, { 6, -2, 0, 0 }, }; var chol = new JaggedCholeskyDecomposition(value.ToJagged(), true); double[][] L = chol.LeftTriangularFactor; double[][] B = Jagged.Identity(4); double[,] expected = { { 0.4000, 1.2000, 1.4000, -0.5000 }, { 1.2000, 3.6000, 4.2000, -2.0000 }, { 1.4000, 4.2000, 5.4000, -2.5000 }, { -0.5000, -2.0000, -2.5000, 1.0000 }, }; double[][] actual = chol.Solve(B); Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-10)); Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-6)); }
public void SolveTest3() { double[,] value = // positive-definite { { 2, -1, 0 }, { -1, 2, -1 }, { 0, -1, 2 } }; var chol = new JaggedCholeskyDecomposition(value.ToJagged()); double[][] L = chol.LeftTriangularFactor; double[] B = new double[] { 1, 2, 3 }; double[] expected = new double[] { 2.5, 4.0, 3.5 }; double[] actual = chol.Solve(B); Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-10)); actual = chol.Solve(B, true); Assert.AreEqual(actual, B); Assert.IsTrue(Matrix.IsEqual(expected, B, 1e-10)); Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-6)); }