public void InverseTestNaN()
{
int n = 5;
var I = Jagged.Identity(n);
for (int i = 0; i < n; i++)
{
for (int j = 0; j < n; j++)
{
double[][] value = Jagged.Magic(n);
// Make symmetric
value = Matrix.Multiply(value, value.Transpose());
value[i][j] = double.NaN;
value[j][i] = double.NaN;
Assert.IsTrue(value.IsSymmetric());
bool thrown = false;
var target = new JaggedCholeskyDecomposition(value);
try
{
target.Solve(I);
}
catch (Exception)
{
thrown = true;
}
Assert.IsTrue(thrown);
}
}
}
/// <summary>
/// Attempts to find the best values for the parameter vector
/// minimizing the discrepancy between the generated outputs
/// and the expected outputs for a given set of input data.
/// </summary>
///
/// <param name="inputs">A set of input data.</param>
/// <param name="outputs">The values associated with each
/// vector in the <paramref name="inputs"/> data.</param>
///
public double Minimize(double[][] inputs, double[] outputs)
{
double sumOfSquaredErrors = 0.0;
// Set upper triangular Hessian to zero
for (int i = 0; i < hessian.Length; i++)
{
Array.Clear(hessian[i], i, hessian.Length - i);
}
// Set Gradient vector to zero
Array.Clear(gradient, 0, gradient.Length);
// Divide the problem into blocks. Instead of computing
// a single Jacobian and a single error vector, we will
// be computing multiple Jacobians for smaller problems
// and then sum all blocks into the final Hessian matrix
// and gradient vector.
int blockSize = inputs.Length / Blocks;
int finalBlock = inputs.Length % Blocks;
int jacobianSize = blockSize * outputCount;
// Re-allocate the partial Jacobian matrix only if needed
if (jacobian[0] == null || jacobian[0].Length < jacobianSize)
{
for (int i = 0; i < jacobian.Length; i++)
{
this.jacobian[i] = new double[jacobianSize];
}
}
// Re-allocate error vector only if needed
if (errors == null || errors.Length < jacobianSize)
{
errors = new double[jacobianSize];
}
// For each block
for (int s = 0; s <= Blocks; s++)
{
if (s == Blocks && finalBlock == 0)
{
continue;
}
int B = (s == Blocks) ? finalBlock : blockSize;
int[] block = Vector.Range(s * blockSize, s * blockSize + B);
// Compute the partial residuals vector
sumOfSquaredErrors += computeErrors(inputs, outputs, block);
// Compute the partial Jacobian
computeJacobian(inputs, block);
if (Double.IsNaN(sumOfSquaredErrors))
{
throw new ArithmeticException("Error calculation has produced a non-finite number."
+ " Please make sure that there are no constant columns in the input data.");
}
// Compute error gradient using Jacobian
for (int i = 0; i < jacobian.Length; i++)
{
double sum = 0;
for (int j = 0; j < jacobianSize; j++)
{
sum += jacobian[i][j] * errors[j];
}
gradient[i] += sum;
}
// Compute Quasi-Hessian Matrix approximation
// using the outer product Jacobian (H ~ J'J)
//
Parallel.For(0, jacobian.Length, i =>
{
double[] ji = jacobian[i];
double[] hi = hessian[i];
for (int j = i; j < hi.Length; j++)
{
double[] jj = jacobian[j];
double sum = 0;
for (int k = 0; k < jj.Length; k++)
{
sum += ji[k] * jj[k];
}
// The Hessian need only be upper-triangular, since
// it is symmetric. The Cholesky decomposition will
// make use of this fact and use the lower-triangular
// portion to hold the decomposition, conserving memory.
hi[j] += 2 * sum;
}
});
}
// Store the Hessian's diagonal for future computations. The
// diagonal will be destroyed in the decomposition, so it can
// still be updated on every iteration by restoring this copy.
//
for (int i = 0; i < hessian.Length; i++)
{
diagonal[i] = hessian[i][i];
}
// Create the initial weights vector
for (int i = 0; i < solution.Length; i++)
{
weights[i] = solution[i];
}
// Define the objective function:
double objective = sumOfSquaredErrors;
double current = objective + 1.0;
// Begin of the main Levenberg-Marquardt method
lambda /= v;
// We'll try to find a direction with less error
// (or where the objective function is smaller)
while (current >= objective && lambda < lambdaMax)
{
lambda *= v;
// Update diagonal (Levenberg-Marquardt)
for (int i = 0; i < diagonal.Length; i++)
{
hessian[i][i] = diagonal[i] + 2 * lambda;
}
// Decompose to solve the linear system. The Cholesky decomposition
// is done in place, occupying the Hessian's lower-triangular part.
decomposition = new JaggedCholeskyDecomposition(hessian, robust: true, inPlace: true);
// Check if the decomposition exists
if (decomposition.IsNotDefined)
{
// The Hessian is singular. Continue to the next
// iteration until the diagonal update transforms
// it back to non-singular.
continue;
}
// Solve using Cholesky decomposition
deltas = decomposition.Solve(gradient);
// Update weights using the calculated deltas
for (int i = 0; i < solution.Length; i++)
{
solution[i] = weights[i] + deltas[i];
}
// Calculate the new error
sumOfSquaredErrors = ComputeError(inputs, outputs);
// Update the objective function
current = sumOfSquaredErrors;
// If the object function is bigger than before, the method
// is tried again using a greater damping factor.
}
// If this iteration caused a error drop, then next iteration
// will use a smaller damping factor.
lambda /= v;
return(sumOfSquaredErrors);
}
/// <summary>
/// Attempts to find the best values for the parameter vector
/// minimizing the discrepancy between the generated outputs
/// and the expected outputs for a given set of input data.
/// </summary>
///
/// <param name="inputs">A set of input data.</param>
/// <param name="outputs">The values associated with each
/// vector in the <paramref name="inputs"/> data.</param>
///
public double Minimize(double[][] inputs, double[] outputs)
{
double sumOfSquaredErrors = 0.0;
// Set upper triangular Hessian to zero
for (int i = 0; i < hessian.Length; i++)
Array.Clear(hessian[i], i, hessian.Length - i);
// Set Gradient vector to zero
Array.Clear(gradient, 0, gradient.Length);
// Divide the problem into blocks. Instead of computing
// a single Jacobian and a single error vector, we will
// be computing multiple Jacobians for smaller problems
// and then sum all blocks into the final Hessian matrix
// and gradient vector.
int blockSize = inputs.Length / Blocks;
int finalBlock = inputs.Length % Blocks;
int jacobianSize = blockSize * outputCount;
// Re-allocate the partial Jacobian matrix only if needed
if (jacobian[0] == null || jacobian[0].Length < jacobianSize)
{
for (int i = 0; i < jacobian.Length; i++)
this.jacobian[i] = new double[jacobianSize];
}
// Re-allocate error vector only if needed
if (errors == null || errors.Length < jacobianSize)
errors = new double[jacobianSize];
// For each block
for (int s = 0; s <= Blocks; s++)
{
if (s == Blocks && finalBlock == 0)
continue;
int B = (s == Blocks) ? finalBlock : blockSize;
int[] block = Matrix.Indices(s * blockSize, s * blockSize + B);
// Compute the partial residuals vector
sumOfSquaredErrors += computeErrors(inputs, outputs, block);
// Compute the partial Jacobian
computeJacobian(inputs, block);
if (Double.IsNaN(sumOfSquaredErrors))
throw new ArithmeticException("Error calculation has produced a non-finite number.");
// Compute error gradient using Jacobian
for (int i = 0; i < jacobian.Length; i++)
{
double sum = 0;
for (int j = 0; j < jacobianSize; j++)
sum += jacobian[i][j] * errors[j];
gradient[i] += sum;
}
// Compute Quasi-Hessian Matrix approximation
// using the outer product Jacobian (H ~ J'J)
//
Parallel.For(0, jacobian.Length, i =>
{
double[] ji = jacobian[i];
double[] hi = hessian[i];
for (int j = i; j < hi.Length; j++)
{
double[] jj = jacobian[j];
double sum = 0;
for (int k = 0; k < jj.Length; k++)
sum += ji[k] * jj[k];
// The Hessian need only be upper-triangular, since
// it is symmetric. The Cholesky decomposition will
// make use of this fact and use the lower-triangular
// portion to hold the decomposition, conserving memory.
hi[j] += 2 * sum;
}
});
}
// Store the Hessian's diagonal for future computations. The
// diagonal will be destroyed in the decomposition, so it can
// still be updated on every iteration by restoring this copy.
//
for (int i = 0; i < hessian.Length; i++)
diagonal[i] = hessian[i][i];
// Create the initial weights vector
for (int i = 0; i < solution.Length; i++)
weights[i] = solution[i];
// Define the objective function:
double objective = sumOfSquaredErrors;
double current = objective + 1.0;
// Begin of the main Levenberg-Marquardt method
lambda /= v;
// We'll try to find a direction with less error
// (or where the objective function is smaller)
while (current >= objective && lambda < lambdaMax)
{
lambda *= v;
// Update diagonal (Levenberg-Marquardt)
for (int i = 0; i < diagonal.Length; i++)
hessian[i][i] = diagonal[i] + 2 * lambda;
// Decompose to solve the linear system. The Cholesky decomposition
// is done in place, occupying the Hessian's lower-triangular part.
decomposition = new JaggedCholeskyDecomposition(hessian, robust: true, inPlace: true);
// Check if the decomposition exists
if (decomposition.IsNotDefined)
{
// The Hessian is singular. Continue to the next
// iteration until the diagonal update transforms
// it back to non-singular.
continue;
}
// Solve using Cholesky decomposition
deltas = decomposition.Solve(gradient);
// Update weights using the calculated deltas
for (int i = 0; i < solution.Length; i++)
solution[i] = weights[i] + deltas[i];
// Calculate the new error
sumOfSquaredErrors = ComputeError(inputs, outputs);
// Update the objective function
current = sumOfSquaredErrors;
// If the object function is bigger than before, the method
// is tried again using a greater damping factor.
}
// If this iteration caused a error drop, then next iteration
// will use a smaller damping factor.
lambda /= v;
return sumOfSquaredErrors;
}
private double iterate(double[][] inputs, double[] outputs, int blockSize, int finalBlock, int jacobianSize)
{
double sumOfSquaredErrors = 0;
// Set upper triangular Hessian to zero
for (var i = 0; i < Hessian.Length; i++)
{
Array.Clear(Hessian[i], i, Hessian.Length - i);
}
// Set Gradient vector to zero
Array.Clear(gradient, 0, gradient.Length);
// For each block
for (var s = 0; s <= Blocks; s++)
{
if (s == Blocks && finalBlock == 0)
{
continue;
}
var B = s == Blocks ? finalBlock : blockSize;
var block = Vector.Range(s * blockSize, s * blockSize + B);
// Compute the partial residuals vector
sumOfSquaredErrors += computeErrors(inputs, outputs, block);
// Compute the partial Jacobian
computeJacobian(inputs, block);
if (double.IsNaN(sumOfSquaredErrors))
{
throw new ArithmeticException("Error calculation has produced a non-finite number."
+ " Please make sure that there are no constant columns in the input data.");
}
// Compute error gradient using Jacobian
for (var i = 0; i < jacobian.Length; i++)
{
double sum = 0;
for (var j = 0; j < jacobianSize; j++)
{
sum += jacobian[i][j] * errors[j];
}
gradient[i] += sum;
}
// Compute Quasi-Hessian Matrix approximation
// using the outer product Jacobian (H ~ J'J)
//
Parallel.For(0, jacobian.Length, ParallelOptions, i =>
{
var ji = jacobian[i];
var hi = Hessian[i];
for (var j = i; j < hi.Length; j++)
{
var jj = jacobian[j];
double sum = 0;
for (var k = 0; k < jj.Length; k++)
{
sum += ji[k] * jj[k];
}
// The Hessian need only be upper-triangular, since
// it is symmetric. The Cholesky decomposition will
// make use of this fact and use the lower-triangular
// portion to hold the decomposition, conserving memory.
hi[j] += 2 * sum;
}
});
}
// Store the Hessian's diagonal for future computations. The
// diagonal will be destroyed in the decomposition, so it can
// still be updated on every iteration by restoring this copy.
//
for (var i = 0; i < Hessian.Length; i++)
{
diagonal[i] = Hessian[i][i] + Epsilon;
}
// Create the initial weights vector
for (var i = 0; i < Solution.Length; i++)
{
weights[i] = Solution[i];
}
// Define the objective function:
var objective = sumOfSquaredErrors;
var current = objective + 1.0;
// Begin of the main Levenberg-Marquardt method
LearningRate /= Adjustment;
// We'll try to find a direction with less error
// (or where the objective function is smaller)
while (current >= objective && LearningRate < lambdaMax)
{
if (Token.IsCancellationRequested)
{
break;
}
LearningRate *= Adjustment;
// Update diagonal (Levenberg-Marquardt)
for (var i = 0; i < diagonal.Length; i++)
{
Hessian[i][i] = diagonal[i] * (1 + LearningRate);
}
// Decompose to solve the linear system. The Cholesky decomposition
// is done in place, occupying the Hessian's lower-triangular part.
decomposition = new JaggedCholeskyDecomposition(Hessian, true, true);
// Check if the decomposition exists
if (decomposition.IsUndefined)
{
// The Hessian is singular. Continue to the next
// iteration until the diagonal update transforms
// it back to non-singular.
continue;
}
// Solve using Cholesky decomposition
deltas = decomposition.Solve(gradient);
// Update weights using the calculated deltas
for (var i = 0; i < Solution.Length; i++)
{
Solution[i] = weights[i] + deltas[i];
}
// Calculate the new error
sumOfSquaredErrors = ComputeError(inputs, outputs);
// Update the objective function
current = sumOfSquaredErrors;
// If the object function is bigger than before, the method
// is tried again using a greater damping factor.
}
// If this iteration caused a error drop, then next iteration
// will use a smaller damping factor.
LearningRate /= Adjustment;
return(sumOfSquaredErrors);
}
public void JaggedCholeskyDecompositionConstructorTest()
{
// Based on tests by Ken Johnson
double[,] value = // positive-definite
{
{ 2, -1, 0 },
{ -1, 2, -1 },
{ 0, -1, 2 }
};
double[,] expected =
{
{ 1.4142, 0.0000, 0.0000 },
{ -0.7071, 1.2247, 0.0000 },
{ 0.0000, -0.8165, 1.1547 }
};
var chol = new JaggedCholeskyDecomposition(value.ToJagged());
double[][] L = chol.LeftTriangularFactor;
Assert.IsTrue(Matrix.IsEqual(L, expected, 0.0001));
// Decomposition Identity
Assert.IsTrue(Matrix.IsEqual(Matrix.Multiply(L, L.Transpose()), value, 1e-3));
Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-3));
Assert.AreEqual(new LuDecomposition(value).Determinant, chol.Determinant, 1e-10);
Assert.IsTrue(chol.IsPositiveDefinite);
//Assert.AreEqual(true, chol.Symmetric);
}
public void LogDeterminantTest2()
{
double[,] value =
{
{ 6, -1, 2, 1 },
{ -1, 9, -3, -2 },
{ 2, -3, 8, 1 },
{ 1, -2, 1, 7 },
};
var chol = new JaggedCholeskyDecomposition(value.ToJagged());
Assert.AreEqual(2232, chol.Determinant, 1e-12);
double expected = System.Math.Log(2232);
double actual = chol.LogDeterminant;
Assert.AreEqual(expected, actual, 1e-10);
}
public void DeterminantTest()
{
double[,] value =
{
{ 6, -1, 2, 1 },
{ -1, 9, -3, -2 },
{ 2, -3, 8, 1 },
{ 1, -2, 1, 7 },
};
var chol = new JaggedCholeskyDecomposition(value.ToJagged());
Assert.AreEqual(2232, chol.Determinant, 1e-12);
Assert.IsTrue(chol.Nonsingular);
//Assert.IsTrue(chol.Symmetric);
}
public void LogDeterminantTest()
{
var chol = new JaggedCholeskyDecomposition(bigmatrix.ToJagged());
Assert.AreEqual(0.0, chol.Determinant);
Assert.AreEqual(-2224.8931093738875, chol.LogDeterminant, 1e-10);
Assert.IsTrue(chol.IsPositiveDefinite);
Assert.IsTrue(chol.Nonsingular);
Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), bigmatrix, 1e-6));
}
public void JaggedCholeskyDecompositionConstructorTest3()
{
double[,] value = // not positive-definite
{
{ 6, -1, 2, 6 },
{ -1, 3, -3, -2 },
{ 2, -3, 2, 0 },
{ 6, -2, 0, 0 },
};
double[,] expected =
{
{ 1.0000, 0, 0, 0 },
{ -0.1667, 1.0000, 0, 0 },
{ 0.3333, -0.9412, 1.0000, 0 },
{ 1.0000, -0.3529, 2.5000, 1.0000 },
};
double[,] diagonal =
{
{ 6.0000, 0, 0, 0 },
{ 0, 2.8333, 0, 0 },
{ 0, 0, -1.1765, 0 },
{ 0, 0, 0, 1.0000 },
};
var chol = new JaggedCholeskyDecomposition(value.ToJagged(), true);
double[][] L = chol.LeftTriangularFactor;
double[][] D = chol.DiagonalMatrix;
Assert.IsTrue(Matrix.IsEqual(L, expected, 0.001));
Assert.IsTrue(Matrix.IsEqual(D, diagonal, 0.001));
// Decomposition Identity
Assert.IsTrue(Matrix.IsEqual(L.Multiply(D).Multiply(L.Transpose()), value, 0.001));
Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-6));
Assert.AreEqual(new LuDecomposition(value).Determinant, chol.Determinant, 1e-10);
Assert.IsFalse(chol.IsPositiveDefinite);
//Assert.AreEqual(true, chol.Symmetric);
}
public void JaggedCholeskyDecompositionConstructorTest4()
{
// Based on tests by Ken Johnson
double[,] value = // positive-definite
{
{ 2, 0, 0 },
{ -1, 2, 0 },
{ 0, -1, 2 }
};
double[,] expected =
{
{ 1.4142, 0.0000, 0.0000 },
{ -0.7071, 1.2247, 0.0000 },
{ 0.0000, -0.8165, 1.1547 }
};
var chol = new JaggedCholeskyDecomposition(value.ToJagged(), robust: false, valueType: MatrixType.LowerTriangular);
double[][] L = chol.LeftTriangularFactor;
Assert.IsTrue(Matrix.IsEqual(L, expected, 1e-4));
Assert.AreEqual(4, chol.Determinant, 1e-10);
Assert.IsTrue(chol.IsPositiveDefinite);
//Assert.AreEqual(true, chol.Symmetric);
Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value.GetSymmetric(type: MatrixType.LowerTriangular), 1e-4));
double[,] expected2 =
{
{ 1.0000, 0.0000, 0.0000 },
{ -0.5000, 1.0000, 0.0000 },
{ 0.0000, -0.6667, 1.0000 }
};
chol = new JaggedCholeskyDecomposition(value.ToJagged(), robust: true, valueType: MatrixType.LowerTriangular);
L = chol.LeftTriangularFactor;
Assert.IsTrue(Matrix.IsEqual(L, expected2, 1e-4));
Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value.GetSymmetric(type: MatrixType.LowerTriangular), 1e-4));
Assert.AreEqual(4, chol.Determinant, 1e-10);
Assert.IsTrue(chol.IsPositiveDefinite);
//Assert.AreEqual(true, chol.Symmetric);
}
public void JaggedCholeskyDecompositionConstructorTest2()
{
double[,] value = // positive-definite
{
{ 2, -1, 0 },
{ -1, 2, -1 },
{ 0, -1, 2 }
};
double[,] expected =
{
{ 1.0000, 0.0000, 0.0000 },
{ -0.5000, 1.0000, 0.0000 },
{ 0.0000, -0.6667, 1.0000 }
};
double[,] diagonal =
{
{ 2.0000, 0.0000, 0.0000 },
{ 0.0000, 1.5000, 0.0000 },
{ 0.0000, 0.0000, 1.3333 },
};
var chol = new JaggedCholeskyDecomposition(value.ToJagged(), true);
double[][] L = chol.LeftTriangularFactor;
double[][] D = chol.DiagonalMatrix;
Assert.IsTrue(Matrix.IsEqual(L, expected, 0.001));
Assert.IsTrue(Matrix.IsEqual(D, diagonal, 0.001));
// Decomposition Identity
Assert.IsTrue(Matrix.IsEqual(Matrix.Multiply(Matrix.Multiply(L, D), L.Transpose()), value, 0.001));
Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-6));
Assert.AreEqual(new LuDecomposition(value).Determinant, chol.Determinant, 1e-10);
Assert.IsTrue(chol.IsPositiveDefinite);
}
public void InverseTest()
{
double[,] value = // not positive-definite
{
{ 6, -1, 2, 6 },
{ -1, 3, -3, -2 },
{ 2, -3, 2, 0 },
{ 6, -2, 0, 0 },
};
var chol = new JaggedCholeskyDecomposition(value.ToJagged(), true);
double[][] L = chol.LeftTriangularFactor;
double[][] expected = Matrix.Inverse(value.ToJagged());
double[][] actual = chol.Inverse();
Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-10));
Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-6));
}
public void SolveTest4()
{
double[,] value = // not positive-definite
{
{ 6, -1, 2, 6 },
{ -1, 3, -3, -2 },
{ 2, -3, 2, 0 },
{ 6, -2, 0, 0 },
};
var chol = new JaggedCholeskyDecomposition(value.ToJagged(), true);
double[][] L = chol.LeftTriangularFactor;
double[] B = new double[] { 1, 2, 3, 4 };
double[] expected = { 5, 13, 16, -8 };
double[] actual = chol.Solve(B);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.0000000000001));
}
public void SolveTest2()
{
double[,] value = // not positive-definite
{
{ 6, -1, 2, 6 },
{ -1, 3, -3, -2 },
{ 2, -3, 2, 0 },
{ 6, -2, 0, 0 },
};
var chol = new JaggedCholeskyDecomposition(value.ToJagged(), true);
double[][] L = chol.LeftTriangularFactor;
double[][] B = Jagged.Identity(4);
double[,] expected =
{
{ 0.4000, 1.2000, 1.4000, -0.5000 },
{ 1.2000, 3.6000, 4.2000, -2.0000 },
{ 1.4000, 4.2000, 5.4000, -2.5000 },
{ -0.5000, -2.0000, -2.5000, 1.0000 },
};
double[][] actual = chol.Solve(B);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-10));
Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-6));
}
public void SolveTest3()
{
double[,] value = // positive-definite
{
{ 2, -1, 0 },
{ -1, 2, -1 },
{ 0, -1, 2 }
};
var chol = new JaggedCholeskyDecomposition(value.ToJagged());
double[][] L = chol.LeftTriangularFactor;
double[] B = new double[] { 1, 2, 3 };
double[] expected = new double[] { 2.5, 4.0, 3.5 };
double[] actual = chol.Solve(B);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-10));
actual = chol.Solve(B, true);
Assert.AreEqual(actual, B);
Assert.IsTrue(Matrix.IsEqual(expected, B, 1e-10));
Assert.IsTrue(Matrix.IsEqual(chol.Reverse(), value, 1e-6));
}
