| public Solve ( double value ) : ].double[ | ||
| value | double | Right hand side matrix with as many rows as |
| return | ].double[ | |
public void SolveTest1()
{
double[,] value =
{
{ 2, 3, 0 },
{ -1, 2, 1 },
{ 0, -1, 3 }
};
double[] rhs = { 5, 0, 1 };
double[] expected =
{
1.6522,
0.5652,
0.5217,
};
var target = new LuDecomposition(value);
double[] actual = target.Solve(rhs);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-3));
Assert.IsTrue(Matrix.IsEqual(value, target.Reverse()));
var target2 = new JaggedLuDecomposition(value.ToJagged());
actual = target2.Solve(rhs);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 1e-3));
Assert.IsTrue(Matrix.IsEqual(value, target2.Reverse()));
}
public void InverseTestNaN()
{
int n = 5;
var I = Matrix.Identity(n);
for (int i = 0; i < n; i++)
{
for (int j = 0; j < n; j++)
{
double[,] value = Matrix.Magic(n);
value[i, j] = double.NaN;
var target = new LuDecomposition(value);
Assert.IsTrue(Matrix.IsEqual(target.Solve(I), target.Inverse()));
var target2 = new JaggedLuDecomposition(value.ToJagged());
Assert.IsTrue(Matrix.IsEqual(target2.Solve(I.ToJagged()), target2.Inverse()));
}
}
}
public void InverseTestNaN()
{
int n = 5;
var I = Matrix.Identity(n);
for (int i = 0; i < n; i++)
{
for (int j = 0; j < n; j++)
{
double[,] value = Matrix.Magic(n);
value[i, j] = double.NaN;
var target = new LuDecomposition(value);
var solution = target.Solve(I);
var inverse = target.Inverse();
Assert.IsTrue(Matrix.IsEqual(solution, inverse));
}
}
}
public void SolveTest1()
{
double[,] value =
{
{ 2, 3, 0 },
{ -1, 2, 1 },
{ 0, -1, 3 }
};
double[] rhs = { 5, 0, 1 };
double[] expected =
{
1.6522,
0.5652,
0.5217,
};
LuDecomposition target = new LuDecomposition(value);
double[] actual = target.Solve(rhs);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001));
}
public void SolveTest5()
{
double[,] value =
{
{ 2.1, 3.1 },
{ 1.6, 4.2 },
{ 2.1, 5.1 },
};
double[] rhs = { 6.1, 4.3, 2.1 };
double[] expected = { 3.1839, -0.1891 };
LuDecomposition target = new LuDecomposition(value);
bool thrown = false;
try
{
double[] actual = target.Solve(rhs);
}
catch (InvalidOperationException)
{
thrown = true;
}
Assert.IsTrue(thrown);
}
public void SolveTest4()
{
double[,] value =
{
{ 2.1, 3.1 },
{ 1.6, 4.2 },
};
double[] rhs = { 6.1, 4.3 };
double[] expected = { 3.1839, -0.1891 };
LuDecomposition target = new LuDecomposition(value);
double[] actual = target.Solve(rhs);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001));
}
public void SolveTest()
{
double[,] value =
{
{ 2, 3, 0 },
{ -1, 2, 1 },
{ 0, -1, 3 }
};
double[,] rhs =
{
{ 1, 2, 3 },
{ 3, 2, 1 },
{ 5, 0, 1 },
};
double[,] expected =
{
{ -0.2174, -0.1739, 0.6522 },
{ 0.4783, 0.7826, 0.5652 },
{ 1.8261, 0.2609, 0.5217 },
};
LuDecomposition target = new LuDecomposition(value);
double[,] actual = target.Solve(rhs);
Assert.IsTrue(Matrix.IsEqual(expected, actual, 0.001));
}
private double run(double[][] inputs, double[][] outputs)
{
// Regress using Lower-Bound Newton-Raphson estimation
//
// The main idea is to replace the Hessian matrix with a
// suitable lower bound. Indeed, the Hessian is lower
// bounded by a negative definite matrix that does not
// even depend on w [Krishnapuram et al].
//
// - http://www.lx.it.pt/~mtf/Krishnapuram_Carin_Figueiredo_Hartemink_2005.pdf
//
// Initial definitions and memory allocations
int N = inputs.Length;
double[][] design = new double[N][];
double[][] coefficients = this.regression.Coefficients;
// Compute the regression matrix
for (int i = 0; i < inputs.Length; i++)
{
double[] row = design[i] = new double[M];
row[0] = 1; // for intercept
for (int j = 0; j < inputs[i].Length; j++)
row[j + 1] = inputs[i][j];
}
// Reset Hessian matrix and gradient
for (int i = 0; i < gradient.Length; i++)
gradient[i] = 0;
if (UpdateLowerBound)
{
for (int i = 0; i < gradient.Length; i++)
for (int j = 0; j < gradient.Length; j++)
lowerBound[i, j] = 0;
}
// In the multinomial logistic regression, the objective
// function is the log-likelihood function l(w). As given
// by Krishnapuram et al and Böhning, this is a concave
// function with Hessian given by:
//
// H(w) = -sum(P(w) - p(w)p(w)') (x) xx'
// (see referenced paper for proper indices)
//
// In which (x) denotes the Kronocker product. By using
// the lower bound principle, Krishnapuram has shown that
// we can replace H(w) with a lower bound approximation B
// which does not depend on w (eq. 8 on aforementined paper):
//
// B = -(1/2) [I - 11/M] (x) sum(xx')
//
// Thus we can compute and invert this matrix only once.
//
// For each input sample in the dataset
for (int i = 0; i < inputs.Length; i++)
{
// Grab variables related to the sample
double[] x = design[i];
double[] y = outputs[i];
// Compute and estimate outputs
this.compute(inputs[i], output);
// Compute errors for the sample
for (int j = 0; j < errors.Length; j++)
errors[j] = y[j + 1] - output[j];
// Compute current gradient and Hessian
// We can take advantage of the block structure of the
// Hessian matrix and gradient vector by employing the
// Kronocker product. See [Böhning, 1992]
// (Re-) Compute error gradient
double[] g = Matrix.KroneckerProduct(errors, x);
for (int j = 0; j < g.Length; j++)
gradient[j] += g[j];
if (UpdateLowerBound)
{
// Compute xxt matrix
for (int k = 0; k < x.Length; k++)
for (int j = 0; j < x.Length; j++)
xxt[k, j] = x[k] * x[j];
// (Re-) Compute weighted "Hessian" matrix
double[,] h = Matrix.KroneckerProduct(weights, xxt);
for (int j = 0; j < parameterCount; j++)
for (int k = 0; k < parameterCount; k++)
lowerBound[j, k] += h[j, k];
}
}
if (UpdateLowerBound)
{
UpdateLowerBound = false;
// Decompose to solve the linear system. Usually the hessian will
// be invertible and LU will succeed. However, sometimes the hessian
// may be singular and a Singular Value Decomposition may be needed.
LuDecomposition lu = new LuDecomposition(lowerBound);
// The SVD is very stable, but is quite expensive, being on average
// about 10-15 times more expensive than LU decomposition. There are
// other ways to avoid a singular Hessian. For a very interesting
// reading on the subject, please see:
//
// - Jeff Gill & Gary King, "What to Do When Your Hessian Is Not Invertible",
// Sociological Methods & Research, Vol 33, No. 1, August 2004, 54-87.
// Available in: http://gking.harvard.edu/files/help.pdf
//
// Moreover, the computation of the inverse is optional, as it will
// be used only to compute the standard errors of the regression.
if (lu.Nonsingular)
{
// Solve using LU decomposition
deltas = lu.Solve(gradient);
decomposition = lu;
}
else
{
// Hessian Matrix is singular, try pseudo-inverse solution
decomposition = new SingularValueDecomposition(lowerBound);
deltas = decomposition.Solve(gradient);
}
}
else
{
deltas = decomposition.Solve(gradient);
}
previous = coefficients.Reshape(1);
// Update coefficients using the calculated deltas
for (int i = 0, k = 0; i < coefficients.Length; i++)
for (int j = 0; j < coefficients[i].Length; j++)
coefficients[i][j] -= deltas[k++];
solution = coefficients.Reshape(1);
if (computeStandardErrors)
{
// Grab the regression information matrix
double[,] inverse = decomposition.Inverse();
// Calculate coefficients' standard errors
double[][] standardErrors = regression.StandardErrors;
for (int i = 0, k = 0; i < standardErrors.Length; i++)
for (int j = 0; j < standardErrors[i].Length; j++, k++)
standardErrors[i][j] = Math.Sqrt(Math.Abs(inverse[k, k]));
}
// Return the relative maximum parameter change
for (int i = 0; i < deltas.Length; i++)
deltas[i] = Math.Abs(deltas[i]) / Math.Abs(previous[i]);
return Matrix.Max(deltas);
}
/// <summary>
/// Runs one iteration of the Reweighted Least Squares algorithm.
/// </summary>
/// <param name="inputs">The input data.</param>
/// <param name="outputs">The outputs associated with each input vector.</param>
/// <returns>The maximum relative change in the parameters after the iteration.</returns>
///
public double Run(double[][] inputs, double[] outputs)
{
// Regress using Iteratively Reweighted Least Squares estimation.
// References:
// - Bishop, Christopher M.; Pattern Recognition
// and Machine Learning. Springer; 1st ed. 2006.
// Initial definitions and memory allocations
int N = inputs.Length;
double[][] design = new double[N][];
double[] errors = new double[N];
double[] weights = new double[N];
double[] coefficients = this.regression.Coefficients;
double[] deltas;
// Compute the regression matrix
for (int i = 0; i < inputs.Length; i++)
{
double[] row = design[i] = new double[parameterCount];
row[0] = 1; // for intercept
for (int j = 0; j < inputs[i].Length; j++)
row[j + 1] = inputs[i][j];
}
// Compute errors and weighing matrix
for (int i = 0; i < inputs.Length; i++)
{
double y = regression.Compute(inputs[i]);
// Calculate error vector
errors[i] = y - outputs[i];
// Calculate weighting matrix
weights[i] = y * (1.0 - y);
}
// Reset Hessian matrix and gradient
for (int i = 0; i < gradient.Length; i++)
{
gradient[i] = 0;
for (int j = 0; j < gradient.Length; j++)
hessian[i, j] = 0;
}
// (Re-) Compute error gradient
for (int j = 0; j < design.Length; j++)
for (int i = 0; i < gradient.Length; i++)
gradient[i] += design[j][i] * errors[j];
// (Re-) Compute weighted "Hessian" matrix
for (int k = 0; k < weights.Length; k++)
{
double[] rk = design[k];
for (int j = 0; j < rk.Length; j++)
for (int i = 0; i < rk.Length; i++)
hessian[j, i] += rk[i] * rk[j] * weights[k];
}
// Decompose to solve the linear system. Usually the hessian will
// be invertible and LU will succeed. However, sometimes the hessian
// may be singular and a Singular Value Decomposition may be needed.
LuDecomposition lu = new LuDecomposition(hessian);
// The SVD is very stable, but is quite expensive, being on average
// about 10-15 times more expensive than LU decomposition. There are
// other ways to avoid a singular Hessian. For a very interesting
// reading on the subject, please see:
//
// - Jeff Gill & Gary King, "What to Do When Your Hessian Is Not Invertible",
// Sociological Methods & Research, Vol 33, No. 1, August 2004, 54-87.
// Available in: http://gking.harvard.edu/files/help.pdf
//
// Moreover, the computation of the inverse is optional, as it will
// be used only to compute the standard errors of the regression.
if (lu.Nonsingular)
{
// Solve using LU decomposition
deltas = lu.Solve(gradient);
decomposition = lu;
}
else
{
// Hessian Matrix is singular, try pseudo-inverse solution
decomposition = new SingularValueDecomposition(hessian);
deltas = decomposition.Solve(gradient);
}
previous = (double[])coefficients.Clone();
// Update coefficients using the calculated deltas
for (int i = 0; i < coefficients.Length; i++)
coefficients[i] -= deltas[i];
if (computeStandardErrors)
{
// Grab the regression information matrix
double[,] inverse = decomposition.Inverse();
// Calculate coefficients' standard errors
double[] standardErrors = regression.StandardErrors;
for (int i = 0; i < standardErrors.Length; i++)
standardErrors[i] = Math.Sqrt(inverse[i, i]);
}
// Return the relative maximum parameter change
for (int i = 0; i < deltas.Length; i++)
deltas[i] = Math.Abs(deltas[i]) / Math.Abs(previous[i]);
return Matrix.Max(deltas);
}
public void SolveTest4()
{
double[,] value =
{
{ 2.1, 3.1 },
{ 1.6, 4.2 },
};
double[] rhs = { 6.1, 4.3 };
double[] expected = { 3.1839, -0.1891 };
var target1 = new LuDecomposition(value);
var target2 = new JaggedLuDecomposition(value.ToJagged());
Assert.IsTrue(Matrix.IsEqual(expected, target1.Solve(rhs), 1e-3));
Assert.IsTrue(Matrix.IsEqual(expected, target2.Solve(rhs), 1e-3));
}
