private void compute_regression(double[,] x, double[] y, double[] w)
{
int m = x.GetLength(1);
Math.Linalg.Matrix A = null;
int n = x.GetLength(0);
if (m_intercept)
{
A = new Linalg.Matrix(n, m + 1);
for (int i = 0; i < n; i++)
{
for (int j = 0; j < m; j++)
{
A[i, j] = x[i, j];
}
A[i, m] = 1.0;
}
}
else
{
A = new Linalg.Matrix(x, false);
}
compute_coefficients(A, y, w);
}
/// <summary>
/// 1D constructor
/// </summary>
/// <param name="x"></param>
/// <param name="y"></param>
/// <param name="w"></param>
/// <param name="intercept"></param>
public LinearRegression(double[] x, double[] y, double[] w, bool intercept)
{
if (x == null || y == null || x.Length != y.Length)
{
throw new ArgumentException("LinearRegression: check input arguments");
}
bool use_weights = (w != null);
if (use_weights && w.Length != y.Length)
{
throw new ArgumentException("LinearRegression: array of weights should have the same length");
}
m_intercept = intercept;
//init variable names
m_names = new string[1 + (m_intercept ? 1 : 0)];
m_names[0] = "c1";
if (m_intercept)
{
m_names[1] = "(intercept)";
}
Math.Linalg.Matrix A = null;
int n = x.Length;
if (m_intercept)
{
int m = 1;
A = new Linalg.Matrix(n, m + 1);
for (int i = 0; i < n; i++)
{
A[i, 0] = x[0];
A[i, 1] = 1.0; //intercept
}
}
else
{
A = new Linalg.Matrix(x, false);
}
compute_coefficients(A, y, w);
}
public static void TestLinearAlgebra()
{
double eps = ACQ.Math.Const.epsilon;
//test magic matrices
for (int n = 3; n <= 32; n++)
{
ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateMagic(n);
double trace = M.Trace(); // trace = diagonal sum, should be the magic sum, (n^3 + n)/2.
ACQ.Math.Linalg.EigenvalueDecomposition E = new ACQ.Math.Linalg.EigenvalueDecomposition(0.5 * (M + (M.Transpose())));
double[] d = E.GetRealEigenvalues(); //maximum eigenvalue of (A + A')/2, should equal trace
//int r = M.rank(); //should equal n if n is odd, be less than n if n is even
//double c = M.cond();
ACQ.Math.Linalg.LuDecomposition LU = new ACQ.Math.Linalg.LuDecomposition(M);
ACQ.Math.Linalg.Matrix L = LU.GetL();
ACQ.Math.Linalg.Matrix U = LU.GetU();
int[] p = LU.GetPivotIndex();
ACQ.Math.Linalg.Matrix R = L * U - M.Submatrix(p, 0, n - 1);
double lu_res = R.Norm1();
ACQ.Math.Linalg.QrDecomposition QR = new ACQ.Math.Linalg.QrDecomposition(M);
ACQ.Math.Linalg.Matrix Q = QR.GetQ();
R = QR.GetR();
R = Q * R - M;
double qr_res = R.Norm1();
ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M);
U = SV.GetU();
ACQ.Math.Linalg.Matrix S = SV.GetS();
ACQ.Math.Linalg.Matrix V = SV.GetV();
R = U * S * V.Transpose() - M;
double sv_res = R.Norm1();
double rank = SV.Rank();
double cond = SV.Cond();
Console.WriteLine("n:{0,3} trace:{1:E4} ev:{2:E4} lu:{3:E4} qr:{4:E4} sv:{5:E4}", n, trace, d[d.Length - 1], lu_res, qr_res, sv_res);
}
//test random matrices
ACQ.Math.Random.IRandomGenerator rng = new ACQ.Math.Random.MersenneTwister();
for (int n = 1; n <= 8; n++)
{
ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(n, n, rng);
ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M);
ACQ.Math.Linalg.Matrix U = SV.GetU();
ACQ.Math.Linalg.Matrix S = SV.GetS();
ACQ.Math.Linalg.Matrix V = SV.GetV();
ACQ.Math.Linalg.Matrix R = U * S * V.Transpose() - M;
ACQ.Math.Linalg.Matrix pinv = SV.PseudoInverse();
double pinv_res1 = (M * pinv * M - M).Norm1();
double pinv_res2 = (pinv * M * pinv - pinv).Norm1();
double sv_res = R.Norm1();
double rank = SV.Rank();
double cond = SV.Cond();
Console.WriteLine("n:{0,3} rank:{1:E4} cond:{2:E4} sv:{3:E4} pinv:{4:E4} {5:E4}", n, rank, cond, sv_res, pinv_res1, pinv_res2);
}
//non square matrixes, m > n
for (int n = 1; n <= 32; n++)
{
for (int m = n; m <= 32; m++)
{
ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(m, n, rng);
double err_scale = 1.0 / (System.Math.Max(m, n) * eps);
ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M);
ACQ.Math.Linalg.Matrix U = SV.GetU();
ACQ.Math.Linalg.Matrix S = SV.GetS();
ACQ.Math.Linalg.Matrix V = SV.GetV();
ACQ.Math.Linalg.Matrix R = U * S * V.Transpose() - M;
ACQ.Math.Linalg.Matrix pinv = SV.PseudoInverse();
double pinv_res1 = (M * pinv * M - M).Norm1() * err_scale;
double pinv_res2 = (pinv * M * pinv - pinv).Norm1() * err_scale;
double sv_res = R.Norm1() * err_scale;
double rank = SV.Rank();
double cond = SV.Cond();
Console.WriteLine("{0} x {1}, rank:{2:F4}, cond:{3:F4}, sv:{4:F4}, pinv:{5:F4}, {6:F4}", m, n, rank, cond, sv_res, pinv_res1, pinv_res2);
}
}
//linear systems: square with full rank
for (int n = 1; n <= 32; n++)
{
int m = 10;
ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(n, n, rng);
ACQ.Math.Linalg.Matrix B = new Math.Linalg.Matrix(n, m);
for (int i = 0; i < M.Rows; i++)
{
double sum = 0;
for (int j = 0; j < M.Columns; j++)
{
sum += M[i, j];
}
for (int k = 0; k < B.Columns; k++)
{
B[i, k] = sum * (k + 1);
}
}
double err_scale = 1.0 / (n * eps);
ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M);
ACQ.Math.Linalg.Matrix X1 = SV.Solve(B);
ACQ.Math.Linalg.Matrix X2 = SV.PseudoInverse() * B;
double sv_res1 = (M * X1 - B).Norm1() * err_scale;
double sv_res2 = (M * X2 - B).Norm1() * err_scale;
ACQ.Math.Linalg.LuDecomposition LU = new ACQ.Math.Linalg.LuDecomposition(M);
X1 = LU.Solve(B);
double lu_res1 = (M * X1 - B).Norm1() * err_scale;
ACQ.Math.Linalg.QrDecomposition QR = new ACQ.Math.Linalg.QrDecomposition(M);
X1 = QR.Solve(B);
double qr_res1 = (M * X1 - B).Norm1() * err_scale;
Console.WriteLine("{0}, svd:{1:F4}, sv:{2:F4} lu:{3:F4} qr:{4:F4}", n, sv_res1, sv_res2, lu_res1, qr_res1);
}
//linear systems: symmetric with full rank
for (int n = 1; n <= 32; n++)
{
int m = 10;
ACQ.Math.Linalg.Matrix A = ACQ.Math.Linalg.Matrix.CreateRandom(n, n, rng);
ACQ.Math.Linalg.Matrix M = A * A.Transpose();
ACQ.Math.Linalg.Matrix B = new Math.Linalg.Matrix(n, m);
for (int i = 0; i < M.Rows; i++)
{
double sum = 0;
for (int j = 0; j < M.Columns; j++)
{
sum += M[i, j];
}
for (int k = 0; k < B.Columns; k++)
{
B[i, k] = sum * (k + 1);
}
}
ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M);
ACQ.Math.Linalg.Matrix X1 = SV.Solve(B);
ACQ.Math.Linalg.Matrix X2 = SV.PseudoInverse() * B;
double sv_res1 = (M * X1 - B).Norm1();
double sv_res2 = (M * X2 - B).Norm1();
ACQ.Math.Linalg.LuDecomposition LU = new ACQ.Math.Linalg.LuDecomposition(M);
X1 = LU.Solve(B);
double lu_res1 = (M * X1 - B).Norm1();
ACQ.Math.Linalg.QrDecomposition QR = new ACQ.Math.Linalg.QrDecomposition(M);
X1 = QR.Solve(B);
double qr_res1 = (M * X1 - B).Norm1();
ACQ.Math.Linalg.CholeskyDecomposition CD = new ACQ.Math.Linalg.CholeskyDecomposition(M);
X1 = CD.Solve(B);
double cd_res1 = (M * X1 - B).Norm1();
Console.WriteLine("{0,3}, sv:{1:E4} pinv:{2:E4} lu:{3:E4} qr:{4:E4}, cd:{5:E4}", n, sv_res1, sv_res2, lu_res1, qr_res1, cd_res1);
}
ACQ.Math.Random.IRandomGenerator rnorm = new ACQ.Math.Random.BoxMuller(rng);
//linear systems: overdetermined, full rank (least squares solution)
for (int n = 1; n <= 32; n++)
{
ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(2 * n, n, rnorm);
ACQ.Math.Linalg.Matrix B = ACQ.Math.Linalg.Matrix.CreateRandom(2 * n, 1, rnorm);
for (int i = 0; i < M.Rows; i++)
{
double sum = 0;
for (int k = 0; k < M.Columns; k++)
{
sum += M[i, k];
}
B[i, 0] = sum + 0.1 * B[i, 0];
}
ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M);
ACQ.Math.Linalg.Matrix Xsv = SV.Solve(B);
double sv_res1 = (M * Xsv - B).Norm1();
ACQ.Math.Linalg.QrDecomposition QR = new ACQ.Math.Linalg.QrDecomposition(M);
ACQ.Math.Linalg.Matrix Xqr = QR.Solve(B);
double qr_res1 = (M * Xqr - B).Norm1();
double res = (Xsv - Xqr).Norm1();
Console.WriteLine("{0,3}, sv:{1:E4} qr:{2:E4} res:{3:E4}", n, sv_res1, qr_res1, res);
}
//linear systems: overdetermined but not full rank (least squares solution),
//QR does not work in this case,
for (int n = 1; n <= 32; n++)
{
ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(2 * n, n + 1, rnorm);
ACQ.Math.Linalg.Matrix B = ACQ.Math.Linalg.Matrix.CreateRandom(2 * n, 1, rnorm);
for (int i = 0; i < M.Rows; i++)
{
M[i, n] = M[i, n - 1];//dublicate column
double sum = 0;
for (int k = 0; k < M.Columns; k++)
{
sum += M[i, k];
}
B[i, 0] = sum + 0.1 * B[i, 0];
}
//solution shold be approximatly all ones, last two values should be the same
//condition number is large (infinite actually, sometimes finite because of truncation errors)
//rank = M.Columns - 1 (i.e. rank = n)
ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M);
ACQ.Math.Linalg.Matrix Xsv = SV.Solve(B);
double sv_res1 = (M * Xsv - B).Norm1();
ACQ.Math.Linalg.QrDecomposition QR = new ACQ.Math.Linalg.QrDecomposition(M);
double qr_res1 = Double.NaN;
double res = Double.NaN;
if (QR.IsFullRank)
{
//occasionally QR thinks that matrix is full rank,
//but it is poorly conditioned so solution is wrong
ACQ.Math.Linalg.Matrix Xqr = QR.Solve(B);
qr_res1 = (M * Xqr - B).Norm1();
res = (Xsv - Xqr).Norm1();
}
Console.WriteLine("{0,3}, sv:{1:E4} qr:{2:E4} res:{3:E4}", n, sv_res1, qr_res1, res);
}
//linear systems: underdetermined (least norm solution)
for (int n = 1; n <= 32; n++)
{
ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(n, 2 * n + 1, rnorm);
ACQ.Math.Linalg.Matrix B = ACQ.Math.Linalg.Matrix.CreateRandom(n, 1, rnorm);
for (int i = 0; i < M.Rows; i++)
{
M[i, 2 * n] = M[i, 2 * n - 1];//dublicate column
}
ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M.Transpose());
ACQ.Math.Linalg.Matrix Xsv = SV.Solve(B, true); //last two values should be the same
double sv_res1 = (M * Xsv - B).Norm1(); //this should be zero
double x_norm = Xsv.Norm1();
Console.WriteLine("{0,3}, sv:{1:E4} xnorm:{2:E4}", n, sv_res1, x_norm);
}
}
/// <summary>
/// 1D constructor
/// </summary>
/// <param name="x"></param>
/// <param name="y"></param>
/// <param name="w"></param>
/// <param name="intercept"></param>
public LinearRegression(double[] x, double[] y, double[] w, bool intercept)
{
if (x == null || y == null || x.Length != y.Length)
{
throw new ArgumentException("LinearRegression: check input arguments (x and y can't be null and must have the same size)");
}
m_weighted = (w != null);
if (m_weighted && w.Length != y.Length)
{
throw new ArgumentException("LinearRegression: array of weights should have the same length as x and y");
}
m_intercept = intercept;
//init variable names
m_names = new string[1 + (m_intercept ? 1 : 0)];
m_names[0] = "c1";
if (m_intercept)
{
m_names[1] = "(intercept)";
}
Math.Linalg.Matrix A = null;
int n = x.Length;
//check input for NaN
for (int i = 0; i < x.Length; i++)
{
if (Double.IsNaN(x[i]) || Double.IsNaN(y[i]))
{
throw new ArgumentException("LinearRegression: there should not be NaN values in x or y");
}
if (m_weighted && Double.IsNaN(w[i]))
{
throw new ArgumentException("LinearRegression: weights vector should not have NaN values");
}
}
if (m_intercept)
{
int m = 1;
A = new Linalg.Matrix(n, m + 1);
for (int i = 0; i < n; i++)
{
A[i, 0] = x[i];
A[i, 1] = 1.0; //intercept
}
}
else
{
A = new Linalg.Matrix(x);
}
compute_coefficients(A, y, w);
}
