private void compute_regression(double[,] x, double[] y, double[] w) { int m = x.GetLength(1); Math.Linalg.Matrix A = null; int n = x.GetLength(0); if (m_intercept) { A = new Linalg.Matrix(n, m + 1); for (int i = 0; i < n; i++) { for (int j = 0; j < m; j++) { A[i, j] = x[i, j]; } A[i, m] = 1.0; } } else { A = new Linalg.Matrix(x, false); } compute_coefficients(A, y, w); }
/// <summary> /// 1D constructor /// </summary> /// <param name="x"></param> /// <param name="y"></param> /// <param name="w"></param> /// <param name="intercept"></param> public LinearRegression(double[] x, double[] y, double[] w, bool intercept) { if (x == null || y == null || x.Length != y.Length) { throw new ArgumentException("LinearRegression: check input arguments"); } bool use_weights = (w != null); if (use_weights && w.Length != y.Length) { throw new ArgumentException("LinearRegression: array of weights should have the same length"); } m_intercept = intercept; //init variable names m_names = new string[1 + (m_intercept ? 1 : 0)]; m_names[0] = "c1"; if (m_intercept) { m_names[1] = "(intercept)"; } Math.Linalg.Matrix A = null; int n = x.Length; if (m_intercept) { int m = 1; A = new Linalg.Matrix(n, m + 1); for (int i = 0; i < n; i++) { A[i, 0] = x[0]; A[i, 1] = 1.0; //intercept } } else { A = new Linalg.Matrix(x, false); } compute_coefficients(A, y, w); }
public static void TestLinearAlgebra() { double eps = ACQ.Math.Const.epsilon; //test magic matrices for (int n = 3; n <= 32; n++) { ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateMagic(n); double trace = M.Trace(); // trace = diagonal sum, should be the magic sum, (n^3 + n)/2. ACQ.Math.Linalg.EigenvalueDecomposition E = new ACQ.Math.Linalg.EigenvalueDecomposition(0.5 * (M + (M.Transpose()))); double[] d = E.GetRealEigenvalues(); //maximum eigenvalue of (A + A')/2, should equal trace //int r = M.rank(); //should equal n if n is odd, be less than n if n is even //double c = M.cond(); ACQ.Math.Linalg.LuDecomposition LU = new ACQ.Math.Linalg.LuDecomposition(M); ACQ.Math.Linalg.Matrix L = LU.GetL(); ACQ.Math.Linalg.Matrix U = LU.GetU(); int[] p = LU.GetPivotIndex(); ACQ.Math.Linalg.Matrix R = L * U - M.Submatrix(p, 0, n - 1); double lu_res = R.Norm1(); ACQ.Math.Linalg.QrDecomposition QR = new ACQ.Math.Linalg.QrDecomposition(M); ACQ.Math.Linalg.Matrix Q = QR.GetQ(); R = QR.GetR(); R = Q * R - M; double qr_res = R.Norm1(); ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M); U = SV.GetU(); ACQ.Math.Linalg.Matrix S = SV.GetS(); ACQ.Math.Linalg.Matrix V = SV.GetV(); R = U * S * V.Transpose() - M; double sv_res = R.Norm1(); double rank = SV.Rank(); double cond = SV.Cond(); Console.WriteLine("n:{0,3} trace:{1:E4} ev:{2:E4} lu:{3:E4} qr:{4:E4} sv:{5:E4}", n, trace, d[d.Length - 1], lu_res, qr_res, sv_res); } //test random matrices ACQ.Math.Random.IRandomGenerator rng = new ACQ.Math.Random.MersenneTwister(); for (int n = 1; n <= 8; n++) { ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(n, n, rng); ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M); ACQ.Math.Linalg.Matrix U = SV.GetU(); ACQ.Math.Linalg.Matrix S = SV.GetS(); ACQ.Math.Linalg.Matrix V = SV.GetV(); ACQ.Math.Linalg.Matrix R = U * S * V.Transpose() - M; ACQ.Math.Linalg.Matrix pinv = SV.PseudoInverse(); double pinv_res1 = (M * pinv * M - M).Norm1(); double pinv_res2 = (pinv * M * pinv - pinv).Norm1(); double sv_res = R.Norm1(); double rank = SV.Rank(); double cond = SV.Cond(); Console.WriteLine("n:{0,3} rank:{1:E4} cond:{2:E4} sv:{3:E4} pinv:{4:E4} {5:E4}", n, rank, cond, sv_res, pinv_res1, pinv_res2); } //non square matrixes, m > n for (int n = 1; n <= 32; n++) { for (int m = n; m <= 32; m++) { ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(m, n, rng); double err_scale = 1.0 / (System.Math.Max(m, n) * eps); ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M); ACQ.Math.Linalg.Matrix U = SV.GetU(); ACQ.Math.Linalg.Matrix S = SV.GetS(); ACQ.Math.Linalg.Matrix V = SV.GetV(); ACQ.Math.Linalg.Matrix R = U * S * V.Transpose() - M; ACQ.Math.Linalg.Matrix pinv = SV.PseudoInverse(); double pinv_res1 = (M * pinv * M - M).Norm1() * err_scale; double pinv_res2 = (pinv * M * pinv - pinv).Norm1() * err_scale; double sv_res = R.Norm1() * err_scale; double rank = SV.Rank(); double cond = SV.Cond(); Console.WriteLine("{0} x {1}, rank:{2:F4}, cond:{3:F4}, sv:{4:F4}, pinv:{5:F4}, {6:F4}", m, n, rank, cond, sv_res, pinv_res1, pinv_res2); } } //linear systems: square with full rank for (int n = 1; n <= 32; n++) { int m = 10; ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(n, n, rng); ACQ.Math.Linalg.Matrix B = new Math.Linalg.Matrix(n, m); for (int i = 0; i < M.Rows; i++) { double sum = 0; for (int j = 0; j < M.Columns; j++) { sum += M[i, j]; } for (int k = 0; k < B.Columns; k++) { B[i, k] = sum * (k + 1); } } double err_scale = 1.0 / (n * eps); ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M); ACQ.Math.Linalg.Matrix X1 = SV.Solve(B); ACQ.Math.Linalg.Matrix X2 = SV.PseudoInverse() * B; double sv_res1 = (M * X1 - B).Norm1() * err_scale; double sv_res2 = (M * X2 - B).Norm1() * err_scale; ACQ.Math.Linalg.LuDecomposition LU = new ACQ.Math.Linalg.LuDecomposition(M); X1 = LU.Solve(B); double lu_res1 = (M * X1 - B).Norm1() * err_scale; ACQ.Math.Linalg.QrDecomposition QR = new ACQ.Math.Linalg.QrDecomposition(M); X1 = QR.Solve(B); double qr_res1 = (M * X1 - B).Norm1() * err_scale; Console.WriteLine("{0}, svd:{1:F4}, sv:{2:F4} lu:{3:F4} qr:{4:F4}", n, sv_res1, sv_res2, lu_res1, qr_res1); } //linear systems: symmetric with full rank for (int n = 1; n <= 32; n++) { int m = 10; ACQ.Math.Linalg.Matrix A = ACQ.Math.Linalg.Matrix.CreateRandom(n, n, rng); ACQ.Math.Linalg.Matrix M = A * A.Transpose(); ACQ.Math.Linalg.Matrix B = new Math.Linalg.Matrix(n, m); for (int i = 0; i < M.Rows; i++) { double sum = 0; for (int j = 0; j < M.Columns; j++) { sum += M[i, j]; } for (int k = 0; k < B.Columns; k++) { B[i, k] = sum * (k + 1); } } ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M); ACQ.Math.Linalg.Matrix X1 = SV.Solve(B); ACQ.Math.Linalg.Matrix X2 = SV.PseudoInverse() * B; double sv_res1 = (M * X1 - B).Norm1(); double sv_res2 = (M * X2 - B).Norm1(); ACQ.Math.Linalg.LuDecomposition LU = new ACQ.Math.Linalg.LuDecomposition(M); X1 = LU.Solve(B); double lu_res1 = (M * X1 - B).Norm1(); ACQ.Math.Linalg.QrDecomposition QR = new ACQ.Math.Linalg.QrDecomposition(M); X1 = QR.Solve(B); double qr_res1 = (M * X1 - B).Norm1(); ACQ.Math.Linalg.CholeskyDecomposition CD = new ACQ.Math.Linalg.CholeskyDecomposition(M); X1 = CD.Solve(B); double cd_res1 = (M * X1 - B).Norm1(); Console.WriteLine("{0,3}, sv:{1:E4} pinv:{2:E4} lu:{3:E4} qr:{4:E4}, cd:{5:E4}", n, sv_res1, sv_res2, lu_res1, qr_res1, cd_res1); } ACQ.Math.Random.IRandomGenerator rnorm = new ACQ.Math.Random.BoxMuller(rng); //linear systems: overdetermined, full rank (least squares solution) for (int n = 1; n <= 32; n++) { ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(2 * n, n, rnorm); ACQ.Math.Linalg.Matrix B = ACQ.Math.Linalg.Matrix.CreateRandom(2 * n, 1, rnorm); for (int i = 0; i < M.Rows; i++) { double sum = 0; for (int k = 0; k < M.Columns; k++) { sum += M[i, k]; } B[i, 0] = sum + 0.1 * B[i, 0]; } ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M); ACQ.Math.Linalg.Matrix Xsv = SV.Solve(B); double sv_res1 = (M * Xsv - B).Norm1(); ACQ.Math.Linalg.QrDecomposition QR = new ACQ.Math.Linalg.QrDecomposition(M); ACQ.Math.Linalg.Matrix Xqr = QR.Solve(B); double qr_res1 = (M * Xqr - B).Norm1(); double res = (Xsv - Xqr).Norm1(); Console.WriteLine("{0,3}, sv:{1:E4} qr:{2:E4} res:{3:E4}", n, sv_res1, qr_res1, res); } //linear systems: overdetermined but not full rank (least squares solution), //QR does not work in this case, for (int n = 1; n <= 32; n++) { ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(2 * n, n + 1, rnorm); ACQ.Math.Linalg.Matrix B = ACQ.Math.Linalg.Matrix.CreateRandom(2 * n, 1, rnorm); for (int i = 0; i < M.Rows; i++) { M[i, n] = M[i, n - 1];//dublicate column double sum = 0; for (int k = 0; k < M.Columns; k++) { sum += M[i, k]; } B[i, 0] = sum + 0.1 * B[i, 0]; } //solution shold be approximatly all ones, last two values should be the same //condition number is large (infinite actually, sometimes finite because of truncation errors) //rank = M.Columns - 1 (i.e. rank = n) ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M); ACQ.Math.Linalg.Matrix Xsv = SV.Solve(B); double sv_res1 = (M * Xsv - B).Norm1(); ACQ.Math.Linalg.QrDecomposition QR = new ACQ.Math.Linalg.QrDecomposition(M); double qr_res1 = Double.NaN; double res = Double.NaN; if (QR.IsFullRank) { //occasionally QR thinks that matrix is full rank, //but it is poorly conditioned so solution is wrong ACQ.Math.Linalg.Matrix Xqr = QR.Solve(B); qr_res1 = (M * Xqr - B).Norm1(); res = (Xsv - Xqr).Norm1(); } Console.WriteLine("{0,3}, sv:{1:E4} qr:{2:E4} res:{3:E4}", n, sv_res1, qr_res1, res); } //linear systems: underdetermined (least norm solution) for (int n = 1; n <= 32; n++) { ACQ.Math.Linalg.Matrix M = ACQ.Math.Linalg.Matrix.CreateRandom(n, 2 * n + 1, rnorm); ACQ.Math.Linalg.Matrix B = ACQ.Math.Linalg.Matrix.CreateRandom(n, 1, rnorm); for (int i = 0; i < M.Rows; i++) { M[i, 2 * n] = M[i, 2 * n - 1];//dublicate column } ACQ.Math.Linalg.SvDecomposition SV = new ACQ.Math.Linalg.SvDecomposition(M.Transpose()); ACQ.Math.Linalg.Matrix Xsv = SV.Solve(B, true); //last two values should be the same double sv_res1 = (M * Xsv - B).Norm1(); //this should be zero double x_norm = Xsv.Norm1(); Console.WriteLine("{0,3}, sv:{1:E4} xnorm:{2:E4}", n, sv_res1, x_norm); } }
/// <summary> /// 1D constructor /// </summary> /// <param name="x"></param> /// <param name="y"></param> /// <param name="w"></param> /// <param name="intercept"></param> public LinearRegression(double[] x, double[] y, double[] w, bool intercept) { if (x == null || y == null || x.Length != y.Length) { throw new ArgumentException("LinearRegression: check input arguments (x and y can't be null and must have the same size)"); } m_weighted = (w != null); if (m_weighted && w.Length != y.Length) { throw new ArgumentException("LinearRegression: array of weights should have the same length as x and y"); } m_intercept = intercept; //init variable names m_names = new string[1 + (m_intercept ? 1 : 0)]; m_names[0] = "c1"; if (m_intercept) { m_names[1] = "(intercept)"; } Math.Linalg.Matrix A = null; int n = x.Length; //check input for NaN for (int i = 0; i < x.Length; i++) { if (Double.IsNaN(x[i]) || Double.IsNaN(y[i])) { throw new ArgumentException("LinearRegression: there should not be NaN values in x or y"); } if (m_weighted && Double.IsNaN(w[i])) { throw new ArgumentException("LinearRegression: weights vector should not have NaN values"); } } if (m_intercept) { int m = 1; A = new Linalg.Matrix(n, m + 1); for (int i = 0; i < n; i++) { A[i, 0] = x[i]; A[i, 1] = 1.0; //intercept } } else { A = new Linalg.Matrix(x); } compute_coefficients(A, y, w); }