Пример #1
0
 /// <summary>
 /// Initializes a new instance of the RidgeRegression class.
 /// </summary>
 /// <param name="alpha">
 /// Small positive values of alpha improve the conditioning of the problem
 /// and reduce the variance of the estimates.  Alpha corresponds to
 /// ``(2*C)^-1`` in other linear models such as LogisticRegression or
 /// LinearSVC.
 /// </param>
 /// <param name="fitIntercept">
 /// Whether to calculate the intercept for this model. If set
 /// to false, no intercept will be used in calculations
 /// (e.g. data is expected to be already centered).
 /// </param>
 /// <param name="normalize">
 /// If True, the regressors X will be normalized before regression.
 /// </param>
 /// <param name="maxIter">
 /// Maximum number of iterations for conjugate gradient solver.
 /// The default value is determined by Math.Net.
 /// </param>
 /// <param name="tol">Precision of the solution.</param>
 /// <param name="solver">Solver to use in the computational routines.</param>
 public RidgeRegression(
     double alpha = 1.0,
     bool fitIntercept = true,
     bool normalize = false,
     int? maxIter = null,
     double tol = 1e-3,
     RidgeSolver solver = RidgeSolver.Auto) : base(fitIntercept, alpha, normalize, maxIter, tol, solver)
 {
 }
Пример #2
0
 /// <summary>
 /// Initializes a new instance of the RidgeRegression class.
 /// </summary>
 /// <param name="alpha">
 /// Small positive values of alpha improve the conditioning of the problem
 /// and reduce the variance of the estimates.  Alpha corresponds to
 /// ``(2*C)^-1`` in other linear models such as LogisticRegression or
 /// LinearSVC.
 /// </param>
 /// <param name="fitIntercept">
 /// Whether to calculate the intercept for this model. If set
 /// to false, no intercept will be used in calculations
 /// (e.g. data is expected to be already centered).
 /// </param>
 /// <param name="normalize">
 /// If True, the regressors X will be normalized before regression.
 /// </param>
 /// <param name="maxIter">
 /// Maximum number of iterations for conjugate gradient solver.
 /// The default value is determined by Math.Net.
 /// </param>
 /// <param name="tol">Precision of the solution.</param>
 /// <param name="solver">Solver to use in the computational routines.</param>
 public RidgeRegression(
     double alpha       = 1.0,
     bool fitIntercept  = true,
     bool normalize     = false,
     int?maxIter        = null,
     double tol         = 1e-3,
     RidgeSolver solver = RidgeSolver.Auto) : base(fitIntercept, alpha, normalize, maxIter, tol, solver)
 {
 }
Пример #3
0
 internal RidgeBase(
     bool fitIntercept,
     double alpha       = 1.0,
     bool normalize     = false,
     int?maxIter        = null,
     double tol         = 1e-3,
     RidgeSolver solver = RidgeSolver.Auto) : base(fitIntercept)
 {
     this.Alpha     = alpha;
     this.Normalize = normalize;
     this.MaxIter   = maxIter;
     this.Tol       = tol;
     this.Solver    = solver;
 }
Пример #4
0
 internal RidgeBase(
     bool fitIntercept,
     double alpha = 1.0,
     bool normalize = false,
     int? maxIter = null,
     double tol = 1e-3,
     RidgeSolver solver = RidgeSolver.Auto) : base(fitIntercept)
 {
     this.Alpha = alpha;
     this.Normalize = normalize;
     this.MaxIter = maxIter;
     this.Tol = tol;
     this.Solver = solver;
 }
Пример #5
0
        /// <summary>
        /// Initializes a new instance of the RidgeClassifier class.
        /// </summary>
        /// <param name="alpha"> Small positive values of alpha improve the conditioning of the problem
        /// and reduce the variance of the estimates.  Alpha corresponds to
        /// ``(2*C)^-1`` in other linear models such as LogisticRegression or
        /// LinearSVC.</param>
        /// <param name="fitIntercept">Whether to calculate the intercept for this model. If set to false, no
        /// intercept will be used in calculations (e.g. data is expected to be
        /// already centered).</param>
        /// <param name="normalize">If True, the regressors X will be normalized before regression.</param>
        /// <param name="maxIter">Maximum number of iterations for conjugate gradient solver.
        /// The default value is determined by Math.Net.</param>
        /// <param name="tol">Precision of the solution.</param>
        /// <param name="classWeightEstimator">Weights associated with classes in the form
        /// {class_label : weight}. If not given, all classes are
        /// supposed to have weight one.</param>
        /// <param name="solver">Solver to use in the computational.</param>
        public RidgeClassifier(
            double alpha      = 1.0,
            bool fitIntercept = true,
            bool normalize    = false,
            int?maxIter       = null,
            double tol        = 1e-3,
            ClassWeightEstimator <TLabel> classWeightEstimator = null,
            RidgeSolver solver = RidgeSolver.Auto) : base(fitIntercept, alpha, normalize, maxIter, tol, solver)
        {
            if (classWeightEstimator == ClassWeightEstimator <TLabel> .Auto)
            {
                throw new ArgumentException("ClassWeight.Auto is not supported.");
            }

            this.ClassWeightEstimator = classWeightEstimator ?? ClassWeightEstimator <TLabel> .Uniform;
        }
Пример #6
0
        /// <summary>
        /// Solve the ridge equation by the method of normal equations.
        /// </summary>
        /// <param name="x">[n_samples, n_features]
        /// Training data</param>
        /// <param name="y">[n_samples, n_targets]
        /// Target values</param>
        /// <param name="alpha"></param>
        /// <param name="sampleWeight">Individual weights for each sample.</param>
        /// <param name="solver">Solver to use in the computational routines.</param>
        /// <param name="maxIter">Maximum number of iterations for least squares solver. </param>
        /// <param name="tol">Precision of the solution.</param>
        /// <returns>[n_targets, n_features]
        /// Weight vector(s)</returns>
        /// <remarks>
        /// This function won't compute the intercept;
        /// </remarks>
        public static Matrix <double> RidgeRegression(
            Matrix <double> x,
            Matrix <double> y,
            double alpha,
            Vector <double> sampleWeight = null,
            RidgeSolver solver           = RidgeSolver.Auto,
            int?maxIter = null,
            double tol  = 1E-3)
        {
            int nSamples  = x.RowCount;
            int nFeatures = x.ColumnCount;

            if (solver == RidgeSolver.Auto)
            {
                // cholesky if it's a dense array and lsqr in
                // any other case
                if (x is DenseMatrix)
                {
                    solver = RidgeSolver.DenseCholesky;
                }
                else
                {
                    solver = RidgeSolver.Lsqr;
                }
            }

            if (sampleWeight != null)
            {
                solver = RidgeSolver.DenseCholesky;
            }

            if (solver == RidgeSolver.Lsqr)
            {
                // According to the lsqr documentation, alpha = damp^2.
                double sqrtAlpha = Math.Sqrt(alpha);
                Matrix coefs     = new DenseMatrix(y.ColumnCount, x.ColumnCount);
                foreach (var column in y.ColumnEnumerator())
                {
                    Vector <double> c = Lsqr.lsqr(
                        x,
                        column.Item2,
                        damp: sqrtAlpha,
                        atol: tol,
                        btol: tol,
                        iterLim: maxIter).X;

                    coefs.SetRow(column.Item1, c);
                }

                return(coefs);
            }

            if (solver == RidgeSolver.DenseCholesky)
            {
                //# normal equations (cholesky) method
                if (nFeatures > nSamples || sampleWeight != null)
                {
                    // kernel ridge
                    // w = X.T * inv(X X^t + alpha*Id) y
                    var             k  = x.TransposeAndMultiply(x);
                    Vector <double> sw = null;
                    if (sampleWeight != null)
                    {
                        sw = sampleWeight.Sqrt();
                        // We are doing a little danse with the sample weights to
                        // avoid copying the original X, which could be big

                        y = y.MulColumnVector(sw);

                        k = k.PointwiseMultiply(sw.Outer(sw));
                    }

                    k.Add(DenseMatrix.Identity(k.RowCount) * alpha, k);
                    try
                    {
                        var dualCoef = k.Cholesky().Solve(y);
                        if (sampleWeight != null)
                        {
                            dualCoef = dualCoef.MulColumnVector(sw);
                        }

                        return(x.TransposeThisAndMultiply(dualCoef).Transpose());
                    }
                    catch (Exception) //todo:
                    {
                        // use SVD solver if matrix is singular
                        solver = RidgeSolver.Svd;
                    }
                }
                else
                {
                    // ridge
                    // w = inv(X^t X + alpha*Id) * X.T y
                    var a = x.TransposeThisAndMultiply(x);
                    a.Add(DenseMatrix.Identity(a.ColumnCount) * alpha, a);

                    var xy = x.TransposeThisAndMultiply(y);

                    try
                    {
                        return(a.Cholesky().Solve(xy).Transpose());
                    }
                    catch (Exception) //todo:
                    {
                        // use SVD solver if matrix is singular
                        solver = RidgeSolver.Svd;
                    }
                }
            }

            if (solver == RidgeSolver.Svd)
            {
                // slower than cholesky but does not break with
                // singular matrices
                var svd = x.Svd(true);
                //U, s, Vt = linalg.svd(X, full_matrices=False)
                int k = Math.Min(x.ColumnCount, x.RowCount);
                var d = svd.S().SubVector(0, k);
                d.MapInplace(v => v > 1e-15 ? v / (v * v + alpha) : 0.0);

                var ud = svd.U().SubMatrix(0, x.RowCount, 0, k).TransposeThisAndMultiply(y).Transpose();
                ud = ud.MulRowVector(d);
                return(ud.Multiply(svd.VT().SubMatrix(0, k, 0, x.ColumnCount)));
            }

            return(null);
        }
Пример #7
0
        /// <summary>
        /// Solve the ridge equation by the method of normal equations.
        /// </summary>
        /// <param name="x">[n_samples, n_features]
        /// Training data</param>
        /// <param name="y">[n_samples, n_targets]
        /// Target values</param>
        /// <param name="alpha"></param>
        /// <param name="sampleWeight">Individual weights for each sample.</param>
        /// <param name="solver">Solver to use in the computational routines.</param>
        /// <param name="maxIter">Maximum number of iterations for least squares solver. </param>
        /// <param name="tol">Precision of the solution.</param>
        /// <returns>[n_targets, n_features]
        /// Weight vector(s)</returns>
        /// <remarks>
        /// This function won't compute the intercept;
        /// </remarks>
        public static Matrix<double> RidgeRegression(
            Matrix<double> x,
            Matrix<double> y,
            double alpha,
            Vector<double> sampleWeight = null,
            RidgeSolver solver = RidgeSolver.Auto,
            int? maxIter = null,
            double tol = 1E-3)
        {
            int nSamples = x.RowCount;
            int nFeatures = x.ColumnCount;

            if (solver == RidgeSolver.Auto)
            {
                // cholesky if it's a dense array and lsqr in
                // any other case
                if (x is DenseMatrix)
                {
                    solver = RidgeSolver.DenseCholesky;
                }
                else
                {
                    solver = RidgeSolver.Lsqr;
                }
            }

            if (sampleWeight != null)
            {
                solver = RidgeSolver.DenseCholesky;
            }

            if (solver == RidgeSolver.Lsqr)
            {
                // According to the lsqr documentation, alpha = damp^2.
                double sqrtAlpha = Math.Sqrt(alpha);
                Matrix coefs = new DenseMatrix(y.ColumnCount, x.ColumnCount);
                foreach (var column in y.ColumnEnumerator())
                {
                    Vector<double> c = Lsqr.lsqr(
                        x,
                        column.Item2,
                        damp: sqrtAlpha,
                        atol: tol,
                        btol: tol,
                        iterLim: maxIter).X;

                    coefs.SetRow(column.Item1, c);
                }

                return coefs;
            }

            if (solver == RidgeSolver.DenseCholesky)
            {
                //# normal equations (cholesky) method
                if (nFeatures > nSamples || sampleWeight != null)
                {
                    // kernel ridge
                    // w = X.T * inv(X X^t + alpha*Id) y
                    var k = x.TransposeAndMultiply(x);
                    Vector<double> sw = null;
                    if (sampleWeight != null)
                    {
                        sw = sampleWeight.Sqrt();
                        // We are doing a little danse with the sample weights to
                        // avoid copying the original X, which could be big

                        y = y.MulColumnVector(sw);

                        k = k.PointwiseMultiply(sw.Outer(sw));
                    }

                    k.Add(DenseMatrix.Identity(k.RowCount)*alpha, k);
                    try
                    {
                        var dualCoef = k.Cholesky().Solve(y);
                        if (sampleWeight != null)
                            dualCoef = dualCoef.MulColumnVector(sw);

                        return x.TransposeThisAndMultiply(dualCoef).Transpose();
                    }
                    catch (Exception) //todo:
                    {
                        // use SVD solver if matrix is singular
                        solver = RidgeSolver.Svd;
                    }
                }
                else
                {
                    // ridge
                    // w = inv(X^t X + alpha*Id) * X.T y
                    var a = x.TransposeThisAndMultiply(x);
                    a.Add(DenseMatrix.Identity(a.ColumnCount)*alpha, a);

                    var xy = x.TransposeThisAndMultiply(y);

                    try
                    {
                        return a.Cholesky().Solve(xy).Transpose();
                    }
                    catch (Exception) //todo:
                    {
                        // use SVD solver if matrix is singular
                        solver = RidgeSolver.Svd;
                    }
                }
            }

            if (solver == RidgeSolver.Svd)
            {
                // slower than cholesky but does not break with
                // singular matrices
                var svd = x.Svd(true);
                //U, s, Vt = linalg.svd(X, full_matrices=False)
                int k = Math.Min(x.ColumnCount, x.RowCount);
                var d = svd.S().SubVector(0, k);
                d.MapInplace(v => v > 1e-15 ? v/(v*v + alpha) : 0.0);

                var ud = svd.U().SubMatrix(0, x.RowCount, 0, k).TransposeThisAndMultiply(y).Transpose();
                ud = ud.MulRowVector(d);
                return ud.Multiply(svd.VT().SubMatrix(0, k, 0, x.ColumnCount));
            }

            return null;
        }