/// <summary>
/// Initializes a new instance of the RidgeRegression class.
/// </summary>
/// <param name="alpha">
/// Small positive values of alpha improve the conditioning of the problem
/// and reduce the variance of the estimates. Alpha corresponds to
/// ``(2*C)^-1`` in other linear models such as LogisticRegression or
/// LinearSVC.
/// </param>
/// <param name="fitIntercept">
/// Whether to calculate the intercept for this model. If set
/// to false, no intercept will be used in calculations
/// (e.g. data is expected to be already centered).
/// </param>
/// <param name="normalize">
/// If True, the regressors X will be normalized before regression.
/// </param>
/// <param name="maxIter">
/// Maximum number of iterations for conjugate gradient solver.
/// The default value is determined by Math.Net.
/// </param>
/// <param name="tol">Precision of the solution.</param>
/// <param name="solver">Solver to use in the computational routines.</param>
public RidgeRegression(
double alpha = 1.0,
bool fitIntercept = true,
bool normalize = false,
int? maxIter = null,
double tol = 1e-3,
RidgeSolver solver = RidgeSolver.Auto) : base(fitIntercept, alpha, normalize, maxIter, tol, solver)
{
}
/// <summary>
/// Initializes a new instance of the RidgeRegression class.
/// </summary>
/// <param name="alpha">
/// Small positive values of alpha improve the conditioning of the problem
/// and reduce the variance of the estimates. Alpha corresponds to
/// ``(2*C)^-1`` in other linear models such as LogisticRegression or
/// LinearSVC.
/// </param>
/// <param name="fitIntercept">
/// Whether to calculate the intercept for this model. If set
/// to false, no intercept will be used in calculations
/// (e.g. data is expected to be already centered).
/// </param>
/// <param name="normalize">
/// If True, the regressors X will be normalized before regression.
/// </param>
/// <param name="maxIter">
/// Maximum number of iterations for conjugate gradient solver.
/// The default value is determined by Math.Net.
/// </param>
/// <param name="tol">Precision of the solution.</param>
/// <param name="solver">Solver to use in the computational routines.</param>
public RidgeRegression(
double alpha = 1.0,
bool fitIntercept = true,
bool normalize = false,
int?maxIter = null,
double tol = 1e-3,
RidgeSolver solver = RidgeSolver.Auto) : base(fitIntercept, alpha, normalize, maxIter, tol, solver)
{
}
internal RidgeBase(
bool fitIntercept,
double alpha = 1.0,
bool normalize = false,
int?maxIter = null,
double tol = 1e-3,
RidgeSolver solver = RidgeSolver.Auto) : base(fitIntercept)
{
this.Alpha = alpha;
this.Normalize = normalize;
this.MaxIter = maxIter;
this.Tol = tol;
this.Solver = solver;
}
internal RidgeBase(
bool fitIntercept,
double alpha = 1.0,
bool normalize = false,
int? maxIter = null,
double tol = 1e-3,
RidgeSolver solver = RidgeSolver.Auto) : base(fitIntercept)
{
this.Alpha = alpha;
this.Normalize = normalize;
this.MaxIter = maxIter;
this.Tol = tol;
this.Solver = solver;
}
/// <summary>
/// Initializes a new instance of the RidgeClassifier class.
/// </summary>
/// <param name="alpha"> Small positive values of alpha improve the conditioning of the problem
/// and reduce the variance of the estimates. Alpha corresponds to
/// ``(2*C)^-1`` in other linear models such as LogisticRegression or
/// LinearSVC.</param>
/// <param name="fitIntercept">Whether to calculate the intercept for this model. If set to false, no
/// intercept will be used in calculations (e.g. data is expected to be
/// already centered).</param>
/// <param name="normalize">If True, the regressors X will be normalized before regression.</param>
/// <param name="maxIter">Maximum number of iterations for conjugate gradient solver.
/// The default value is determined by Math.Net.</param>
/// <param name="tol">Precision of the solution.</param>
/// <param name="classWeightEstimator">Weights associated with classes in the form
/// {class_label : weight}. If not given, all classes are
/// supposed to have weight one.</param>
/// <param name="solver">Solver to use in the computational.</param>
public RidgeClassifier(
double alpha = 1.0,
bool fitIntercept = true,
bool normalize = false,
int?maxIter = null,
double tol = 1e-3,
ClassWeightEstimator <TLabel> classWeightEstimator = null,
RidgeSolver solver = RidgeSolver.Auto) : base(fitIntercept, alpha, normalize, maxIter, tol, solver)
{
if (classWeightEstimator == ClassWeightEstimator <TLabel> .Auto)
{
throw new ArgumentException("ClassWeight.Auto is not supported.");
}
this.ClassWeightEstimator = classWeightEstimator ?? ClassWeightEstimator <TLabel> .Uniform;
}
/// <summary>
/// Solve the ridge equation by the method of normal equations.
/// </summary>
/// <param name="x">[n_samples, n_features]
/// Training data</param>
/// <param name="y">[n_samples, n_targets]
/// Target values</param>
/// <param name="alpha"></param>
/// <param name="sampleWeight">Individual weights for each sample.</param>
/// <param name="solver">Solver to use in the computational routines.</param>
/// <param name="maxIter">Maximum number of iterations for least squares solver. </param>
/// <param name="tol">Precision of the solution.</param>
/// <returns>[n_targets, n_features]
/// Weight vector(s)</returns>
/// <remarks>
/// This function won't compute the intercept;
/// </remarks>
public static Matrix <double> RidgeRegression(
Matrix <double> x,
Matrix <double> y,
double alpha,
Vector <double> sampleWeight = null,
RidgeSolver solver = RidgeSolver.Auto,
int?maxIter = null,
double tol = 1E-3)
{
int nSamples = x.RowCount;
int nFeatures = x.ColumnCount;
if (solver == RidgeSolver.Auto)
{
// cholesky if it's a dense array and lsqr in
// any other case
if (x is DenseMatrix)
{
solver = RidgeSolver.DenseCholesky;
}
else
{
solver = RidgeSolver.Lsqr;
}
}
if (sampleWeight != null)
{
solver = RidgeSolver.DenseCholesky;
}
if (solver == RidgeSolver.Lsqr)
{
// According to the lsqr documentation, alpha = damp^2.
double sqrtAlpha = Math.Sqrt(alpha);
Matrix coefs = new DenseMatrix(y.ColumnCount, x.ColumnCount);
foreach (var column in y.ColumnEnumerator())
{
Vector <double> c = Lsqr.lsqr(
x,
column.Item2,
damp: sqrtAlpha,
atol: tol,
btol: tol,
iterLim: maxIter).X;
coefs.SetRow(column.Item1, c);
}
return(coefs);
}
if (solver == RidgeSolver.DenseCholesky)
{
//# normal equations (cholesky) method
if (nFeatures > nSamples || sampleWeight != null)
{
// kernel ridge
// w = X.T * inv(X X^t + alpha*Id) y
var k = x.TransposeAndMultiply(x);
Vector <double> sw = null;
if (sampleWeight != null)
{
sw = sampleWeight.Sqrt();
// We are doing a little danse with the sample weights to
// avoid copying the original X, which could be big
y = y.MulColumnVector(sw);
k = k.PointwiseMultiply(sw.Outer(sw));
}
k.Add(DenseMatrix.Identity(k.RowCount) * alpha, k);
try
{
var dualCoef = k.Cholesky().Solve(y);
if (sampleWeight != null)
{
dualCoef = dualCoef.MulColumnVector(sw);
}
return(x.TransposeThisAndMultiply(dualCoef).Transpose());
}
catch (Exception) //todo:
{
// use SVD solver if matrix is singular
solver = RidgeSolver.Svd;
}
}
else
{
// ridge
// w = inv(X^t X + alpha*Id) * X.T y
var a = x.TransposeThisAndMultiply(x);
a.Add(DenseMatrix.Identity(a.ColumnCount) * alpha, a);
var xy = x.TransposeThisAndMultiply(y);
try
{
return(a.Cholesky().Solve(xy).Transpose());
}
catch (Exception) //todo:
{
// use SVD solver if matrix is singular
solver = RidgeSolver.Svd;
}
}
}
if (solver == RidgeSolver.Svd)
{
// slower than cholesky but does not break with
// singular matrices
var svd = x.Svd(true);
//U, s, Vt = linalg.svd(X, full_matrices=False)
int k = Math.Min(x.ColumnCount, x.RowCount);
var d = svd.S().SubVector(0, k);
d.MapInplace(v => v > 1e-15 ? v / (v * v + alpha) : 0.0);
var ud = svd.U().SubMatrix(0, x.RowCount, 0, k).TransposeThisAndMultiply(y).Transpose();
ud = ud.MulRowVector(d);
return(ud.Multiply(svd.VT().SubMatrix(0, k, 0, x.ColumnCount)));
}
return(null);
}
/// <summary>
/// Solve the ridge equation by the method of normal equations.
/// </summary>
/// <param name="x">[n_samples, n_features]
/// Training data</param>
/// <param name="y">[n_samples, n_targets]
/// Target values</param>
/// <param name="alpha"></param>
/// <param name="sampleWeight">Individual weights for each sample.</param>
/// <param name="solver">Solver to use in the computational routines.</param>
/// <param name="maxIter">Maximum number of iterations for least squares solver. </param>
/// <param name="tol">Precision of the solution.</param>
/// <returns>[n_targets, n_features]
/// Weight vector(s)</returns>
/// <remarks>
/// This function won't compute the intercept;
/// </remarks>
public static Matrix<double> RidgeRegression(
Matrix<double> x,
Matrix<double> y,
double alpha,
Vector<double> sampleWeight = null,
RidgeSolver solver = RidgeSolver.Auto,
int? maxIter = null,
double tol = 1E-3)
{
int nSamples = x.RowCount;
int nFeatures = x.ColumnCount;
if (solver == RidgeSolver.Auto)
{
// cholesky if it's a dense array and lsqr in
// any other case
if (x is DenseMatrix)
{
solver = RidgeSolver.DenseCholesky;
}
else
{
solver = RidgeSolver.Lsqr;
}
}
if (sampleWeight != null)
{
solver = RidgeSolver.DenseCholesky;
}
if (solver == RidgeSolver.Lsqr)
{
// According to the lsqr documentation, alpha = damp^2.
double sqrtAlpha = Math.Sqrt(alpha);
Matrix coefs = new DenseMatrix(y.ColumnCount, x.ColumnCount);
foreach (var column in y.ColumnEnumerator())
{
Vector<double> c = Lsqr.lsqr(
x,
column.Item2,
damp: sqrtAlpha,
atol: tol,
btol: tol,
iterLim: maxIter).X;
coefs.SetRow(column.Item1, c);
}
return coefs;
}
if (solver == RidgeSolver.DenseCholesky)
{
//# normal equations (cholesky) method
if (nFeatures > nSamples || sampleWeight != null)
{
// kernel ridge
// w = X.T * inv(X X^t + alpha*Id) y
var k = x.TransposeAndMultiply(x);
Vector<double> sw = null;
if (sampleWeight != null)
{
sw = sampleWeight.Sqrt();
// We are doing a little danse with the sample weights to
// avoid copying the original X, which could be big
y = y.MulColumnVector(sw);
k = k.PointwiseMultiply(sw.Outer(sw));
}
k.Add(DenseMatrix.Identity(k.RowCount)*alpha, k);
try
{
var dualCoef = k.Cholesky().Solve(y);
if (sampleWeight != null)
dualCoef = dualCoef.MulColumnVector(sw);
return x.TransposeThisAndMultiply(dualCoef).Transpose();
}
catch (Exception) //todo:
{
// use SVD solver if matrix is singular
solver = RidgeSolver.Svd;
}
}
else
{
// ridge
// w = inv(X^t X + alpha*Id) * X.T y
var a = x.TransposeThisAndMultiply(x);
a.Add(DenseMatrix.Identity(a.ColumnCount)*alpha, a);
var xy = x.TransposeThisAndMultiply(y);
try
{
return a.Cholesky().Solve(xy).Transpose();
}
catch (Exception) //todo:
{
// use SVD solver if matrix is singular
solver = RidgeSolver.Svd;
}
}
}
if (solver == RidgeSolver.Svd)
{
// slower than cholesky but does not break with
// singular matrices
var svd = x.Svd(true);
//U, s, Vt = linalg.svd(X, full_matrices=False)
int k = Math.Min(x.ColumnCount, x.RowCount);
var d = svd.S().SubVector(0, k);
d.MapInplace(v => v > 1e-15 ? v/(v*v + alpha) : 0.0);
var ud = svd.U().SubMatrix(0, x.RowCount, 0, k).TransposeThisAndMultiply(y).Transpose();
ud = ud.MulRowVector(d);
return ud.Multiply(svd.VT().SubMatrix(0, k, 0, x.ColumnCount));
}
return null;
}