/// <summary>
/// Computes the total error of the solution with parameters theta.
/// </summary>
/// <param name="X">Training data</param>
/// <param name="y">Target variable</param>
/// <param name="theta">Model parameters</param>
/// <param name="lambda">Regularization weight</param
/// <returns>Solution error</returns>
private double ComputeError(InsightMatrix X, InsightVector y, InsightVector theta, double lambda)
{
var inner = ((X * theta.ToColumnMatrix()) - y.ToColumnMatrix()).Power(2);
var thetaSub = theta.SubVector(1, theta.Count - 1);
var reg = lambda * thetaSub.Multiply(thetaSub).Sum();
return (inner.Column(0).Sum() / (2 * X.RowCount)) + reg;
}
/// <summary>
/// Performs linear regression on the input data.
/// </summary>
/// <param name="data">Training data</param>
/// <param name="alpha">The learning rate for the algorithm</param>
/// <param name="lambda">The regularization weight for the algorithm</param>
/// <param name="iters">The number of training iterations to run</param>
/// <returns>Tuple containing the parameter and error vectors</returns>
private Tuple<InsightVector, InsightVector> PerformLinearRegression(InsightMatrix data, double alpha,
double lambda, int iters)
{
// First add a ones column for the intercept term
data = data.InsertColumn(0, 1);
// Split the data into training data and the target variable
var X = data.RemoveColumn(data.ColumnCount - 1);
var y = data.Column(data.ColumnCount - 1);
// Initialize several variables needed for the computation
var theta = new InsightVector(X.ColumnCount);
var temp = new InsightVector(X.ColumnCount);
var error = new InsightVector(iters);
// Perform gradient descent on the parameters theta
for (int i = 0; i < iters; i++)
{
var delta = (X * theta.ToColumnMatrix()) - y.ToColumnMatrix();
for (int j = 0; j < theta.Count; j++)
{
var inner = delta.Multiply(X.SubMatrix(0, X.RowCount, j, 1));
if (j == 0)
{
temp[j] = theta[j] - ((alpha / X.RowCount) * inner.Column(0).Sum());
}
else
{
var reg = (2 * lambda) * theta[j];
temp[j] = theta[j] - ((alpha / X.RowCount) * inner.Column(0).Sum()) + reg;
}
}
theta = temp.Clone();
error[i] = ComputeError(X, y, theta, lambda);
}
return new Tuple<InsightVector, InsightVector>(theta, error);
}
/// <summary>
/// Computes the total error of the solution with parameters theta.
/// </summary>
/// <param name="X">Training data</param>
/// <param name="y">Target variable</param>
/// <param name="theta">Model parameters</param>
/// <param name="lambda">Regularization weight</param
/// <returns>Solution error</returns>
private double ComputeError(InsightMatrix X, InsightVector y, InsightVector theta, double lambda)
{
var first = y.Multiply(Sigmoid((X * theta.ToColumnMatrix()).Column(0)).Log());
var second = (1 - y).Multiply(1 - Sigmoid((X * theta.ToColumnMatrix()).Column(0)).Log());
var thetaSub = theta.SubVector(1, theta.Count - 1);
var reg = (lambda / 2 * X.RowCount) * thetaSub.Power(2).Sum();
return (first - second).Sum() / X.RowCount + reg;
}
