/// <summary>
/// Performs gradient descent to optomise theta parameters.
/// </summary>
/// <param name="theta">Initial Theta (Zeros)</param>
/// <param name="x">Training set</param>
/// <param name="y">Training labels</param>
/// <param name="maxIterations">Maximum number of iterations to run gradient descent</param>
/// <param name="learningRateAlpha">The learning rate (Alpha)</param>
/// <param name="costFunction">Cost function to use for gradient descent</param>
/// <param name="lambda">The regularization constant to apply</param>
/// <param name="regularizer">The regularization function to apply</param>
/// <returns></returns>
public static Tuple<double, Vector> Run(
Vector theta,
Matrix x,
Vector y,
int maxIterations,
double learningRateAlpha,
ICostFunction costFunction,
double lambda,
IRegularizer regularizer)
{
var bestTheta = theta.Copy();
var bestCost = double.PositiveInfinity;
double currentCost = 0;
var currentGradient = theta.Copy();
for (var i = 0; i <= maxIterations; i++)
{
currentCost = costFunction.ComputeCost(bestTheta, x, y, lambda, regularizer);
currentGradient = costFunction.ComputeGradient(bestTheta, x, y, lambda, regularizer);
if (currentCost < bestCost)
{
bestTheta = bestTheta - learningRateAlpha * currentGradient;
bestCost = currentCost;
}
else
{
learningRateAlpha = learningRateAlpha * 0.99;
}
}
return new Tuple<double, Vector>(bestCost, bestTheta);
}
public void Logistic_Regression_Test_CostFunction_2_WithoutRegularization()
{
Matrix X = new[,] {
{ 8, 1, 6 },
{ 3, 5 ,7 },
{ 4, 9, 2 }};
Vector y = new Vector(new double[] { 1, 1, 0 });
Vector theta = new Vector(new double[] { 0, 1, 0 });
ICostFunction logisticCostFunction = new LogisticCostFunction()
{
X = X,
Y = y,
Lambda = 0,
};
double cost = logisticCostFunction.ComputeCost(theta.Copy());
theta = logisticCostFunction.ComputeGradient(theta.Copy());
Assert.Equal(3.1067d, System.Math.Round(cost, 4));
Assert.Equal(0.6093d, System.Math.Round(theta[0], 4));
Assert.Equal(2.8988d, System.Math.Round(theta[1], 4));
Assert.Equal(0.1131d, System.Math.Round(theta[2], 4));
}
public void Logistic_Regression_Test_CostFunction_1()
{
Matrix X = new[,]
{{ 1, 1, 1 },
{ 1, 1, 1 },
{ 1, 1, 1 },
{ 8, 1, 6 },
{ 3, 5 ,7 },
{ 4, 9, 2 }};
Vector y = new Vector(new double[] { 1, 0, 1, 0, 1, 0 });
Vector theta = new Vector(new double[] { 0, 1, 0 });
ICostFunction logisticCostFunction = new LogisticCostFunction()
{
X = X,
Y = y,
Lambda = 3,
Regularizer = new L2Regularizer()
};
double cost = logisticCostFunction.ComputeCost(theta.Copy());
theta = logisticCostFunction.ComputeGradient(theta.Copy());
Assert.Equal(2.2933d, System.Math.Round(cost, 4));
Assert.Equal(1.6702d, System.Math.Round(theta[0], 4));
Assert.Equal(2.1483d, System.Math.Round(theta[1], 4));
Assert.Equal(1.0887d, System.Math.Round(theta[2], 4));
}
public static Vector Calc(Vector v, Func<int, double, double> f)
{
var result = v.Copy();
for (int i = 0; i < v.Length; i++)
result[i] = f(i, result[i]);
return result;
}
public static Vector Calc(Vector v, Func <int, double, double> f)
{
var result = v.Copy();
for (int i = 0; i < v.Length; i++)
{
result[i] = f(i, result[i]);
}
return(result);
}
/// <summary>A Vector extension method that eaches.</summary>
/// <param name="v">The v to act on.</param>
/// <param name="transform">The transform.</param>
/// <param name="asCopy">(Optional) true to as copy.</param>
/// <returns>A Vector.</returns>
public static Vector Each(this Vector v, Func <double, double> transform, bool asCopy = false)
{
Vector vector = (asCopy ? v.Copy() : v);
for (int i = 0; i < vector.Length; i++)
{
vector[i] = transform(vector[i]);
}
return(vector);
}
/// <summary>A Vector extension method that eaches.</summary>
/// <param name="v">The v to act on.</param>
/// <param name="transform">The transform including value and coordinate.</param>
/// <param name="asCopy">(Optional) true to as copy.</param>
/// <returns>A Vector.</returns>
public static Vector Each(this Vector v, Func <double, int, double> transform, bool asCopy = false)
{
Vector vector = v;
if (asCopy)
{
vector = v.Copy();
}
for (int i = 0; i < vector.Length; i++)
{
vector[i] = transform(vector[i], i);
}
return(v);
}
/// <summary>
/// Adds a specified number of polynomial features to the training / test Vector.
/// </summary>
/// <param name="x">Training / Testing record</param>
/// <param name="polynomialFeatures">Number of polynomial features to add</param>
/// <returns></returns>
public static Vector IncreaseDimensions(Vector x, int polynomialFeatures)
{
Vector xtemp = x.Copy();
int maxCols = xtemp.Length;
for (int j = 0; j < maxCols - 1; j++)
{
for (int k = 0; k <= polynomialFeatures; k++)
{
for (int m = 0; m <= k; m++)
{
double v = (System.Math.Pow(xtemp[j], k - m) * System.Math.Pow(xtemp[j + 1], m));
xtemp = xtemp.Insert(xtemp.Length - 1, v);
}
}
}
return xtemp;
}
/// <summary>
/// Adds a specified number of polynomial features to the training / test Vector.
/// </summary>
/// <param name="x">Training / Testing record</param>
/// <param name="polynomialFeatures">Number of polynomial features to add</param>
/// <returns></returns>
public static Vector IncreaseDimensions(Vector x, int polynomialFeatures)
{
var xtemp = x.Copy();
var maxCols = xtemp.Length;
for (var j = 0; j < maxCols - 1; j++)
{
for (var k = 0; k <= polynomialFeatures; k++)
{
for (var m = 0; m <= k; m++)
{
var v = Math.Pow(xtemp[j], (double)(k - m)) * Math.Pow(xtemp[j + 1], (double)m);
xtemp = xtemp.Insert(xtemp.Length - 1, v);
}
}
}
return xtemp;
}
public void Logistic_Regression_Test_CostFunction_1()
{
Matrix X = new[,]
{{ 1, 1, 1 },
{ 1, 1, 1 },
{ 1, 1, 1 },
{ 8, 1, 6 },
{ 3, 5 ,7 },
{ 4, 9, 2 }};
Vector y = new Vector(new double[] { 1, 0, 1, 0, 1, 0 });
Vector theta = new Vector(new double[] { 0, 1, 0 });
numl.Math.Functions.Cost.ICostFunction logisticCostFunction = new numl.Math.Functions.Cost.LogisticCostFunction();
numl.Math.Functions.Regularization.IRegularizer regularizer = new numl.Math.Functions.Regularization.Regularization();
double cost = logisticCostFunction.ComputeCost(theta.Copy(), X, y, 3, regularizer);
theta = logisticCostFunction.ComputeGradient(theta.Copy(), X, y, 3, regularizer);
Assert.AreEqual(2.2933d, System.Math.Round(cost, 4));
Assert.AreEqual(1.6702d, System.Math.Round(theta[0], 4));
Assert.AreEqual(2.1483d, System.Math.Round(theta[1], 4));
Assert.AreEqual(1.0887d, System.Math.Round(theta[2], 4));
}
