public static Tuple <Matrix <double>, Matrix <double> > GradientDescent(CostFunctionWithThetaParameter func,
Matrix <double> theta, double alpha, int numberIterations)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
Matrix <double> JHistory = new DenseMatrix(numberIterations, 1);
for (int i = 0; i < numberIterations; i++)
{
var res = func(theta);
var h = res.Item1;
var grad = res.Item2;
JHistory[i, 0] = h;
// "bold driver" - if we decrease the cost function, increase the learning rate by 5% but
// in case when we increase the cost function, decrease the learning rate by 50%
if (i > 0)
{
if (JHistory[i, 0] < JHistory[i - 1, 0])
{
alpha += (double)0.05 * alpha;
}
else
{
alpha -= (double)0.5 * alpha;
}
}
theta = theta - grad * alpha;
if (i > 0 && JHistory[i, 0] < JHistory[i - 1, 0] &&
Equalities.DoubleEquals(JHistory[i, 0], JHistory[i - 1, 0]))
{
break;
}
}
stopWatch.Stop();
// Get the elapsed time as a TimeSpan value.
TimeSpan ts = stopWatch.Elapsed;
// Format and display the TimeSpan value.
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
Console.WriteLine("RunTime " + elapsedTime);
return(Tuple.Create(theta, JHistory));
}
public static Matrix <double> ComputeNumericalGradient(CostFunctionWithThetaParameter J, Matrix <double> theta)
{
double epsilon = 0.0001;
Matrix <double> numericalGradient = new DenseMatrix(theta.RowCount, 1);
var perturbations = new DenseMatrix(theta.RowCount, 1); // смущения ;))
for (int p = 0; p < theta.RowCount; p++)
{
perturbations[p, 0] = epsilon;
double loss1 = J(theta + perturbations).Item1;
double loss2 = J(theta - perturbations).Item1;
numericalGradient[p, 0] = ((loss1 - loss2) / (double)(2d * epsilon));
perturbations[p, 0] = 0;
}
return(numericalGradient);
}