public static Tuple <Matrix <double>, Matrix <double> > GradientDescent(CostFunctionWithThetaParameter func, Matrix <double> theta, double alpha, int numberIterations) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); Matrix <double> JHistory = new DenseMatrix(numberIterations, 1); for (int i = 0; i < numberIterations; i++) { var res = func(theta); var h = res.Item1; var grad = res.Item2; JHistory[i, 0] = h; // "bold driver" - if we decrease the cost function, increase the learning rate by 5% but // in case when we increase the cost function, decrease the learning rate by 50% if (i > 0) { if (JHistory[i, 0] < JHistory[i - 1, 0]) { alpha += (double)0.05 * alpha; } else { alpha -= (double)0.5 * alpha; } } theta = theta - grad * alpha; if (i > 0 && JHistory[i, 0] < JHistory[i - 1, 0] && Equalities.DoubleEquals(JHistory[i, 0], JHistory[i - 1, 0])) { break; } } stopWatch.Stop(); // Get the elapsed time as a TimeSpan value. TimeSpan ts = stopWatch.Elapsed; // Format and display the TimeSpan value. string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); Console.WriteLine("RunTime " + elapsedTime); return(Tuple.Create(theta, JHistory)); }
public static Matrix <double> ComputeNumericalGradient(CostFunctionWithThetaParameter J, Matrix <double> theta) { double epsilon = 0.0001; Matrix <double> numericalGradient = new DenseMatrix(theta.RowCount, 1); var perturbations = new DenseMatrix(theta.RowCount, 1); // смущения ;)) for (int p = 0; p < theta.RowCount; p++) { perturbations[p, 0] = epsilon; double loss1 = J(theta + perturbations).Item1; double loss2 = J(theta - perturbations).Item1; numericalGradient[p, 0] = ((loss1 - loss2) / (double)(2d * epsilon)); perturbations[p, 0] = 0; } return(numericalGradient); }