public static DV SecondOrder_DivisionMethod(Func <DV, D> f, DV startPoint, double accuracy, out int calcsF, out int calcsGradient, out int calcsHessian, out DV[] x, out double[] fx)
{
//Counters
calcsF = 0; //Count how many times the objective function was used.
calcsGradient = 0; //Count how many times the gradient was calculated.
calcsHessian = 0; //Count how many times the second gradient was calculated.
//Define our X vector
int maxIterations = 10000;
x = new DV[maxIterations];
fx = new double[maxIterations];
//Pick an initial guess for x
int i = 0;
x[i] = startPoint;
fx[i] = f(x[i]); calcsF++;
//Loop through gradient steps until zeros are found
double alpha = 1;
while (true)
{
//Compute next step, using previous step
i++;
//Step 1 - Determine the gradients
DV gradient = AD.Grad(f, x[i - 1]); calcsGradient++;
var hess = AD.Hessian(f, x[i - 1]); calcsHessian++;
//Step 2 - Compute full step (alpha = 1). Loop through every entry in the DV and compute the step for each one.
List <D> listSteps = new List <D>();
while (true)
{
try
{
int c = listSteps.Count;
listSteps.Add(-gradient[c] / hess[c, c]); // first-gradient divided by second-gradient
}
catch
{ break; }
}
DV fullStep = new DV(listSteps.ToArray());
//Step 3 - Division method, to compute the new x[i] and fx[i]
DV xPrev = x[i - 1];
Func <D, D> objFAlpha = delegate(D a)
{
DV xNext = xPrev + (a * fullStep);
return(f(xNext));
};
alpha = alpha * 0.8;
double beta = UnimodalMinimization.DivisionSearch(objFAlpha, fx[i - 1], alpha, out fx[i], ref calcsF);
x[i] = x[i - 1] + (beta * fullStep);
//Check if accuracy has been met
double magGradient = Math.Sqrt(AD.Pow(gradient[0], 2) + AD.Pow(gradient[1], 2));
if (magGradient < accuracy)
{
break;
}
DV dx = AD.Abs(x[i] - x[i - 1]);
if ((dx[0] < accuracy * 0.1) && (dx[1] < accuracy * 0.1))
{
break;
}
}
//Return the minimization point
x = x.Take(i + 1).ToArray();
fx = fx.Take(i + 1).ToArray();
return(x[i]);
}
public static DV FirstOrder_OneDimensionalMethod(Func <DV, D> f, DV startPoint, double accuracy, out int calcsF, out int calcsGradient, out DV[] x, out double[] fx)
{
//Counters
calcsF = 0; //Count how many times the objective function was used.
calcsGradient = 0; //Count how many times the gradient was calculated.
//Define our X vector
int maxIterations = 1000;
x = new DV[maxIterations];
fx = new double[maxIterations];
//Pick an initial guess for x
int i = 0;
x[i] = startPoint;
fx[i] = f(x[i]); calcsF++;
//Loop through gradient steps until min points are found, recompute gradient and repeat.
while (true)
{
//Compute next step, using previous step
i++;
//Return failed results
if (double.IsNaN(x[i - 1][0]) || double.IsNaN(x[i - 1][1]) || (i == maxIterations))
{
x = x.Take(i).ToArray();
fx = fx.Take(i).ToArray();
return(null);
}
//Step 1 - Determine the gradient
DV gradient = 0 - AD.Grad(f, x[i - 1]); calcsGradient++;
DV direction = gradient / Math.Sqrt(AD.Pow(gradient[0], 2) + AD.Pow(gradient[1], 2)); //Normalize Gradient
//Step 2 - Build an objective function using the gradient.
// This objective function moves downward in the direction of the gradient.
// It uses golden ratio optimization to find the minimum point in this direction
DV xPrev = x[i - 1];
Func <D, D> objFStep = delegate(D alpha)
{
DV xNew = xPrev + (alpha * direction);
return(f(xNew));
};
var stepSearchResults = UnimodalMinimization.goldenRatioSearch(objFStep, 0, 1, accuracy); //alpha can only be between 0 and 1
double step = (stepSearchResults.a + stepSearchResults.b) / 2; //The step required to get to the bottom
calcsF += stepSearchResults.CalculationsUntilAnswer; //The number of calculations of f that were required.
//Step 3 - Move to the discovered minimum point
x[i] = x[i - 1] + (step * direction);
fx[i] = f(x[i]); calcsF++;
//Step 4 - Check if accuracy has been met. If so, then end.
double magGradient = Math.Sqrt(AD.Pow(gradient[0], 2) + AD.Pow(gradient[1], 2));
if (magGradient < accuracy)
{
break;
}
DV dx = AD.Abs(x[i] - x[i - 1]);
if (((dx[0] < accuracy) && (dx[1] < accuracy)))
{
break;
}
}
//Return the minimization point.
x = x.Take(i + 1).ToArray();
fx = fx.Take(i + 1).ToArray();
return(x[i]);
}
static void Main(string[] args)
{
#region Intro
Console.WriteLine(@"
///////////////////////////////////////
Task: Lab 2 - Gradient Descent
Written By: Christopher W. Blake
Date: 22 Nov. 2016
///////////////////////////////////////
Description:
Creates 2 different seach algorithms for finding
the minimum of a given function f(x) in a range. It then tests
these results and prints them to the console.
1. First-order gradient descent method.
(one – dimensional minimization method for choosing the step)
2. Second-order gradient descent method
(division method for choosing the step)
Verification: f(0.5, -0.44963) = 0.27696
///////////////////////////////////////
");
#endregion
//Required accuracy values
List <double> epsValues = new List <double> {
0.1, 0.01, 0.001
}; //accuracy
//Objective function
Func <DV, D> f = delegate(DV x)
{
D x1 = x[0];
D x2 = x[1];
return(x1 * x1 + x2 * x2 + AD.Exp(x2 * x2) - x1 + 2 * x2);
//return AD.Pow(x1-7, 2) + AD.Pow(x2-3, 2);
};
#region 1.) First Order, One-Dimensional Method
//Show the table header
Console.WriteLine("----- Gradient Search, First Order, One-Dimensional Method -----");
Console.WriteLine(" eps X1 X2 f(x) Calcs F Calcs Gr");
foreach (double eps in epsValues)
{
//Get solution
int calcsF;
int calcsGradient;
DV startPoint = new DV(new D[] { 0, 0 });
DV xMin = Optimization.GradientDescent.FirstOrder_OneDimensionalMethod(f, startPoint, eps, out calcsF, out calcsGradient);
//determine number of decimal places to show
int dp = BitConverter.GetBytes(decimal.GetBits((decimal)eps)[3])[2] + 1;
//Show on console
Console.WriteLine("{0,8}{1,8:F" + dp + "}{2,8:F" + dp + "}{3,8:F" + dp + "}{4,8}{5,8}", eps, (double)xMin[0], (double)xMin[1], (double)f(xMin), calcsF, calcsGradient);
}
#endregion
DV[] xFirstOrder_DivMethod = null;
double[] fxFirstOrder_DivMethod = null;
#region 2.) First Order, Division Method
//Show the table header
Console.WriteLine();
Console.WriteLine("----- Gradient Search, First Order, Division Method -----");
Console.WriteLine(" eps X1 X2 f(x) Calcs F Calcs dF");
foreach (double eps in epsValues)
{
//Get solution
int calcsF;
int calcsGradient;
DV startPoint = new DV(new D[] { 0, 0 });
DV xMin = Optimization.GradientDescent.FirstOrder_DivisionMethod(f, startPoint, eps, out calcsF, out calcsGradient, out xFirstOrder_DivMethod, out fxFirstOrder_DivMethod);
//determine number of decimal places to show
int dp = BitConverter.GetBytes(decimal.GetBits((decimal)eps)[3])[2] + 1;
//Show on console
Console.WriteLine("{0,8}{1,8:F" + dp + "}{2,8:F" + dp + "}{3,8:F" + dp + "}{4,8}{5,8}", eps, (double)xMin[0], (double)xMin[1], (double)f(xMin), calcsF, calcsGradient);
}
#endregion
DV[] xSecondOrder_FullStep = null;
double[] fxSecondOrder_FullStep = null;
#region 3.) Second Order - Newtons Method, FullStep
//Show the table header
Console.WriteLine();
Console.WriteLine("----- Gradient Search, Second Order, FullStep -----");
Console.WriteLine(" eps X1 X2 f(x) Calcs F Calcs Gr Calcs Hess");
//Show Results for each accuracy
foreach (double eps in epsValues)
{
//Get solution
int calcsF;
int calcsGradient;
int calcsHessian;
DV startPoint = new DV(new D[] { 0, 0 });
DV xMin = Optimization.GradientDescent.SecondOrder_FullStep(f, startPoint, eps, out calcsF, out calcsGradient, out calcsHessian, out xSecondOrder_FullStep, out fxSecondOrder_FullStep);
//determine number of decimal places to show
int dp = BitConverter.GetBytes(decimal.GetBits((decimal)eps)[3])[2] + 1;
//Show on console
Console.WriteLine("{0,8}{1,8:F" + dp + "}{2,8:F" + dp + "}{3,8:F" + dp + "}{4,8}{5,8}{6,8}", eps, (double)xMin[0], (double)xMin[1], (double)f(xMin), calcsF, calcsGradient, calcsHessian);
}
#endregion
#region 4.) Second Order - Newtons Method, Division Method
//Show the table header
Console.WriteLine();
Console.WriteLine("----- Gradient Search, Second Order, Division Method -----");
Console.WriteLine(" eps X1 X2 f(x) Calcs F Calcs Gr Calcs Hess");
//Show Results for each accuracy
foreach (double eps in epsValues)
{
//Get solution
int calcsF;
int calcsGradient;
int calcsHessian;
DV startPoint = new DV(new D[] { 0, 0 });
DV xMin = Optimization.GradientDescent.SecondOrder_DivisionMethod(f, startPoint, eps, out calcsF, out calcsGradient, out calcsHessian);
//determine number of decimal places to show
int dp = BitConverter.GetBytes(decimal.GetBits((decimal)eps)[3])[2] + 1;
//Show on console
Console.WriteLine("{0,8}{1,8:F" + dp + "}{2,8:F" + dp + "}{3,8:F" + dp + "}{4,8}{5,8}{6,8}", eps, (double)xMin[0], (double)xMin[1], (double)f(xMin), calcsF, calcsGradient, calcsHessian);
}
#endregion
//Below is all extra work that she asked for (for fun) in class
#region 5.) Speed Comparison, First Order, Div method
DV xStarFirstOrder_DivMethod = xFirstOrder_DivMethod[xFirstOrder_DivMethod.Length - 1];
Console.WriteLine();
Console.WriteLine("---- Speed (q), First Order, Division Method --- ");
Console.WriteLine("K X1 X2 qX1 qX2");
for (int k = 0; k < xFirstOrder_DivMethod.Length - 1; k++)
{
//Display row information
Console.Write("{0}: {1,10:F3}{2,10:F3}", k, (double)xFirstOrder_DivMethod[k][0], (double)xFirstOrder_DivMethod[k][1]);
if (k > 0 && k < xFirstOrder_DivMethod.Length - 2)
{
DV x = xFirstOrder_DivMethod[k] - xStarFirstOrder_DivMethod;
DV xNext = xFirstOrder_DivMethod[k - 1] - xStarFirstOrder_DivMethod;
double qX1 = AD.Abs(xNext[0] - xStarFirstOrder_DivMethod[0]);
double qX2 = AD.Abs(xNext[1] - xStarFirstOrder_DivMethod[1]);
Console.Write("{0,10:F3}{1,10:F3}", qX1, qX2);
}
Console.WriteLine();
}
#endregion
#region 6.) Speed Comparison, Second Order, Full Step
DV xStarSecondOrder_FullStep = xSecondOrder_FullStep[xSecondOrder_FullStep.Length - 1];
Console.WriteLine();
Console.WriteLine("---- Speed (q), Second Order, Full Step --- ");
Console.WriteLine("K X1 X2 qX1 qX2");
for (int k = 0; k < xSecondOrder_FullStep.Length - 1; k++)
{
//Display row information
Console.Write("{0}: {1,10:F3}{2,10:F3}", k, (double)xSecondOrder_FullStep[k][0], (double)xSecondOrder_FullStep[k][1]);
if (k > 0 && k < xSecondOrder_FullStep.Length - 2)
{
DV x = xSecondOrder_FullStep[k] - xStarSecondOrder_FullStep;
DV xNext = xSecondOrder_FullStep[k - 1] - xStarSecondOrder_FullStep;
double qX1 = AD.Abs(xNext[0] - xStarSecondOrder_FullStep[0]);
double qX2 = AD.Abs(xNext[1] - xStarSecondOrder_FullStep[1]);
Console.Write("{0,10:F3}{1,10:F3}", qX1, qX2);
}
Console.WriteLine();
}
#endregion
//Wait for use to click something to exit
Console.ReadKey();
}