Exemplo n.º 1
0
        public static DV SecondOrder_DivisionMethod(Func <DV, D> f, DV startPoint, double accuracy, out int calcsF, out int calcsGradient, out int calcsHessian, out DV[] x, out double[] fx)
        {
            //Counters
            calcsF        = 0; //Count how many times the objective function was used.
            calcsGradient = 0; //Count how many times the gradient was calculated.
            calcsHessian  = 0; //Count how many times the second gradient was calculated.

            //Define our X vector
            int maxIterations = 10000;

            x  = new DV[maxIterations];
            fx = new double[maxIterations];

            //Pick an initial guess for x
            int i = 0;

            x[i]  = startPoint;
            fx[i] = f(x[i]); calcsF++;

            //Loop through gradient steps until zeros are found
            double alpha = 1;

            while (true)
            {
                //Compute next step, using previous step
                i++;

                //Step 1 - Determine the gradients
                DV  gradient = AD.Grad(f, x[i - 1]); calcsGradient++;
                var hess     = AD.Hessian(f, x[i - 1]); calcsHessian++;

                //Step 2 - Compute full step (alpha = 1). Loop through every entry in the DV and compute the step for each one.
                List <D> listSteps = new List <D>();
                while (true)
                {
                    try
                    {
                        int c = listSteps.Count;
                        listSteps.Add(-gradient[c] / hess[c, c]); // first-gradient divided by second-gradient
                    }
                    catch
                    { break; }
                }
                DV fullStep = new DV(listSteps.ToArray());

                //Step 3 - Division method, to compute the new x[i] and fx[i]
                DV          xPrev     = x[i - 1];
                Func <D, D> objFAlpha = delegate(D a)
                {
                    DV xNext = xPrev + (a * fullStep);
                    return(f(xNext));
                };
                alpha = alpha * 0.8;
                double beta = UnimodalMinimization.DivisionSearch(objFAlpha, fx[i - 1], alpha, out fx[i], ref calcsF);
                x[i] = x[i - 1] + (beta * fullStep);

                //Check if accuracy has been met
                double magGradient = Math.Sqrt(AD.Pow(gradient[0], 2) + AD.Pow(gradient[1], 2));
                if (magGradient < accuracy)
                {
                    break;
                }
                DV dx = AD.Abs(x[i] - x[i - 1]);
                if ((dx[0] < accuracy * 0.1) && (dx[1] < accuracy * 0.1))
                {
                    break;
                }
            }

            //Return the minimization point
            x  = x.Take(i + 1).ToArray();
            fx = fx.Take(i + 1).ToArray();
            return(x[i]);
        }
Exemplo n.º 2
0
        public static DV FirstOrder_OneDimensionalMethod(Func <DV, D> f, DV startPoint, double accuracy, out int calcsF, out int calcsGradient, out DV[] x, out double[] fx)
        {
            //Counters
            calcsF        = 0; //Count how many times the objective function was used.
            calcsGradient = 0; //Count how many times the gradient was calculated.

            //Define our X vector
            int maxIterations = 1000;

            x  = new DV[maxIterations];
            fx = new double[maxIterations];

            //Pick an initial guess for x
            int i = 0;

            x[i]  = startPoint;
            fx[i] = f(x[i]); calcsF++;

            //Loop through gradient steps until min points are found, recompute gradient and repeat.
            while (true)
            {
                //Compute next step, using previous step
                i++;

                //Return failed results
                if (double.IsNaN(x[i - 1][0]) || double.IsNaN(x[i - 1][1]) || (i == maxIterations))
                {
                    x  = x.Take(i).ToArray();
                    fx = fx.Take(i).ToArray();
                    return(null);
                }

                //Step 1 - Determine the gradient
                DV gradient  = 0 - AD.Grad(f, x[i - 1]); calcsGradient++;
                DV direction = gradient / Math.Sqrt(AD.Pow(gradient[0], 2) + AD.Pow(gradient[1], 2)); //Normalize Gradient

                //Step 2 - Build an objective function using the gradient.
                // This objective function moves downward in the direction of the gradient.
                // It uses golden ratio optimization to find the minimum point in this direction
                DV          xPrev    = x[i - 1];
                Func <D, D> objFStep = delegate(D alpha)
                {
                    DV xNew = xPrev + (alpha * direction);
                    return(f(xNew));
                };
                var    stepSearchResults = UnimodalMinimization.goldenRatioSearch(objFStep, 0, 1, accuracy); //alpha can only be between 0 and 1
                double step = (stepSearchResults.a + stepSearchResults.b) / 2;                               //The step required to get to the bottom
                calcsF += stepSearchResults.CalculationsUntilAnswer;                                         //The number of calculations of f that were required.

                //Step 3 - Move to the discovered minimum point
                x[i]  = x[i - 1] + (step * direction);
                fx[i] = f(x[i]); calcsF++;

                //Step 4 - Check if accuracy has been met. If so, then end.
                double magGradient = Math.Sqrt(AD.Pow(gradient[0], 2) + AD.Pow(gradient[1], 2));
                if (magGradient < accuracy)
                {
                    break;
                }
                DV dx = AD.Abs(x[i] - x[i - 1]);
                if (((dx[0] < accuracy) && (dx[1] < accuracy)))
                {
                    break;
                }
            }

            //Return the minimization point.
            x  = x.Take(i + 1).ToArray();
            fx = fx.Take(i + 1).ToArray();
            return(x[i]);
        }
Exemplo n.º 3
0
        static void Main(string[] args)
        {
            #region Intro
            Console.WriteLine(@"
///////////////////////////////////////
Task: Lab 2 - Gradient Descent
Written By: Christopher W. Blake
Date: 22 Nov. 2016
///////////////////////////////////////
Description:
Creates 2 different seach algorithms for finding
the minimum of a given function f(x) in a range. It then tests
these results and prints them to the console.

1. First-order gradient descent method.
   (one – dimensional minimization method for choosing the step)

2. Second-order gradient descent method
   (division method for choosing the step)

Verification: f(0.5, -0.44963) = 0.27696
///////////////////////////////////////
");
            #endregion

            //Required accuracy values
            List <double> epsValues = new List <double> {
                0.1, 0.01, 0.001
            };                                                              //accuracy

            //Objective function
            Func <DV, D> f = delegate(DV x)
            {
                D x1 = x[0];
                D x2 = x[1];
                return(x1 * x1 + x2 * x2 + AD.Exp(x2 * x2) - x1 + 2 * x2);
                //return AD.Pow(x1-7, 2) + AD.Pow(x2-3, 2);
            };

            #region 1.) First Order, One-Dimensional Method
            //Show the table header
            Console.WriteLine("----- Gradient Search, First Order, One-Dimensional Method -----");
            Console.WriteLine("     eps      X1      X2    f(x)   Calcs F   Calcs Gr");
            foreach (double eps in epsValues)
            {
                //Get solution
                int calcsF;
                int calcsGradient;
                DV  startPoint = new DV(new D[] { 0, 0 });
                DV  xMin       = Optimization.GradientDescent.FirstOrder_OneDimensionalMethod(f, startPoint, eps, out calcsF, out calcsGradient);

                //determine number of decimal places to show
                int dp = BitConverter.GetBytes(decimal.GetBits((decimal)eps)[3])[2] + 1;

                //Show on console
                Console.WriteLine("{0,8}{1,8:F" + dp + "}{2,8:F" + dp + "}{3,8:F" + dp + "}{4,8}{5,8}", eps, (double)xMin[0], (double)xMin[1], (double)f(xMin), calcsF, calcsGradient);
            }
            #endregion

            DV[]     xFirstOrder_DivMethod  = null;
            double[] fxFirstOrder_DivMethod = null;
            #region 2.) First Order, Division Method
            //Show the table header
            Console.WriteLine();
            Console.WriteLine("----- Gradient Search, First Order, Division Method -----");
            Console.WriteLine("     eps      X1      X2    f(x)   Calcs F   Calcs dF");
            foreach (double eps in epsValues)
            {
                //Get solution
                int calcsF;
                int calcsGradient;
                DV  startPoint = new DV(new D[] { 0, 0 });
                DV  xMin       = Optimization.GradientDescent.FirstOrder_DivisionMethod(f, startPoint, eps, out calcsF, out calcsGradient, out xFirstOrder_DivMethod, out fxFirstOrder_DivMethod);

                //determine number of decimal places to show
                int dp = BitConverter.GetBytes(decimal.GetBits((decimal)eps)[3])[2] + 1;

                //Show on console
                Console.WriteLine("{0,8}{1,8:F" + dp + "}{2,8:F" + dp + "}{3,8:F" + dp + "}{4,8}{5,8}", eps, (double)xMin[0], (double)xMin[1], (double)f(xMin), calcsF, calcsGradient);
            }
            #endregion

            DV[]     xSecondOrder_FullStep  = null;
            double[] fxSecondOrder_FullStep = null;
            #region 3.) Second Order - Newtons Method, FullStep
            //Show the table header
            Console.WriteLine();
            Console.WriteLine("----- Gradient Search, Second Order, FullStep -----");
            Console.WriteLine("     eps      X1      X2    f(x)   Calcs F  Calcs Gr  Calcs Hess");

            //Show Results for each accuracy
            foreach (double eps in epsValues)
            {
                //Get solution
                int calcsF;
                int calcsGradient;
                int calcsHessian;
                DV  startPoint = new DV(new D[] { 0, 0 });
                DV  xMin       = Optimization.GradientDescent.SecondOrder_FullStep(f, startPoint, eps, out calcsF, out calcsGradient, out calcsHessian, out xSecondOrder_FullStep, out fxSecondOrder_FullStep);

                //determine number of decimal places to show
                int dp = BitConverter.GetBytes(decimal.GetBits((decimal)eps)[3])[2] + 1;

                //Show on console
                Console.WriteLine("{0,8}{1,8:F" + dp + "}{2,8:F" + dp + "}{3,8:F" + dp + "}{4,8}{5,8}{6,8}", eps, (double)xMin[0], (double)xMin[1], (double)f(xMin), calcsF, calcsGradient, calcsHessian);
            }
            #endregion

            #region 4.) Second Order - Newtons Method, Division Method
            //Show the table header
            Console.WriteLine();
            Console.WriteLine("----- Gradient Search, Second Order, Division Method -----");
            Console.WriteLine("     eps      X1      X2    f(x)   Calcs F  Calcs Gr  Calcs Hess");

            //Show Results for each accuracy
            foreach (double eps in epsValues)
            {
                //Get solution
                int calcsF;
                int calcsGradient;
                int calcsHessian;
                DV  startPoint = new DV(new D[] { 0, 0 });
                DV  xMin       = Optimization.GradientDescent.SecondOrder_DivisionMethod(f, startPoint, eps, out calcsF, out calcsGradient, out calcsHessian);

                //determine number of decimal places to show
                int dp = BitConverter.GetBytes(decimal.GetBits((decimal)eps)[3])[2] + 1;

                //Show on console
                Console.WriteLine("{0,8}{1,8:F" + dp + "}{2,8:F" + dp + "}{3,8:F" + dp + "}{4,8}{5,8}{6,8}", eps, (double)xMin[0], (double)xMin[1], (double)f(xMin), calcsF, calcsGradient, calcsHessian);
            }
            #endregion


            //Below is all extra work that she asked for (for fun) in class

            #region 5.) Speed Comparison, First Order, Div method
            DV xStarFirstOrder_DivMethod = xFirstOrder_DivMethod[xFirstOrder_DivMethod.Length - 1];

            Console.WriteLine();
            Console.WriteLine("---- Speed (q), First Order, Division Method --- ");
            Console.WriteLine("K         X1        X2      qX1      qX2");
            for (int k = 0; k < xFirstOrder_DivMethod.Length - 1; k++)
            {
                //Display row information
                Console.Write("{0}: {1,10:F3}{2,10:F3}", k, (double)xFirstOrder_DivMethod[k][0], (double)xFirstOrder_DivMethod[k][1]);

                if (k > 0 && k < xFirstOrder_DivMethod.Length - 2)
                {
                    DV x     = xFirstOrder_DivMethod[k] - xStarFirstOrder_DivMethod;
                    DV xNext = xFirstOrder_DivMethod[k - 1] - xStarFirstOrder_DivMethod;

                    double qX1 = AD.Abs(xNext[0] - xStarFirstOrder_DivMethod[0]);
                    double qX2 = AD.Abs(xNext[1] - xStarFirstOrder_DivMethod[1]);

                    Console.Write("{0,10:F3}{1,10:F3}", qX1, qX2);
                }

                Console.WriteLine();
            }
            #endregion

            #region 6.) Speed Comparison, Second Order, Full Step
            DV xStarSecondOrder_FullStep = xSecondOrder_FullStep[xSecondOrder_FullStep.Length - 1];

            Console.WriteLine();
            Console.WriteLine("---- Speed (q), Second Order, Full Step --- ");
            Console.WriteLine("K         X1        X2      qX1      qX2");
            for (int k = 0; k < xSecondOrder_FullStep.Length - 1; k++)
            {
                //Display row information
                Console.Write("{0}: {1,10:F3}{2,10:F3}", k, (double)xSecondOrder_FullStep[k][0], (double)xSecondOrder_FullStep[k][1]);

                if (k > 0 && k < xSecondOrder_FullStep.Length - 2)
                {
                    DV x     = xSecondOrder_FullStep[k] - xStarSecondOrder_FullStep;
                    DV xNext = xSecondOrder_FullStep[k - 1] - xStarSecondOrder_FullStep;

                    double qX1 = AD.Abs(xNext[0] - xStarSecondOrder_FullStep[0]);
                    double qX2 = AD.Abs(xNext[1] - xStarSecondOrder_FullStep[1]);

                    Console.Write("{0,10:F3}{1,10:F3}", qX1, qX2);
                }

                Console.WriteLine();
            }

            #endregion



            //Wait for use to click something to exit
            Console.ReadKey();
        }