public void FitBFGS2() { double eps = 1e-5; double rEps = eps*10; int itMax = 50; double step=1; double[] vector = {6,45,22}; double[] expVector = {0,0,0}; Parabola f = new Parabola(); int dim = f.Dimension; BFGS alg = new BFGS(dim,vector,f,step,eps,itMax); alg.FindMinimum(); Assert.IsTrue(Diff(expVector,alg.Minimum,dim)<rEps); }
public void FitBFGS3() { double eps = 1e-5; double rEps = eps; int itMax = 200; double step=1; double[] vector = {6,-5}; double[] expVector = {1,1}; Banana f = new Banana(); int dim = f.Dimension; BFGS alg = new BFGS(dim,vector,f,step,eps,itMax); alg.FindMinimum(); Assert.IsTrue(Diff(expVector,alg.Minimum,dim)<rEps); }
public void FitBFGS1() { double eps = 1e-5; double rEps = eps*10; double step = 2; double[] vector = {0,0,0}; double[] expVector = {1,-2,3}; int itMax = 50; TestFunction f = new TestFunction(); int dim = f.Dimension; BFGS alg = new BFGS(dim, vector, f, step, eps, itMax); alg.FindMinimum(); Assert.IsTrue(Diff(expVector,alg.Minimum,dim)<rEps); }
public static void Run(CostFunction f, int max_iterations = 200) { BFGS s = new BFGS(); double[] x_0 = f.CreateRandomSolution(); s.SolutionUpdated += (best_solution, step) => { Console.WriteLine("Step {0}: Fitness = {1}", step, best_solution.Cost); }; s.Minimize(x_0, f, max_iterations); }
public void testFunctionValueEqualsCostFunctionAtCurrentValue() { var testCostFunction = new TestCostFunction(); var problem = new Problem(testCostFunction, new NoConstraint(), new Vector(new List <double> { 3, 7.4 })); var endCriteria = new EndCriteria(maxIterations: 1000, maxStationaryStateIterations: 10, rootEpsilon: 0, functionEpsilon: 1e-10, gradientNormEpsilon: null); var method = new BFGS(); var endType = method.minimize(problem, endCriteria); QAssert.AreEqual(EndCriteria.Type.StationaryFunctionValue, endType); QAssert.AreEqual(problem.functionValue(), testCostFunction.value(problem.currentValue())); }
internal static global::System.Runtime.InteropServices.HandleRef getCPtr(BFGS obj) { return (obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr; }
static void Main(string[] args) { double oneOver2Pi = 1.0 / (1.0 * Math.Sqrt(2 * Math.PI)); Console.WriteLine(NormalCDFInverse(0.99)); Console.WriteLine(inv_cdf(0.99)); Console.WriteLine(InverseCDF.QNorm(0.99, 0, 1, true, false)); Console.WriteLine(NormalCDFInverse(0.5)); Console.WriteLine(inv_cdf(0.5)); Console.WriteLine(InverseCDF.QNorm(0.5, 0, 1, true, false)); Console.WriteLine(NormalCDFInverse(0.31)); Console.WriteLine(inv_cdf(0.31)); Console.WriteLine(InverseCDF.QNorm(0.31, 0, 1, true, false)); //x4−8x2 + 5 Func <double[], double> testFunc1 = (x) => { return(Math.Pow(x[0], 4) - 8 * Math.Pow(x[0], 2) + 5); }; Func <double[], double>[] dtestFunc1 = new Func <double[], double> [1]; dtestFunc1[0] = (x) => { return(4 * Math.Pow(x[0], 3) - 16 * x[0]); }; Func <double[], double> testConstr = (x) => { return(5 - Math.Exp(x[0]) + 2.0 * Math.Pow(x[0] - 1, 2)); }; Func <double[], double> testv = (x) => { double a = x[0]; return(0.0); }; //Func<Variables, double> testFunc = (x) => { // return Math.Pow(x.Vars[0], 4) - 3 * Math.Pow(x.Vars[0], 3) + 2; //}; //Func<Variables, double> dTestfunc = (x) => //{ // return 4 * Math.Pow(x.Vars[0], 3) - 9 * Math.Pow(x.Vars[0], 2); //}; //x4−3x3 + 2 //TestFunc(); Func <double[], double> bananaFunc = (x) => { return(Math.Pow(1 - x[0], 2) + 100 * Math.Pow(x[1] - x[0] * x[0], 2)); }; Func <double[], double> powell = (x) => { return(Math.Pow(x[0] + 10 * x[1], 2) + 5 * Math.Pow(x[2] - x[3], 2) + Math.Pow(x[1] + 2 * x[2], 4) + 10 * Math.Pow(x[0] - x[3], 4)); }; OptVector[] ttt = new OptVector[3]; ttt[0] = new OptVector(new double[3] { 1, 2, 3 }); ttt[1] = new OptVector(new double[3] { 4, 5, 6 }); ttt[2] = new OptVector(new double[3] { 7, 8, 9 }); var tr = OptVector.Transpose(ttt); //TestsSQP.Test0(); TestsSQP.Test1(); TestsSQP.Test2(); TestsSQP.Test3(); ////TestsSQP.Test4(); ////TestsSQP.Test5(); TestsSQP.Test6(); TestsSQP.Test7(); TestsSQP.Test8(); TestsSQP.Test9(); TestsSQP.Test10(); TestsSQP.Test11(); TestsSQP.Test12(); TestsSQP.Test13(); TestsSQP.Test14(); TestsSQP.Test15(); TestsSQP.Test16(); TestsSQP.Test17(); TestsSQP.Test18(); TestsSQP.Test19(); TestsSQP.Test20(); TestsSQP.Test21(); TestsSQP.Test22(); TestsSQP.Test23(); TestsSQP.Test24(); TestsSQP.Test25(); TestsSQP.Test26(); TestsSQP.Test27(); TestsSQP.Test28(); //TestCGMethod(); BFGS bfsg = new BFGS(); var result0 = bfsg.Solve(powell, new double[] { 3.0, -1.0, 0.0, 1.0 }, 10000); Console.WriteLine("Min " + powell(result0)); NLCG gradient = new NLCG(); var result = gradient.Solve(powell, new double[] { 3.0, -1.0, 0.0, 1.0 }, 4000); Console.WriteLine("Min " + powell(result)); SteepestDescent gradientSteep = new SteepestDescent(); var result1 = gradientSteep.Solve(powell, new double[] { 3.0, -1.0, 0.0, 1.0 }, 10000); Console.WriteLine("Min " + powell(result1)); //Variables result = SteepestDescent(testFunc, dTestFunc, 2, 20); for (int i = 0; i < result.Length; i++) { Console.WriteLine("result " + result[i]); } //Console.WriteLine("ver " + testFunc(result)); Console.ReadLine(); }
internal static global::System.Runtime.InteropServices.HandleRef getCPtr(BFGS obj) { return((obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr); }
/// <summary> /// VMP message to 'd' /// </summary> /// <param name="exp">Incoming message from 'exp'. Must be a proper distribution. If uniform, the result will be uniform.</param> /// <param name="d">Incoming message from 'd'. Must be a proper distribution. If uniform, the result will be uniform.</param> /// <param name="to_d">Previous outgoing message to 'd'.</param> /// <returns>The outgoing VMP message to the 'd' argument</returns> /// <remarks><para> /// The outgoing message is the factor viewed as a function of 'd' with 'exp' integrated out. /// The formula is <c>sum_exp p(exp) factor(exp,d)</c>. /// </para></remarks> /// <exception cref="ImproperMessageException"><paramref name="exp"/> is not a proper distribution</exception> /// <exception cref="ImproperMessageException"><paramref name="d"/> is not a proper distribution</exception> public static Gaussian DAverageLogarithm([Proper] Gamma exp, [Proper, Stochastic] Gaussian d, Gaussian to_d) { if (exp.IsPointMass) { return(ExpOp.DAverageLogarithm(exp.Point)); } double m, v; d.GetMeanAndVariance(out m, out v); Gaussian msg = new Gaussian(); double mu, s2; var prior = d / to_d; prior.GetMeanAndVariance(out mu, out s2); var z = Vector.Zero(2); z[0] = m; z[1] = Math.Log(v); double startingValue = GradientAndValueAtPoint(mu, s2, z, exp.Shape, exp.Rate, null); var s = new BFGS(); int evalCounter = 0; s.MaximumStep = 1e3; s.MaximumIterations = 100; s.Epsilon = 1e-5; s.convergenceCriteria = BFGS.ConvergenceCriteria.Objective; z = s.Run(z, 1.0, delegate(Vector y, ref Vector grad) { evalCounter++; return(GradientAndValueAtPoint(mu, s2, y, exp.Shape, exp.Rate, grad)); }); m = z[0]; v = Math.Exp(z[1]); to_d.SetMeanAndVariance(m, v); to_d.SetToRatio(to_d, prior); double endValue = GradientAndValueAtPoint(mu, s2, z, exp.Shape, exp.Rate, null); //Console.WriteLine("Went from {0} to {1} in {2} steps, {3} evals", startingValue, endValue, s.IterationsPerformed, evalCounter); if (startingValue < endValue) { Console.WriteLine("Warning: BFGS resulted in an increased objective function"); } return(to_d); /* ---------------- NEWTON ITERATION VERSION 1 ------------------- * double meanTimesPrec, prec; * d.GetNatural(out meanTimesPrec, out prec); * Matrix K = new Matrix(2, 2); * K[0, 0]=1/prec; // d2K by dmu^2 * K[1, 0]=K[0, 1]=-meanTimesPrec/(prec*prec); * K[1, 1]=meanTimesPrec*meanTimesPrec/Math.Pow(prec, 3)+1/(2*prec*prec); * double[,,] Kprime = new double[2, 2, 2]; * Kprime[0, 0, 0]=0; * Kprime[0, 0, 1]=Kprime[0, 1, 0]=Kprime[1, 0, 0]=-1/(prec*prec); * Kprime[0, 1, 1]=Kprime[1, 1, 0]=Kprime[1, 0, 1]=2*meanTimesPrec/Math.Pow(prec, 3); * Kprime[1, 1, 1]=-3*meanTimesPrec*meanTimesPrec/Math.Pow(prec, 4)-1/Math.Pow(prec, 3); * Vector gradS = new Vector(2); * gradS[0]=(exp.Shape-1)/prec-exp.Rate/prec*Math.Exp((meanTimesPrec+.5)/prec); * gradS[1]=-(exp.Shape-1)*meanTimesPrec/(prec*prec)+exp.Rate*(meanTimesPrec+.5)/(prec*prec)*Math.Exp((meanTimesPrec+.5)/prec); * Matrix grad2S = new Matrix(2, 2); * grad2S[0, 0]=-exp.Rate/(prec*prec)*Math.Exp((meanTimesPrec+.5)/prec); * grad2S[0, 1]=grad2S[1, 0]=-(exp.Shape-1)/(prec*prec)+exp.Rate*(1/(prec*prec)+(meanTimesPrec+.5)/Math.Pow(prec, 3))*Math.Exp((meanTimesPrec+.5)/prec); * grad2S[1, 1]=2*(exp.Shape-1)*meanTimesPrec/Math.Pow(prec, 3)-exp.Rate*(meanTimesPrec+.5)/(prec*prec)*(2/prec+(meanTimesPrec+.5)/(prec*prec))*Math.Exp((meanTimesPrec+.5)/prec); * Vector phi = new Vector(new double[] { result.MeanTimesPrecision, result.Precision }); * Vector gradKL = K*phi-gradS; * Matrix hessianKL = K - grad2S; * for (int i=0; i<2; i++) * for (int j=0; j<2; j++) * for (int k=0; k<2; k++) * hessianKL[i, j]+=Kprime[i, j, k]*phi[k]; * double step = 1; * Vector direction = GammaFromShapeAndRate.twoByTwoInverse(hessianKL)*gradKL; * Vector newPhi = phi - step * direction; * result.SetNatural(newPhi[0], newPhi[1]); * return result; * * ---------------- NEWTON ITERATION VERSION 2 ------------------- * double mean, variance; * d.GetMeanAndVariance(out mean, out variance); * Gaussian prior = d / result; * double mean1, variance1; * prior.GetMeanAndVariance(out mean1, out variance1); * Vector gradKL = new Vector(2); * gradKL[0]=-(exp.Shape-1)+exp.Rate*Math.Exp(mean+variance/2)+mean/variance1-mean1/variance1; * gradKL[1]=-1/(2*variance)+exp.Rate*Math.Exp(mean+variance/2)+1/(2*variance1); * Matrix hessianKL = new Matrix(2, 2); * hessianKL[0, 0]=exp.Rate*Math.Exp(mean+variance/2)+1/variance1; * hessianKL[0, 1]=hessianKL[1, 0]=.5*exp.Rate*Math.Exp(mean+variance/2); * hessianKL[1, 1]=1/(2*variance*variance)+exp.Rate*Math.Exp(mean+variance/2)/4; * result.GetMeanAndVariance(out mean, out variance); * if (double.IsInfinity(variance)) * variance=1000; * Vector theta = new Vector(new double[] { mean, variance }); * theta -= GammaFromShapeAndRate.twoByTwoInverse(hessianKL)*gradKL; * result.SetMeanAndVariance(theta[0], theta[1]); * return result; * ----------------------------------------------------------------- */ }
public static Gamma AlphaAverageLogarithm([Proper] Dirichlet prob, [SkipIfUniform] Gamma alpha, [Fresh] Vector probMeanLog, Gamma to_Alpha) { if (alpha.IsPointMass) return Gamma.Uniform(); var s = new BFGS(); int K = probMeanLog.Count; var prior = alpha / to_Alpha; int evalCounter = 0; s.MaximumStep = 20; s.MaximumIterations = 100; s.Epsilon = 1e-5; s.convergenceCriteria = BFGS.ConvergenceCriteria.Objective; double SumElogP = probMeanLog.Sum(); var z = Vector.FromArray(new double[] { Math.Log(alpha.Shape), Math.Log(alpha.Rate) }); double startingValue = GradientAndValueAtPoint(prior.Shape, prior.Rate, z, SumElogP, null, (double)K); FunctionEval f = delegate(Vector y, ref Vector grad) { evalCounter++; return GradientAndValueAtPoint(prior.Shape, prior.Rate, y, SumElogP, grad, (double)K); }; //DerivativeChecker.CheckDerivatives(f, z); z = s.Run(z, 1.0, f); var result = Gamma.FromShapeAndRate(Math.Exp(z[0]), Math.Exp(z[1])); result.SetToRatio(result, prior); double endValue = GradientAndValueAtPoint(prior.Shape, prior.Rate, z, SumElogP, null, (double)K); //Console.WriteLine("Went from {0} to {1} in {2} steps, {3} evals", startingValue, endValue, s.IterationsPerformed, evalCounter); if (startingValue < endValue) Console.WriteLine("Warning: BFGS resulted in an increased objective function"); return result; }
/// <summary> /// VMP message to 'd' /// </summary> /// <param name="exp">Incoming message from 'exp'. Must be a proper distribution. If uniform, the result will be uniform.</param> /// <param name="d">Incoming message from 'd'. Must be a proper distribution. If uniform, the result will be uniform.</param> /// <param name="to_d">Previous outgoing message to 'd'.</param> /// <returns>The outgoing VMP message to the 'd' argument</returns> /// <remarks><para> /// The outgoing message is the factor viewed as a function of 'd' with 'exp' integrated out. /// The formula is <c>sum_exp p(exp) factor(exp,d)</c>. /// </para></remarks> /// <exception cref="ImproperMessageException"><paramref name="exp"/> is not a proper distribution</exception> /// <exception cref="ImproperMessageException"><paramref name="d"/> is not a proper distribution</exception> public static Gaussian DAverageLogarithm([Proper] Gamma exp, [Proper, Stochastic] Gaussian d, Gaussian to_d) { if (exp.IsPointMass) return ExpOp.DAverageLogarithm(exp.Point); double m, v; d.GetMeanAndVariance(out m, out v); Gaussian msg = new Gaussian(); double mu, s2; var prior = d / to_d; prior.GetMeanAndVariance(out mu, out s2); var z = Vector.Zero(2); z[0] = m; z[1] = Math.Log(v); double startingValue = GradientAndValueAtPoint(mu, s2, z, exp.Shape, exp.Rate, null); var s = new BFGS(); int evalCounter = 0; s.MaximumStep = 1e3; s.MaximumIterations = 100; s.Epsilon = 1e-5; s.convergenceCriteria = BFGS.ConvergenceCriteria.Objective; z = s.Run(z, 1.0, delegate(Vector y, ref Vector grad) { evalCounter++; return GradientAndValueAtPoint(mu, s2, y, exp.Shape, exp.Rate, grad); }); m = z[0]; v = Math.Exp(z[1]); to_d.SetMeanAndVariance(m, v); to_d.SetToRatio(to_d, prior); double endValue = GradientAndValueAtPoint(mu, s2, z, exp.Shape, exp.Rate, null); //Console.WriteLine("Went from {0} to {1} in {2} steps, {3} evals", startingValue, endValue, s.IterationsPerformed, evalCounter); if (startingValue < endValue) Console.WriteLine("Warning: BFGS resulted in an increased objective function"); return to_d; /* ---------------- NEWTON ITERATION VERSION 1 ------------------- double meanTimesPrec, prec; d.GetNatural(out meanTimesPrec, out prec); Matrix K = new Matrix(2, 2); K[0, 0]=1/prec; // d2K by dmu^2 K[1, 0]=K[0, 1]=-meanTimesPrec/(prec*prec); K[1, 1]=meanTimesPrec*meanTimesPrec/Math.Pow(prec, 3)+1/(2*prec*prec); double[,,] Kprime = new double[2, 2, 2]; Kprime[0, 0, 0]=0; Kprime[0, 0, 1]=Kprime[0, 1, 0]=Kprime[1, 0, 0]=-1/(prec*prec); Kprime[0, 1, 1]=Kprime[1, 1, 0]=Kprime[1, 0, 1]=2*meanTimesPrec/Math.Pow(prec, 3); Kprime[1, 1, 1]=-3*meanTimesPrec*meanTimesPrec/Math.Pow(prec, 4)-1/Math.Pow(prec, 3); Vector gradS = new Vector(2); gradS[0]=(exp.Shape-1)/prec-exp.Rate/prec*Math.Exp((meanTimesPrec+.5)/prec); gradS[1]=-(exp.Shape-1)*meanTimesPrec/(prec*prec)+exp.Rate*(meanTimesPrec+.5)/(prec*prec)*Math.Exp((meanTimesPrec+.5)/prec); Matrix grad2S = new Matrix(2, 2); grad2S[0, 0]=-exp.Rate/(prec*prec)*Math.Exp((meanTimesPrec+.5)/prec); grad2S[0, 1]=grad2S[1, 0]=-(exp.Shape-1)/(prec*prec)+exp.Rate*(1/(prec*prec)+(meanTimesPrec+.5)/Math.Pow(prec, 3))*Math.Exp((meanTimesPrec+.5)/prec); grad2S[1, 1]=2*(exp.Shape-1)*meanTimesPrec/Math.Pow(prec, 3)-exp.Rate*(meanTimesPrec+.5)/(prec*prec)*(2/prec+(meanTimesPrec+.5)/(prec*prec))*Math.Exp((meanTimesPrec+.5)/prec); Vector phi = new Vector(new double[] { result.MeanTimesPrecision, result.Precision }); Vector gradKL = K*phi-gradS; Matrix hessianKL = K - grad2S; for (int i=0; i<2; i++) for (int j=0; j<2; j++) for (int k=0; k<2; k++) hessianKL[i, j]+=Kprime[i, j, k]*phi[k]; double step = 1; Vector direction = GammaFromShapeAndRate.twoByTwoInverse(hessianKL)*gradKL; Vector newPhi = phi - step * direction; result.SetNatural(newPhi[0], newPhi[1]); return result; ---------------- NEWTON ITERATION VERSION 2 ------------------- double mean, variance; d.GetMeanAndVariance(out mean, out variance); Gaussian prior = d / result; double mean1, variance1; prior.GetMeanAndVariance(out mean1, out variance1); Vector gradKL = new Vector(2); gradKL[0]=-(exp.Shape-1)+exp.Rate*Math.Exp(mean+variance/2)+mean/variance1-mean1/variance1; gradKL[1]=-1/(2*variance)+exp.Rate*Math.Exp(mean+variance/2)+1/(2*variance1); Matrix hessianKL = new Matrix(2, 2); hessianKL[0, 0]=exp.Rate*Math.Exp(mean+variance/2)+1/variance1; hessianKL[0, 1]=hessianKL[1, 0]=.5*exp.Rate*Math.Exp(mean+variance/2); hessianKL[1, 1]=1/(2*variance*variance)+exp.Rate*Math.Exp(mean+variance/2)/4; result.GetMeanAndVariance(out mean, out variance); if (double.IsInfinity(variance)) variance=1000; Vector theta = new Vector(new double[] { mean, variance }); theta -= GammaFromShapeAndRate.twoByTwoInverse(hessianKL)*gradKL; result.SetMeanAndVariance(theta[0], theta[1]); return result; ----------------------------------------------------------------- */ }
static void Main(string[] args) { RandomGenerator generator = new RandomGenerator(); //RandomGenerator generator = new RandomGenerator(3); ReadData(@"d:\Projects\HeroesModel\HeroesModel\data.csv"); //GenerateModelData(generator, 1000); data.Sort((x, y) => { if (x.red != y.red) { return(x.red.CompareTo(y.red)); } return(x.blue.CompareTo(y.blue)); }); PrintData(); BFGS bfgs = new BFGS(TOTAL_PARAMS + 1 + TOWN_TYPES * TOWN_TYPES * 4, LogLikelyhood, LogLikelyHoodGradient); //BFGS bfgs = new BFGS(TOTAL_PARAMS + 1, LogLikelyhood, LogLikelyHoodGradient); //bfgs.FunctionTolerance = EPS; DateTime startTime = DateTime.Now; ManualResetEvent interrupted = new ManualResetEvent(false); Thread reportThread = new Thread(() => { while (!interrupted.WaitOne(10000)) { Console.WriteLine($"Time elapsed {(DateTime.Now - startTime)}, iterations complete {bfgs.IterationsDone}"); } }); reportThread.Start(); //bfgs.Minimize(new Vector(TOTAL_PARAMS, 0.01)); //RandomGenerator generator = new RandomGenerator(1); //RandomGenerator generator = new RandomGenerator(1); Normal normal = new Normal(generator, 0, 0.1); bfgs.Minimize(normal.Sample(TOTAL_PARAMS + 1 + TOWN_TYPES * TOWN_TYPES * 4)); //bfgs.Minimize(normal.Sample(TOTAL_PARAMS + 1)); interrupted.Set(); reportThread.Join(); Vector result = bfgs.MinimumPoint; double min = -LogLikelyhood(result); Console.WriteLine($"Converged. Time elapsed {(DateTime.Now - startTime)}. Iterations: {bfgs.IterationsDone}"); for (double money = -0.5; money <= 0.5; money += 0.1) { Console.WriteLine($"Red money {(money + 1) * 10000}"); PrintProbabilities(result, money); } PrintCoeficients(result); EstimateInputData(result, @"d:\Projects\HeroesModel\display\data.js"); PrintCoefficients(result, @"d:\Projects\HeroesModel\display\coefficients.js"); //GaussKronrodSingular integration = new GaussKronrodSingular(); //integration.Integrate(x => ConditionalDensity(x, 0, 1, 0, 1), Double.NegativeInfinity, Double.PositiveInfinity, 100); //Console.WriteLine(integration.Result); }