/// <summary>
        /// Testing a hyper-parameter configuration for all test functions
        /// </summary>
        /// <param name="args"></param>
        static void TestHyperParam(string solver, double [] hp, int testfuncdim, int evalbudgetmultipl, int rerunsTestFuncs)
        {
            // solver = "SGA";
            ////double[] hp;
            // testfuncdim = 13;
            //int evalbudgetmultipl = 100;
            //int rerunsTestFuncs = 30;


            ////SGA PARAMETERS
            //double[] hp = new double[7];
            //hp[0] = 15.20115;   // popsize
            //hp[1] = 90.24627;   // k
            //hp[2] = 1;  // pcross
            //hp[3] = 0.80054;    // pmut
            //hp[4] = 1.80471;    // d
            //hp[5] = 0.01;   // r
            //hp[6] = 0.26437;	// elites


            ////ES parameters
            //double[] hp = new double[8];
            //// cluster a
            //hp[0] = 2;
            //hp[1] = 1;
            //hp[2] = 0.16085;
            //hp[3] = 0.07627;
            //hp[4] = 0.01;
            //hp[5] = 0.2288;
            //hp[6] = 0.01136;
            //hp[7] = 0.92639;


            ////PSO PARAmeters
            //double[] hp = new double[7];
            //// cluster
            //hp[0] = 9.03316;
            //hp[1] = 0.001;
            //hp[2] = 18.24318;
            //hp[3] = 0;
            //hp[4] = 1;
            //hp[5] = 0;
            //hp[6] = 3.22193;



            ////FIPSO PARAMeters
            //double[] hp = new double[5];
            //// cluster a
            //hp[0] = 12.16184;
            //hp[1] = 0.55492;
            //hp[2] = 9.31087;
            //hp[3] = 0.59655;
            //hp[4] = 1;



            //TestHyperParam("SGA", hp, 13, 100, 30);



            //int evalbudgetmultiplier = 100;
            //int testfuncdim = 20;                //tuned to this n
            //int rerunsTestFuncs = 30;            // 30
            HyperParameterFunctions hf = new HyperParameterFunctions(testfuncdim, rerunsTestFuncs, solver, evalbudgetmultipl);   //problem dim; reruns of testfuncs; solver to be tuned

            Func <double[], double>[] manyhyperfuncs = new Func <double[], double> [20];
            manyhyperfuncs[0]  = hf.HyperFunc_B_Perm0db;
            manyhyperfuncs[1]  = hf.HyperFunc_B_RotHypEll;
            manyhyperfuncs[2]  = hf.HyperFunc_B_Sphere;
            manyhyperfuncs[3]  = hf.HyperFunc_B_SumSquares;
            manyhyperfuncs[4]  = hf.HyperFunc_B_Trid;
            manyhyperfuncs[5]  = hf.HyperFunc_L_Ackley;
            manyhyperfuncs[6]  = hf.HyperFunc_L_Griewank;
            manyhyperfuncs[7]  = hf.HyperFunc_L_Levy;
            manyhyperfuncs[8]  = hf.HyperFunc_L_Rastrigin;
            manyhyperfuncs[9]  = hf.HyperFunc_L_Schwefel;
            manyhyperfuncs[10] = hf.HyperFunc_O_PermDB;
            manyhyperfuncs[11] = hf.HyperFunc_O_StyblinskiTang;
            manyhyperfuncs[12] = hf.HyperFunc_P_Zhakarov;
            manyhyperfuncs[13] = hf.HyperFunc_V_DixonPrice;
            manyhyperfuncs[14] = hf.HyperFunc_V_Rosenbrock;
            manyhyperfuncs[15] = hf.HyperFunc_L_Levy_Edge;
            manyhyperfuncs[16] = hf.HyperFunc_L_Schwefel_Edge;
            manyhyperfuncs[17] = hf.HyperFunc_O_StyblinskiTang_Edge;
            manyhyperfuncs[18] = hf.HyperFunc_V_DixonPrice_Edge;
            manyhyperfuncs[19] = hf.HyperFunc_V_Rosenbrock_Edge;

            double[] means = new double[manyhyperfuncs.Length];
            for (int i = 0; i < manyhyperfuncs.Length; i++)
            {
                means[i] = manyhyperfuncs[i](hp);
                Console.WriteLine(means[i]);
            }
            Console.ReadKey();
        }
        /// <summary>
        /// Testing a hyper-parameter configuration for one test function
        /// </summary>
        /// <param name="args"></param>
        static void TestHyperParamOneFuncOnly()
        {
            string solver = "SGA";
            //double[] hp;
            int testfuncIndex     = 19;
            int testfuncdim       = 35;
            int evalbudgetmultipl = 35;
            int rerunsTestFunc    = 35;

            //SGA PARAMETERS
            double[] hp = new double[7];
            hp[0] = 15.20115; // popsize
            hp[1] = 90.24627; // k
            hp[2] = 1;        // pcross
            hp[3] = 0.80054;  // pmut
            hp[4] = 1.80471;  // d
            hp[5] = 0.01;     // r
            hp[6] = 0.26437;  // elites


            ////ES parameters
            //double[] hp = new double[8];
            //// cluster a
            //hp[0] = 2;
            //hp[1] = 1;
            //hp[2] = 0.16085;
            //hp[3] = 0.07627;
            //hp[4] = 0.01;
            //hp[5] = 0.2288;
            //hp[6] = 0.01136;
            //hp[7] = 0.92639;


            ////PSO PARAmeters
            //double[] hp = new double[7];
            //// cluster
            //hp[0] = 9.03316;
            //hp[1] = 0.001;
            //hp[2] = 18.24318;
            //hp[3] = 0;
            //hp[4] = 1;
            //hp[5] = 0;
            //hp[6] = 3.22193;



            ////FIPSO PARAMeters
            //double[] hp = new double[5];
            //// cluster a
            //hp[0] = 12.16184;
            //hp[1] = 0.55492;
            //hp[2] = 9.31087;
            //hp[3] = 0.59655;
            //hp[4] = 1;


            //TestHyperParamOneFuncOnly("FIPSO", hp, 19, 35, 30, 30);



            //int evalbudgetmultiplier = 100;
            //int testfuncdim = 20;                //tuned to this n
            //int rerunsTestFuncs = 30;            // 30
            HyperParameterFunctions hf = new HyperParameterFunctions(testfuncdim, rerunsTestFunc, solver, evalbudgetmultipl);   //problem dim; reruns of testfuncs; solver to be tuned

            Func <double[], double[]>[] manyhyperfuncs = new Func <double[], double[]> [20];
            manyhyperfuncs[0]  = hf.HyperFunc_B_Perm0db_dist;
            manyhyperfuncs[1]  = hf.HyperFunc_B_RotHypEll_dist;
            manyhyperfuncs[2]  = hf.HyperFunc_B_Sphere_dist;
            manyhyperfuncs[3]  = hf.HyperFunc_B_SumSquares_dist;
            manyhyperfuncs[4]  = hf.HyperFunc_B_Trid_dist;
            manyhyperfuncs[5]  = hf.HyperFunc_L_Ackley_dist;
            manyhyperfuncs[6]  = hf.HyperFunc_L_Griewank_dist;
            manyhyperfuncs[7]  = hf.HyperFunc_L_Levy_dist;
            manyhyperfuncs[8]  = hf.HyperFunc_L_Rastrigin_dist;
            manyhyperfuncs[9]  = hf.HyperFunc_L_Schwefel_dist;
            manyhyperfuncs[10] = hf.HyperFunc_O_PermDB_dist;
            manyhyperfuncs[11] = hf.HyperFunc_O_StyblinskiTang_dist;
            manyhyperfuncs[12] = hf.HyperFunc_P_Zhakarov_dist;
            manyhyperfuncs[13] = hf.HyperFunc_V_DixonPrice_dist;
            manyhyperfuncs[14] = hf.HyperFunc_V_Rosenbrock_dist;
            manyhyperfuncs[15] = hf.HyperFunc_L_Levy_Edge_dist;
            manyhyperfuncs[16] = hf.HyperFunc_L_Schwefel_Edge_dist;
            manyhyperfuncs[17] = hf.HyperFunc_O_StyblinskiTang_Edge_dist;
            manyhyperfuncs[18] = hf.HyperFunc_V_DixonPrice_Edge_dist;
            manyhyperfuncs[19] = hf.HyperFunc_V_Rosenbrock_Edge_dist;

            double [] dist = manyhyperfuncs[testfuncIndex](hp);
            foreach (double fx in dist)
            {
                Console.WriteLine(fx);
            }
            Console.WriteLine();
            Console.WriteLine("expected value: {0}", dist.Average());
            Console.ReadKey();
        }
        /// <summary>
        /// Tuning hyper-parameters of a solver
        /// </summary>
        /// <param name="solver">Options: SGA, ES, FIPSO, PSO</param>
        /// <param name="testfuncdim">dimensionality n</param>
        /// <param name="evalbudgetmultilp">evaluation budget per test function</param>
        internal static void TuneSolver(string solver, int testfuncdim, int evalbudgetmultilp)
        {
            //string solver = "PSO";               //choosing solver to be tuned. string: "SGA", "ES", "FIPSO", "SA"
            int maxfunc = 5000;                  //func calls of the meta-optimizer. 5000
                                                 //int testfuncdim = 20;                //tuned to this n
            int rerunsMeta      = 10;            // 10
            int rerunsTestFuncs = 30;            // 30


            HyperParameterFunctions hf = new HyperParameterFunctions(testfuncdim, rerunsTestFuncs, solver, evalbudgetmultilp); //problem dim; reruns of testfuncs; solver to be tuned

            int[] seeds_in = new int[rerunsMeta];                                                                              //reruns of meta-optimizer
            for (int i = 0; i < seeds_in.Length; i++)
            {
                seeds_in[i] = i;
            }

            int dvar;

            double[] ub, lb;
            bool[]   xint;
            hf.getXandBounds(out dvar, out lb, out ub, out xint, solver);


            Console.WriteLine();
            Console.WriteLine(@"///////////////////////////////////////");
            Console.WriteLine("Hyperparameter optimization of...: {0}", solver);
            Console.WriteLine(@"tuned to n = {0}", testfuncdim);
            Console.WriteLine(@"Meta-optimizers: SGA and ES. Each with {0} re-runs", seeds_in.Length);
            Console.WriteLine(@"///////////////////////////////////////");
            Console.WriteLine();
            Console.WriteLine(@"Please enter an existing output path for results. E.g.: c:\n13\");
            string basepath = Console.ReadLine();

            Console.WriteLine();
            Console.WriteLine(@"How many threads would you like to assign to this? Please enter an integer between 1 and 20.");
            int maxthreads = Convert.ToInt16(Console.ReadLine());

            Console.WriteLine();
            Console.WriteLine("Thanks! Starting... This might take a couple of days.");



            //Func<double[], double>[] manyhyperfuncs = new Func<double[], double>[1];
            //manyhyperfuncs[0] = hf.HyperFunc_B_Perm0db;
            //manyhyperfuncs[1] = hf.HyperFunc_B_RotHypEll;
            //manyhyperfuncs[2] = hf.HyperFunc_L_Ackley;
            //manyhyperfuncs[3] = hf.HyperFunc_O_PermDB;
            //manyhyperfuncs[4] = hf.HyperFunc_L_Schwefel_Edge;
            //manyhyperfuncs[0] = hf.HyperFunc_L_Levy_Edge;

            Func <double[], double>[] manyhyperfuncs = new Func <double[], double> [20];
            manyhyperfuncs[0]  = hf.HyperFunc_B_Perm0db;
            manyhyperfuncs[1]  = hf.HyperFunc_B_RotHypEll;
            manyhyperfuncs[2]  = hf.HyperFunc_B_Sphere;
            manyhyperfuncs[3]  = hf.HyperFunc_B_SumSquares;
            manyhyperfuncs[4]  = hf.HyperFunc_B_Trid;
            manyhyperfuncs[5]  = hf.HyperFunc_L_Ackley;
            manyhyperfuncs[6]  = hf.HyperFunc_L_Griewank;
            manyhyperfuncs[7]  = hf.HyperFunc_L_Levy;
            manyhyperfuncs[8]  = hf.HyperFunc_L_Rastrigin;
            manyhyperfuncs[9]  = hf.HyperFunc_L_Schwefel;
            manyhyperfuncs[10] = hf.HyperFunc_O_PermDB;
            manyhyperfuncs[11] = hf.HyperFunc_O_StyblinskiTang;
            manyhyperfuncs[12] = hf.HyperFunc_P_Zhakarov;
            manyhyperfuncs[13] = hf.HyperFunc_V_DixonPrice;
            manyhyperfuncs[14] = hf.HyperFunc_V_Rosenbrock;
            manyhyperfuncs[15] = hf.HyperFunc_L_Levy_Edge;
            manyhyperfuncs[16] = hf.HyperFunc_L_Schwefel_Edge;
            manyhyperfuncs[17] = hf.HyperFunc_O_StyblinskiTang_Edge;
            manyhyperfuncs[18] = hf.HyperFunc_V_DixonPrice_Edge;
            manyhyperfuncs[19] = hf.HyperFunc_V_Rosenbrock_Edge;



            //manyhyperfuncs[15] = hf.HyperFunc_B_RotHypEll_Edge;
            //manyhyperfuncs[16] = hf.HyperFunc_B_Sphere_Edge;
            //manyhyperfuncs[17] = hf.HyperFunc_B_SumSquares_Edge;
            //manyhyperfuncs[18] = hf.HyperFunc_L_Ackley_Edge;
            //manyhyperfuncs[19] = hf.HyperFunc_L_Griewank_Edge;
            //manyhyperfuncs[21] = hf.HyperFunc_L_Rastrigin_Edge;
            //manyhyperfuncs[24] = hf.HyperFunc_P_Zhakarov_Edge;



            //meta optimizer SGA
            Dictionary <string, object> settingsSGA = new Dictionary <string, object>();

            settingsSGA.Add("maxgen", 10000);
            settingsSGA.Add("popsize", 20);
            settingsSGA.Add("elite", 1);
            settingsSGA.Add("pcross", 1);
            settingsSGA.Add("pmut", 0.3);
            settingsSGA.Add("d", 1.1);
            settingsSGA.Add("r", 0.1);
            settingsSGA.Add("k", 6);


            //meta optimizer FIPSO
            Dictionary <string, object> settingsPSO = new Dictionary <string, object>();

            //meta optimizer ES
            Dictionary <string, object> settingsES = new Dictionary <string, object>();

            settingsES.Add("popsize", 20);        // ∈ {2,...,200}
            settingsES.Add("lambda", 10);         // ∈ {1,...,200}
            settingsES.Add("roh", 2);             // ∈ {1,...,popsize}  . in hyperoptimization, express as percentage of lambda
            settingsES.Add("x0sampling", 1);      // ∈ {0,1}  0=uniform, 1=gaussian
            settingsES.Add("stepsize0", 5);       // ∈ [0.01, 10]
            settingsES.Add("stepsize", 2.2);      // ∈ [0.01, 10]
            settingsES.Add("tauc", 1);            // ∈ [0.01, 50]



            Parallel.For(0, manyhyperfuncs.Length, new ParallelOptions {
                MaxDegreeOfParallelism = maxthreads
            }, hh =>
            {
                //for(int hh=0; hh<manyhyperfuncs.Length; hh++){
                Console.WriteLine("started hh {0} of {1}", hh, manyhyperfuncs.Length - 1);
                List <string> log_SA  = new List <string>();
                List <string> log_ES  = new List <string>();
                List <string> log_PSO = new List <string>();
                List <string> log_SGA = new List <string>();
                string str;
                for (int iseeds = 0; iseeds < seeds_in.Length; iseeds++)
                {
                    /////////////////////////////////////////            SGA          /////////////////////////////////////////
                    ///////////////////////////////////////////////////////////////////////////////////////////////////////////
                    GeneticAlgorithm ga = new GeneticAlgorithm(lb, ub, xint, maxfunc, manyhyperfuncs[hh], iseeds, settingsSGA);
                    ga.solve();
                    ga.get_fxoptimum();
                    str = "Last call;" + Math.Round(ga.get_fxoptimum(), 4);
                    for (int xx = 0; xx < dvar; xx++)
                    {
                        str += ";" + Math.Round(ga.get_Xoptimum()[xx], 5);
                    }
                    str += ";SGA";
                    log_SGA.Add(str);
                    Console.WriteLine("Metaoptimizer SGA DONE, seed {0} and hh {1}", iseeds, hh);


                    /////////////////////////////////////////            ES           /////////////////////////////////////////
                    ///////////////////////////////////////////////////////////////////////////////////////////////////////////
                    EvolutionStrategy es = new EvolutionStrategy(lb, ub, xint, maxfunc, manyhyperfuncs[hh], iseeds, settingsSGA);
                    es.solve();
                    es.get_fxoptimum();
                    str = "Last call;" + Math.Round(es.get_fxoptimum(), 4);
                    for (int xx = 0; xx < dvar; xx++)
                    {
                        str += ";" + Math.Round(es.get_Xoptimum()[xx], 5);
                    }
                    str += ";ES";
                    log_ES.Add(str);
                    Console.WriteLine("Metaoptimizer ES DONE, seed {0} and hh {1}", iseeds, hh);



                    ///////////////////////////////////////////            FIPSO        /////////////////////////////////////////
                    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
                    //PSO pso = new PSO(lb, ub, xint, maxfunc, manyhyperfuncs[hh], iseeds, settingsPSO);
                    //pso.solve();
                    //pso.get_fxoptimum();
                    //str = "Last call;" + Math.Round(pso.get_fxoptimum(), 4);
                    //for (int xx = 0; xx < dvar; xx++)
                    //{
                    //    str += ";" + Math.Round(pso.get_Xoptimum()[xx], 5);
                    //}
                    //str += ";PSO";
                    //log_PSO.Add(str);
                    //Console.WriteLine("Metaoptimizer PSO DONE, seed {0} and hh {1}", iseeds, hh);
                }


                /////////////////////////////////////////            Writing          ///////////////////////////////////
                ///////////////////////////////////////////////////////////////////////////////////////////////////////////
                string fileName = basepath + solver + @"_hf_" + hh + "_AllSeeds.txt";
                using (FileStream fs = new FileStream(fileName, FileMode.Append, FileAccess.Write))
                    using (StreamWriter sw = new StreamWriter(fs))
                    {
                        for (int i = 0; i < log_SA.Count; i++)
                        {
                            sw.WriteLine(log_SA[i]);
                        }
                        for (int i = 0; i < log_ES.Count; i++)
                        {
                            sw.WriteLine(log_ES[i]);
                        }
                        for (int i = 0; i < log_SGA.Count; i++)
                        {
                            sw.WriteLine(log_SGA[i]);
                        }
                        for (int i = 0; i < log_PSO.Count; i++)
                        {
                            sw.WriteLine(log_PSO[i]);
                        }
                    }



                Console.WriteLine("Done hh {0} of {1}", hh, manyhyperfuncs.Length);
            });
            //}

            //hf.printLogs(basepath, 15);

            Console.WriteLine();
            Console.WriteLine(@"///////////////////////////////////////");
            Console.WriteLine("Done with everything, tuning the {0}", solver);
            Console.WriteLine(@"///////////////////////////////////////");
            Console.ReadKey();
        }