/// <summary> /// Starts a generic version of OPTANO Algorithm Tuner, using the specified arguments. /// </summary> /// <param name="additionalArguments">The additional arguments.</param> /// <param name="bbobConfig">The bbob configuration.</param> public static void Run(IEnumerable <string> additionalArguments, BbobRunnerConfiguration bbobConfig) { string pathToInstanceFolder = null; var bestParameters = Master <BbobRunner, InstanceFile, ContinuousResult, TLearnerModel, TPredictorModel, TSamplingStrategy> .Run( args : additionalArguments.ToArray(), algorithmTunerBuilder : ( config, trainingInstances, testInstance) => { pathToInstanceFolder = trainingInstances ?? "DummyInstances"; return(GenericBbobEntryPoint <TLearnerModel, TPredictorModel, TSamplingStrategy> .BuildBbobRunner( config, pathToInstanceFolder, testInstance, bbobConfig)); }); // log results GenericBbobEntryPoint <TLearnerModel, TPredictorModel, TSamplingStrategy> .LogBestParameters( bestParameters, bbobConfig, pathToInstanceFolder); }
private static void LogBestParameters( Dictionary <string, IAllele> bestParameters, BbobRunnerConfiguration bbobConfig, string pathToInstanceFolder) { var bestParamsConsole = string.Join( " ", GenericBbobEntryPoint <TLearnerModel, TPredictorModel, TSamplingStrategy> .FormatParametersAsEnumerable(bestParameters)); // evaluate the best found config on all instances. print commands to execute python + compute average performance var pythonCommand = string.Concat( bbobConfig.PythonBin, " ", bbobConfig.PathToExecutable, $" {bbobConfig.FunctionId} {"{0}"} ", bestParamsConsole); var instances = BbobUtils.CreateInstanceList(pathToInstanceFolder); Thread.Sleep(500); LoggingHelper.WriteLine(VerbosityLevel.Info, "################################################"); LoggingHelper.WriteLine(VerbosityLevel.Info, "Commands to evaluate parameters:"); var instanceResults = new List <double>(instances.Count); foreach (var instance in instances) { var bbobRunner = new BbobRunner(bbobConfig.FunctionId, bestParameters, bbobConfig.PythonBin, bbobConfig.PathToExecutable); var runTask = bbobRunner.Run(instance, new CancellationToken(false)); var currentResult = runTask.Result; // read instance id var instanceId = -1; using (var reader = File.OpenText(instance.Path)) { instanceId = int.Parse(reader.ReadLine()); } var finalCommand = string.Format(pythonCommand, instanceId); LoggingHelper.WriteLine(VerbosityLevel.Info, finalCommand, false); LoggingHelper.WriteLine(VerbosityLevel.Info, FormattableString.Invariant($"result={currentResult.Value}"), false); instanceResults.Add(currentResult.Value); } LoggingHelper.WriteLine(VerbosityLevel.Info, "################################################"); var averageResult = instanceResults.Any() ? instanceResults.Average() : double.NaN; if (!File.Exists("averageResults.csv")) { var header = string.Join(";", Enumerable.Range(0, instances.Count).Select(i => $"instance_{i}")) + ";average\r\n"; File.WriteAllText("averageResults.csv", header); } var averageInvariant = string.Format(CultureInfo.InvariantCulture, "{0}\r\n", averageResult); var resultLine = string.Join(";", instanceResults.Select(r => string.Format(CultureInfo.InvariantCulture, "{0}", r))); File.AppendAllText("averageResults.csv", string.Concat(resultLine, ";", averageInvariant)); LoggingHelper.WriteLine(VerbosityLevel.Info, FormattableString.Invariant($"Average Result={averageResult}")); }
/// <summary> /// Entry point to the program. /// </summary> /// <param name="args">If 'master' is included as argument, a /// <see cref="Master{TTargetAlgorithm,TInstance,TResult}"/> is starting using the remaining arguments. /// Otherwise, a <see cref="Worker"/> is started with the provided arguments.</param> public static void Main(string[] args) { ProcessUtils.SetDefaultCultureInfo(CultureInfo.InvariantCulture); LoggingHelper.Configure($"parserLog_{ProcessUtils.GetCurrentProcessId()}.log"); // Parse arguments. var argsParser = new BbobRunnerConfigurationParser(); if (!ArgumentParserUtils.ParseArguments(argsParser, args)) { return; } var bbobConfig = argsParser.ConfigurationBuilder.Build(); if (argsParser.IsMaster) { switch (bbobConfig.GenericParameterization) { case GenericParameterization.RandomForestAverageRank: GenericBbobEntryPoint < GenomePredictionRandomForest <AverageRankStrategy>, GenomePredictionForestModel <GenomePredictionTree>, AverageRankStrategy> .Run(argsParser.AdditionalArguments, bbobConfig); break; case GenericParameterization.RandomForestReuseOldTrees: GenericBbobEntryPoint < GenomePredictionRandomForest <ReuseOldTreesStrategy>, GenomePredictionForestModel <GenomePredictionTree>, ReuseOldTreesStrategy> .Run(argsParser.AdditionalArguments, bbobConfig); break; case GenericParameterization.StandardRandomForest: case GenericParameterization.Default: default: GenericBbobEntryPoint < StandardRandomForestLearner <ReuseOldTreesStrategy>, GenomePredictionForestModel <GenomePredictionTree>, ReuseOldTreesStrategy> .Run(argsParser.AdditionalArguments, bbobConfig); break; } } else { Worker.Run(args); return; } }
/// <summary> /// Entry point to the program. /// </summary> /// <param name="args">If 'master' is included as argument, a /// <see cref="Master{TTargetAlgorithm,TInstance,TResult}"/> is starting using the remaining arguments. /// Otherwise, a <see cref="Worker"/> is started with the provided arguments.</param> public static void Main(string[] args) { LoggingHelper.Configure("bbobParserLog.txt"); var configParser = new BbobRunnerConfigurationParser(); if (!ArgumentParserUtils.ParseArguments(configParser, args)) { return; } var bbobConfig = configParser.ConfigurationBuilder.Build(); if (bbobConfig.IsMaster) { switch (bbobConfig.GenericParameterization) { case GenericParameterization.RandomForestAverageRank: GenericBbobEntryPoint < GenomePredictionRandomForest <AverageRankStrategy>, GenomePredictionForestModel <GenomePredictionTree>, AverageRankStrategy> .Run(configParser.AdditionalArguments, bbobConfig); break; case GenericParameterization.RandomForestReuseOldTrees: GenericBbobEntryPoint < GenomePredictionRandomForest <ReuseOldTreesStrategy>, GenomePredictionForestModel <GenomePredictionTree>, ReuseOldTreesStrategy> .Run(configParser.AdditionalArguments, bbobConfig); break; case GenericParameterization.StandardRandomForest: case GenericParameterization.Default: default: GenericBbobEntryPoint < StandardRandomForestLearner <ReuseOldTreesStrategy>, GenomePredictionForestModel <GenomePredictionTree>, ReuseOldTreesStrategy> .Run(configParser.AdditionalArguments, bbobConfig); break; } } else { Worker.Run(args); return; } }