/// <summary>
/// Starts a generic version of OPTANO Algorithm Tuner, using the specified arguments.
/// </summary>
/// <param name="additionalArguments">The additional arguments.</param>
/// <param name="bbobConfig">The bbob configuration.</param>
public static void Run(IEnumerable <string> additionalArguments, BbobRunnerConfiguration bbobConfig)
{
string pathToInstanceFolder = null;
var bestParameters =
Master <BbobRunner,
InstanceFile,
ContinuousResult,
TLearnerModel,
TPredictorModel,
TSamplingStrategy>
.Run(
args : additionalArguments.ToArray(),
algorithmTunerBuilder :
(
config,
trainingInstances,
testInstance) =>
{
pathToInstanceFolder = trainingInstances ?? "DummyInstances";
return(GenericBbobEntryPoint <TLearnerModel, TPredictorModel, TSamplingStrategy> .BuildBbobRunner(
config,
pathToInstanceFolder,
testInstance,
bbobConfig));
});
// log results
GenericBbobEntryPoint <TLearnerModel, TPredictorModel, TSamplingStrategy> .LogBestParameters(
bestParameters,
bbobConfig,
pathToInstanceFolder);
}
private static void LogBestParameters(
Dictionary <string, IAllele> bestParameters,
BbobRunnerConfiguration bbobConfig,
string pathToInstanceFolder)
{
var bestParamsConsole = string.Join(
" ",
GenericBbobEntryPoint <TLearnerModel, TPredictorModel, TSamplingStrategy> .FormatParametersAsEnumerable(bestParameters));
// evaluate the best found config on all instances. print commands to execute python + compute average performance
var pythonCommand = string.Concat(
bbobConfig.PythonBin,
" ",
bbobConfig.PathToExecutable,
$" {bbobConfig.FunctionId} {"{0}"} ",
bestParamsConsole);
var instances = BbobUtils.CreateInstanceList(pathToInstanceFolder);
Thread.Sleep(500);
LoggingHelper.WriteLine(VerbosityLevel.Info, "################################################");
LoggingHelper.WriteLine(VerbosityLevel.Info, "Commands to evaluate parameters:");
var instanceResults = new List <double>(instances.Count);
foreach (var instance in instances)
{
var bbobRunner = new BbobRunner(bbobConfig.FunctionId, bestParameters, bbobConfig.PythonBin, bbobConfig.PathToExecutable);
var runTask = bbobRunner.Run(instance, new CancellationToken(false));
var currentResult = runTask.Result;
// read instance id
var instanceId = -1;
using (var reader = File.OpenText(instance.Path))
{
instanceId = int.Parse(reader.ReadLine());
}
var finalCommand = string.Format(pythonCommand, instanceId);
LoggingHelper.WriteLine(VerbosityLevel.Info, finalCommand, false);
LoggingHelper.WriteLine(VerbosityLevel.Info, FormattableString.Invariant($"result={currentResult.Value}"), false);
instanceResults.Add(currentResult.Value);
}
LoggingHelper.WriteLine(VerbosityLevel.Info, "################################################");
var averageResult = instanceResults.Any() ? instanceResults.Average() : double.NaN;
if (!File.Exists("averageResults.csv"))
{
var header = string.Join(";", Enumerable.Range(0, instances.Count).Select(i => $"instance_{i}")) + ";average\r\n";
File.WriteAllText("averageResults.csv", header);
}
var averageInvariant = string.Format(CultureInfo.InvariantCulture, "{0}\r\n", averageResult);
var resultLine = string.Join(";", instanceResults.Select(r => string.Format(CultureInfo.InvariantCulture, "{0}", r)));
File.AppendAllText("averageResults.csv", string.Concat(resultLine, ";", averageInvariant));
LoggingHelper.WriteLine(VerbosityLevel.Info, FormattableString.Invariant($"Average Result={averageResult}"));
}
/// <summary>
/// Entry point to the program.
/// </summary>
/// <param name="args">If 'master' is included as argument, a
/// <see cref="Master{TTargetAlgorithm,TInstance,TResult}"/> is starting using the remaining arguments.
/// Otherwise, a <see cref="Worker"/> is started with the provided arguments.</param>
public static void Main(string[] args)
{
ProcessUtils.SetDefaultCultureInfo(CultureInfo.InvariantCulture);
LoggingHelper.Configure($"parserLog_{ProcessUtils.GetCurrentProcessId()}.log");
// Parse arguments.
var argsParser = new BbobRunnerConfigurationParser();
if (!ArgumentParserUtils.ParseArguments(argsParser, args))
{
return;
}
var bbobConfig = argsParser.ConfigurationBuilder.Build();
if (argsParser.IsMaster)
{
switch (bbobConfig.GenericParameterization)
{
case GenericParameterization.RandomForestAverageRank:
GenericBbobEntryPoint <
GenomePredictionRandomForest <AverageRankStrategy>,
GenomePredictionForestModel <GenomePredictionTree>,
AverageRankStrategy> .Run(argsParser.AdditionalArguments, bbobConfig);
break;
case GenericParameterization.RandomForestReuseOldTrees:
GenericBbobEntryPoint <
GenomePredictionRandomForest <ReuseOldTreesStrategy>,
GenomePredictionForestModel <GenomePredictionTree>,
ReuseOldTreesStrategy> .Run(argsParser.AdditionalArguments, bbobConfig);
break;
case GenericParameterization.StandardRandomForest:
case GenericParameterization.Default:
default:
GenericBbobEntryPoint <
StandardRandomForestLearner <ReuseOldTreesStrategy>,
GenomePredictionForestModel <GenomePredictionTree>,
ReuseOldTreesStrategy> .Run(argsParser.AdditionalArguments, bbobConfig);
break;
}
}
else
{
Worker.Run(args);
return;
}
}
/// <summary>
/// Entry point to the program.
/// </summary>
/// <param name="args">If 'master' is included as argument, a
/// <see cref="Master{TTargetAlgorithm,TInstance,TResult}"/> is starting using the remaining arguments.
/// Otherwise, a <see cref="Worker"/> is started with the provided arguments.</param>
public static void Main(string[] args)
{
LoggingHelper.Configure("bbobParserLog.txt");
var configParser = new BbobRunnerConfigurationParser();
if (!ArgumentParserUtils.ParseArguments(configParser, args))
{
return;
}
var bbobConfig = configParser.ConfigurationBuilder.Build();
if (bbobConfig.IsMaster)
{
switch (bbobConfig.GenericParameterization)
{
case GenericParameterization.RandomForestAverageRank:
GenericBbobEntryPoint <
GenomePredictionRandomForest <AverageRankStrategy>,
GenomePredictionForestModel <GenomePredictionTree>,
AverageRankStrategy> .Run(configParser.AdditionalArguments, bbobConfig);
break;
case GenericParameterization.RandomForestReuseOldTrees:
GenericBbobEntryPoint <
GenomePredictionRandomForest <ReuseOldTreesStrategy>,
GenomePredictionForestModel <GenomePredictionTree>,
ReuseOldTreesStrategy> .Run(configParser.AdditionalArguments, bbobConfig);
break;
case GenericParameterization.StandardRandomForest:
case GenericParameterization.Default:
default:
GenericBbobEntryPoint <
StandardRandomForestLearner <ReuseOldTreesStrategy>,
GenomePredictionForestModel <GenomePredictionTree>,
ReuseOldTreesStrategy> .Run(configParser.AdditionalArguments, bbobConfig);
break;
}
}
else
{
Worker.Run(args);
return;
}
}
