private static AlgorithmTuner <BbobRunner, InstanceFile, ContinuousResult, TLearnerModel, TPredictorModel, TSamplingStrategy> BuildBbobRunner(
AlgorithmTunerConfiguration configuration,
string trainingInstanceFolder,
string testInstanceFolder,
BbobRunnerConfiguration bbobRunnerConfig)
{
var requiredInstances = (int)Math.Max(configuration.StartNumInstances, configuration.EndNumInstances);
var random = new Random(bbobRunnerConfig.InstanceSeed);
var tuner = new AlgorithmTuner <BbobRunner, InstanceFile, ContinuousResult, TLearnerModel, TPredictorModel, TSamplingStrategy>(
targetAlgorithmFactory: new BbobRunnerFactory(
bbobRunnerConfig.PythonBin,
bbobRunnerConfig.PathToExecutable,
bbobRunnerConfig.FunctionId),
runEvaluator: new SortByValue <InstanceFile>(@ascending: true),
trainingInstances: BbobUtils.CreateInstancesFilesAndReturnAsList(trainingInstanceFolder, requiredInstances, random),
parameterTree: BbobUtils.CreateParameterTree(bbobRunnerConfig.Dimensions),
configuration: configuration);
if (!string.IsNullOrWhiteSpace(testInstanceFolder))
{
tuner.SetTestInstances(BbobUtils.CreateInstancesFilesAndReturnAsList(testInstanceFolder, requiredInstances, random));
}
return(tuner);
}
private static void LogBestParameters(
Dictionary <string, IAllele> bestParameters,
BbobRunnerConfiguration bbobConfig,
string pathToInstanceFolder)
{
var bestParamsConsole = string.Join(
" ",
GenericBbobEntryPoint <TLearnerModel, TPredictorModel, TSamplingStrategy> .FormatParametersAsEnumerable(bestParameters));
// evaluate the best found config on all instances. print commands to execute python + compute average performance
var pythonCommand = string.Concat(
bbobConfig.PythonBin,
" ",
bbobConfig.PathToExecutable,
$" {bbobConfig.FunctionId} {"{0}"} ",
bestParamsConsole);
var instances = BbobUtils.CreateInstanceList(pathToInstanceFolder);
Thread.Sleep(500);
LoggingHelper.WriteLine(VerbosityLevel.Info, "################################################");
LoggingHelper.WriteLine(VerbosityLevel.Info, "Commands to evaluate parameters:");
var instanceResults = new List <double>(instances.Count);
foreach (var instance in instances)
{
var bbobRunner = new BbobRunner(bbobConfig.FunctionId, bestParameters, bbobConfig.PythonBin, bbobConfig.PathToExecutable);
var runTask = bbobRunner.Run(instance, new CancellationToken(false));
var currentResult = runTask.Result;
// read instance id
var instanceId = -1;
using (var reader = File.OpenText(instance.Path))
{
instanceId = int.Parse(reader.ReadLine());
}
var finalCommand = string.Format(pythonCommand, instanceId);
LoggingHelper.WriteLine(VerbosityLevel.Info, finalCommand, false);
LoggingHelper.WriteLine(VerbosityLevel.Info, FormattableString.Invariant($"result={currentResult.Value}"), false);
instanceResults.Add(currentResult.Value);
}
LoggingHelper.WriteLine(VerbosityLevel.Info, "################################################");
var averageResult = instanceResults.Any() ? instanceResults.Average() : double.NaN;
if (!File.Exists("averageResults.csv"))
{
var header = string.Join(";", Enumerable.Range(0, instances.Count).Select(i => $"instance_{i}")) + ";average\r\n";
File.WriteAllText("averageResults.csv", header);
}
var averageInvariant = string.Format(CultureInfo.InvariantCulture, "{0}\r\n", averageResult);
var resultLine = string.Join(";", instanceResults.Select(r => string.Format(CultureInfo.InvariantCulture, "{0}", r)));
File.AppendAllText("averageResults.csv", string.Concat(resultLine, ";", averageInvariant));
LoggingHelper.WriteLine(VerbosityLevel.Info, FormattableString.Invariant($"Average Result={averageResult}"));
}
/// <summary>
/// Creates the instances files and return it as list.
/// </summary>
/// <param name="pathToInstanceFolder">The path to instance folder.</param>
/// <param name="requiredInstanceNumber">The required instance number.</param>
/// <param name="random">The random generator for seeds.</param>
/// <returns>The instance files as list.</returns>
public static List <InstanceFile> CreateInstancesFilesAndReturnAsList(string pathToInstanceFolder, int requiredInstanceNumber, Random random)
{
BbobUtils.CreateInstanceFiles(pathToInstanceFolder, requiredInstanceNumber, random);
return(BbobUtils.CreateInstanceList(pathToInstanceFolder));
}
