private static AlgorithmTuner <BbobRunner, InstanceFile, ContinuousResult, TLearnerModel, TPredictorModel, TSamplingStrategy> BuildBbobRunner(
AlgorithmTunerConfiguration configuration,
string trainingInstanceFolder,
string testInstanceFolder,
BbobRunnerConfiguration bbobRunnerConfig)
{
var requiredInstances = (int)Math.Max(configuration.StartNumInstances, configuration.EndNumInstances);
var random = new Random(bbobRunnerConfig.InstanceSeed);
var tuner = new AlgorithmTuner <BbobRunner, InstanceFile, ContinuousResult, TLearnerModel, TPredictorModel, TSamplingStrategy>(
targetAlgorithmFactory: new BbobRunnerFactory(
bbobRunnerConfig.PythonBin,
bbobRunnerConfig.PathToExecutable,
bbobRunnerConfig.FunctionId),
runEvaluator: new SortByValue <InstanceFile>(@ascending: true),
trainingInstances: BbobUtils.CreateInstancesFilesAndReturnAsList(trainingInstanceFolder, requiredInstances, random),
parameterTree: BbobUtils.CreateParameterTree(bbobRunnerConfig.Dimensions),
configuration: configuration);
if (!string.IsNullOrWhiteSpace(testInstanceFolder))
{
tuner.SetTestInstances(BbobUtils.CreateInstancesFilesAndReturnAsList(testInstanceFolder, requiredInstances, random));
}
return(tuner);
}
/// <summary>
/// Builds a OPTANO Algorithm Tuner instance that tunes by process runtime.
/// </summary>
/// <param name="config">
/// The configuration.
/// </param>
/// <param name="trainingInstanceFolder">
/// The path to the folder containing training instances.
/// </param>
/// <param name="testInstanceFolder">
/// The path to the folder containing test instances.
/// </param>
/// <param name="basicCommand">
/// The basic command to the target algorithm as it should be executed by the
/// command line. The path to the instance file and the parameters will be set by replacing
/// <see cref="CommandExecutorBase{TResult}.InstanceReplacement"/> and
/// <see cref="CommandExecutorBase{TResult}.ParameterReplacement"/>.
/// </param>
/// <param name="pathToParameterTree">
/// The path to a parameter tree defined via XML.
/// </param>
/// <param name="factorParK">
/// The factor for the penalization of the average runtime.
/// </param>
/// <returns>
/// The built OPTANO Algorithm Tuner instance.
/// </returns>
private static AlgorithmTuner <TimeMeasuringExecutor, InstanceFile, RuntimeResult> BuildRuntimeTuner(
AlgorithmTunerConfiguration config,
string trainingInstanceFolder,
string testInstanceFolder,
string basicCommand,
string pathToParameterTree,
int factorParK)
{
IRunEvaluator <InstanceFile, RuntimeResult> runEvaluator;
if (factorParK == 0)
{
runEvaluator = new SortByUnpenalizedRuntime <InstanceFile>(config.CpuTimeout);
}
else
{
runEvaluator = new SortByPenalizedRuntime <InstanceFile>(factorParK, config.CpuTimeout);
}
var tuner = new AlgorithmTuner <TimeMeasuringExecutor, InstanceFile, RuntimeResult>(
targetAlgorithmFactory: new TimeMeasuringExecutorFactory(basicCommand, config.CpuTimeout),
runEvaluator: runEvaluator,
trainingInstances: ExtractInstances(trainingInstanceFolder),
parameterTree: ParameterTreeConverter.ConvertToParameterTree(pathToParameterTree),
configuration: config);
if (testInstanceFolder != null)
{
tuner.SetTestInstances(ExtractInstances(testInstanceFolder));
}
return(tuner);
}
/// <summary>
/// Builds an instance of the <see cref="AlgorithmTuner{TTargetAlorithm,TInstance,TResult}" /> class for tuning Gurobi.
/// </summary>
/// <param name="configuration">The <see cref="AlgorithmTunerConfiguration" /> to use.</param>
/// <param name="pathToTrainingInstanceFolder">The path to the folder containing training instances.</param>
/// <param name="pathToTestInstanceFolder">The path to test instance folder.</param>
/// <param name="gurobiConfigBuilder">The gurobi configuration builder.</param>
/// <returns>
/// The built instance.
/// </returns>
public static AlgorithmTuner <GurobiRunner, InstanceSeedFile, GurobiResult> BuildGurobiRunner(
AlgorithmTunerConfiguration configuration,
string pathToTrainingInstanceFolder,
string pathToTestInstanceFolder,
GurobiRunnerConfiguration.GurobiRunnerConfigBuilder gurobiConfigBuilder)
{
var gurobiConfig = gurobiConfigBuilder.Build(configuration.CpuTimeout);
var tuner = new AlgorithmTuner <GurobiRunner, InstanceSeedFile, GurobiResult>(
targetAlgorithmFactory: new GurobiRunnerFactory(gurobiConfig),
runEvaluator: new GurobiRunEvaluator(),
trainingInstances: GurobiUtils.CreateInstances(pathToTrainingInstanceFolder, gurobiConfig.NumberOfSeeds, gurobiConfig.RngSeed),
parameterTree: GurobiUtils.CreateParameterTree(),
configuration: configuration);
try
{
if (!string.IsNullOrWhiteSpace(pathToTestInstanceFolder))
{
var testInstances = GurobiUtils.CreateInstances(pathToTestInstanceFolder, gurobiConfig.NumberOfSeeds, gurobiConfig.RngSeed);
tuner.SetTestInstances(testInstances);
}
}
catch
{
}
return(tuner);
}
BuildLingelingAlgorithmTuner(
AlgorithmTunerConfiguration tunerConfig,
string trainingInstanceFolder,
string testInstanceFolder,
LingelingRunnerConfiguration runnerConfig)
{
IRunEvaluator <InstanceSeedFile, RuntimeResult> runEvaluator;
if (runnerConfig.FactorParK == 0)
{
runEvaluator = new SortByUnpenalizedRuntime <InstanceSeedFile>(tunerConfig.CpuTimeout);
}
else
{
runEvaluator = new SortByPenalizedRuntime <InstanceSeedFile>(runnerConfig.FactorParK, tunerConfig.CpuTimeout);
}
var tuner = new AlgorithmTuner <LingelingRunner, InstanceSeedFile, RuntimeResult, TLearnerModel,
TPredictorModel,
TSamplingStrategy>(
targetAlgorithmFactory: new LingelingRunnerFactory(runnerConfig.PathToExecutable, tunerConfig, runnerConfig.MemoryLimitMegabyte),
runEvaluator: runEvaluator,
trainingInstances: InstanceSeedFile.CreateInstanceSeedFilesFromDirectory(
trainingInstanceFolder,
LingelingUtils.ListOfValidFileExtensions,
runnerConfig.NumberOfSeeds,
runnerConfig.RngSeed),
parameterTree: LingelingUtils.CreateParameterTree(),
configuration: tunerConfig);
if (!string.IsNullOrWhiteSpace(testInstanceFolder))
{
tuner.SetTestInstances(
InstanceSeedFile.CreateInstanceSeedFilesFromDirectory(
testInstanceFolder,
LingelingUtils.ListOfValidFileExtensions,
runnerConfig.NumberOfSeeds,
runnerConfig.RngSeed));
}
return(tuner);
}
BuildSapsAlgorithmTuner(
AlgorithmTunerConfiguration tunerConfig,
string trainingInstanceFolder,
string testInstanceFolder,
SapsRunnerConfiguration runnerConfig)
{
var tuner = new AlgorithmTuner <SapsRunner, InstanceSeedFile, RuntimeResult, TLearnerModel, TPredictorModel, TSamplingStrategy>(
targetAlgorithmFactory: new SapsRunnerFactory(runnerConfig.PathToExecutable, tunerConfig.CpuTimeout),
runEvaluator: new SortByRuntime(runnerConfig.FactorParK),
trainingInstances: SapsUtils.CreateInstances(trainingInstanceFolder, runnerConfig.NumberOfSeeds, runnerConfig.RngSeed),
parameterTree: SapsUtils.CreateParameterTree(),
configuration: tunerConfig);
if (!string.IsNullOrWhiteSpace(testInstanceFolder))
{
tuner.SetTestInstances(SapsUtils.CreateInstances(testInstanceFolder, runnerConfig.NumberOfSeeds, runnerConfig.RngSeed));
}
return(tuner);
}
/// <summary>
/// Builds a OPTANO Algorithm Tuner instance that tunes by last value printed to console.
/// </summary>
/// <param name="config">The configuration.</param>
/// <param name="trainingInstanceFolder">The path to the folder containing training instances.</param>
/// <param name="testInstanceFolder">The path to the folder containing test instances.</param>
/// <param name="basicCommand">The basic command to the target algorithm as it should be executed by the
/// command line. The path to the instance file and the parameters will be set by replacing
/// <see cref="CommandExecutorBase{TResult}.InstanceReplacement"/> and
/// <see cref="CommandExecutorBase{TResult}.ParameterReplacement"/>.</param>
/// <param name="pathToParameterTree">The path to a parameter tree defined via XML.</param>
/// <param name="ascending">Whether smaller values are better.</param>
/// <returns>The built OPTANO Algorithm Tuner instance.</returns>
private static AlgorithmTuner <ValueReadingExecutor, InstanceFile, ContinuousResult> BuildValueTuner(
AlgorithmTunerConfiguration config,
string trainingInstanceFolder,
string testInstanceFolder,
string basicCommand,
string pathToParameterTree,
bool ascending)
{
var tuner = new AlgorithmTuner <ValueReadingExecutor, InstanceFile, ContinuousResult>(
targetAlgorithmFactory: new ValueReadingExecutorFactory(basicCommand, config.CpuTimeout),
runEvaluator: new SortByValue(ascending),
trainingInstances: ExtractInstances(trainingInstanceFolder),
parameterTree: ParameterTreeConverter.ConvertToParameterTree(pathToParameterTree),
configuration: config);
if (testInstanceFolder != null)
{
tuner.SetTestInstances(ExtractInstances(testInstanceFolder));
}
return(tuner);
}
GenomePredictionForestModel <GenomePredictionTree>, ReuseOldTreesStrategy> BuildRuntimeTuner(
ParameterTree parameterTree,
Scenario scenario,
ParameterConfigurationSpaceSpecification parameterSpecification,
IEnumerable <InstanceSeedFile> trainingInstances,
IEnumerable <InstanceSeedFile> testInstances,
AlgorithmTunerConfiguration configuration)
{
LoggingHelper.WriteLine(VerbosityLevel.Info, $"Tuning '{scenario.Command}' for PAR-{scenario.PenalizationFactor}.");
var runtimeTuner =
new AlgorithmTuner <RuntimeRunner, InstanceSeedFile, RuntimeResult, StandardRandomForestLearner <ReuseOldTreesStrategy>,
GenomePredictionForestModel <GenomePredictionTree>, ReuseOldTreesStrategy>(
targetAlgorithmFactory: new RuntimeRunnerFactory(scenario, parameterSpecification),
runEvaluator: new SortByPenalizedRuntime <InstanceSeedFile>(scenario.PenalizationFactor, configuration.CpuTimeout),
trainingInstances: trainingInstances,
parameterTree: parameterTree,
configuration: configuration,
genomeBuilder: new ParameterConfigurationSpaceGenomeBuilder(parameterTree, parameterSpecification, configuration));
runtimeTuner.SetTestInstances(testInstances);
return(runtimeTuner);
}
GenomePredictionForestModel <GenomePredictionTree>, ReuseOldTreesStrategy> BuildValueTuner(
ParameterTree parameterTree,
Scenario scenario,
ParameterConfigurationSpaceSpecification parameterSpecification,
IEnumerable <InstanceSeedFile> trainingInstances,
IEnumerable <InstanceSeedFile> testInstances,
AlgorithmTunerConfiguration configuration)
{
LoggingHelper.WriteLine(VerbosityLevel.Info, $"Tuning '{scenario.Command}' for minimum cost.");
var qualityTuner =
new AlgorithmTuner <QualityRunner, InstanceSeedFile, ContinuousResult, StandardRandomForestLearner <ReuseOldTreesStrategy>,
GenomePredictionForestModel <GenomePredictionTree>, ReuseOldTreesStrategy>(
targetAlgorithmFactory: new QualityRunnerFactory(scenario, parameterSpecification),
runEvaluator: new SortByValue <InstanceSeedFile>(ascending: true),
trainingInstances: trainingInstances,
parameterTree: parameterTree,
configuration: configuration,
genomeBuilder: new ParameterConfigurationSpaceGenomeBuilder(parameterTree, parameterSpecification, configuration));
qualityTuner.SetTestInstances(testInstances);
return(qualityTuner);
}
/// <summary>
/// Builds an instance of the <see cref="AlgorithmTuner{TTargetAlorithm,TInstance,TResult}" /> class for tuning Gurobi.
/// </summary>
/// <param name="tunerConfig">The <see cref="AlgorithmTunerConfiguration" /> to use.</param>
/// <param name="pathToTrainingInstanceFolder">The path to the folder containing training instances.</param>
/// <param name="pathToTestInstanceFolder">The path to test instance folder.</param>
/// <param name="gurobiConfigBuilder">The gurobi configuration builder.</param>
/// <returns>
/// The built instance.
/// </returns>
public static AlgorithmTuner <GurobiRunner, InstanceSeedFile, GurobiResult> BuildGurobiRunner(
AlgorithmTunerConfiguration tunerConfig,
string pathToTrainingInstanceFolder,
string pathToTestInstanceFolder,
GurobiRunnerConfiguration.GurobiRunnerConfigBuilder gurobiConfigBuilder)
{
var gurobiConfig = Program.BuildGurobiConfigAndCheckThreadCount(gurobiConfigBuilder, tunerConfig);
var tuner = new AlgorithmTuner <GurobiRunner, InstanceSeedFile, GurobiResult>(
targetAlgorithmFactory: new GurobiRunnerFactory(gurobiConfig, tunerConfig),
runEvaluator: new GurobiRunEvaluator(tunerConfig.CpuTimeout),
trainingInstances: InstanceSeedFile.CreateInstanceSeedFilesFromDirectory(
pathToTrainingInstanceFolder,
GurobiUtils.ListOfValidFileExtensions,
gurobiConfig.NumberOfSeeds,
gurobiConfig.RngSeed),
parameterTree: GurobiUtils.CreateParameterTree(),
configuration: tunerConfig,
customGrayBoxMethods: new GurobiGrayBoxMethods());
try
{
if (!string.IsNullOrWhiteSpace(pathToTestInstanceFolder))
{
var testInstances = InstanceSeedFile.CreateInstanceSeedFilesFromDirectory(
pathToTestInstanceFolder,
GurobiUtils.ListOfValidFileExtensions,
gurobiConfig.NumberOfSeeds,
gurobiConfig.RngSeed);
tuner.SetTestInstances(testInstances);
}
}
catch
{
}
return(tuner);
}