public void ResultsAreLoggedCorrectly()
{
var results = new SortedDictionary <InstanceFile, RuntimeResult>(
Comparer <InstanceFile> .Create((file1, file2) => string.Compare(file1.ToString(), file2.ToString())))
{
{ new InstanceFile("a"), new RuntimeResult(TimeSpan.FromMilliseconds(42)) },
{ new InstanceFile("foo/bar"), ResultBase <RuntimeResult> .CreateCancelledResult(TimeSpan.FromMilliseconds(11)) },
};
var resultMessage = new GenomeResults <InstanceFile, RuntimeResult>(results);
this._writer.LogFinishedGeneration(0, 1, this._genomeBuilder.CreateRandomGenome(0), resultMessage);
// Check results were logged correclty.
// Ignore first six lines as they do not describe the results.
var relevantLinesInFile = File.ReadLines(LogWriterTest.LogFilePath).Skip(6).ToList();
Assert.True(
relevantLinesInFile.Contains(
FormattableString.Invariant($"\ta:\t{TimeSpan.FromMilliseconds(42):G}")),
"Expected first result to be printed.");
Assert.True(
relevantLinesInFile.Contains(
FormattableString.Invariant($"\tfoo/bar:\tCancelled after {TimeSpan.FromMilliseconds(11):G}")),
"Expected second result to be printed.");
}
public void CancelledResultReturnsCorrectRuntime()
{
var runtime = TimeSpan.FromMilliseconds(42);
var result = ResultBase <TestResult> .CreateCancelledResult(runtime);
Assert.Equal(runtime, result.Runtime);
}
/// <summary>
/// Starts an evaluation on the given instance.
/// </summary>
/// <param name="instance">The instance to run the configured target algorithm on.</param>
private void StartEvaluation(TInstance instance)
{
LoggingHelper.WriteLine(
VerbosityLevel.Trace,
$"Starting target algorithm run with configuration {this._currentGenome.ToFilteredGeneString(this._parameterTree)} on instance {instance}.");
// Set a CPU timeout.
this._evaluationCancellationTokenSource = new CancellationTokenSource(this._configuration.CpuTimeout);
// Start the target algorithm run.
this._configuredTargetAlgorithm.Run(instance, this._evaluationCancellationTokenSource.Token)
.ContinueWith <object>(
finishedTask =>
{
if (finishedTask.IsCanceled)
{
var cancellationResult = ResultBase <TResult> .CreateCancelledResult(this._configuration.CpuTimeout);
return(new ResultMessage <TInstance, TResult>(this._currentGenome, instance, cancellationResult));
}
if (finishedTask.IsFaulted)
{
return(new Faulted <TInstance>(this._currentGenome, instance, finishedTask.Exception));
}
return(new ResultMessage <TInstance, TResult>(this._currentGenome, instance, finishedTask.Result));
})
.PipeTo(this.Self);
}
/// <summary>
/// Extracts the run statistics.
/// </summary>
/// <param name="consoleOutput">The console output.</param>
/// <returns>The run statistics.</returns>
public RuntimeResult ExtractRunStatistics(string consoleOutput)
{
var satMatches = new Regex(@"s SATISFIABLE").Match(consoleOutput);
var runtimeMatches = new Regex(@"c[ \t]+\d+(?:\.\d+)? seconds,").Match(consoleOutput);
if (!satMatches.Success || !runtimeMatches.Success)
{
return(ResultBase <RuntimeResult> .CreateCancelledResult(this._tunerConfig.CpuTimeout));
}
var extractedRuntime = new Regex(@"\d+(?:\.\d+)?").Match(runtimeMatches.Value);
if (!extractedRuntime.Success)
{
return(ResultBase <RuntimeResult> .CreateCancelledResult(this._tunerConfig.CpuTimeout));
}
var extractedRuntimeSeconds = double.Parse(extractedRuntime.Value, CultureInfo.InvariantCulture);
var reportedRuntime = TimeSpan.FromSeconds(extractedRuntimeSeconds);
if (reportedRuntime <= this._tunerConfig.CpuTimeout)
{
return(new RuntimeResult(reportedRuntime));
}
return(ResultBase <RuntimeResult> .CreateCancelledResult(this._tunerConfig.CpuTimeout));
}
/// <summary>
/// Starts the desired single post tuning run.
/// </summary>
internal void ExecuteSinglePostTuningRun()
{
this._evaluationCancellationTokenSource = new CancellationTokenSource(this._tunerConfiguration.CpuTimeout);
using var runnerTask = this._configuredTargetAlgorithm.Run(this._instance, this._evaluationCancellationTokenSource.Token);
try
{
// ReSharper disable once MethodSupportsCancellation
runnerTask.Wait();
}
catch (Exception exception)
{
LoggingHelper.WriteLine(VerbosityLevel.Debug, exception.ToString());
}
if (runnerTask.IsFaulted)
{
throw new TaskCanceledException($"{runnerTask.Exception?.Message ?? "Undefined exception!"}");
}
var result = runnerTask.IsCanceled ? ResultBase <TResult> .CreateCancelledResult(this._tunerConfiguration.CpuTimeout) : runnerTask.Result;
this._grayBoxHandler.WriteListOfDataRecordsToFile(result);
this.DisposeTargetAlgorithmAndGrayBoxHandler();
}
public void GetMetricRepresentationAddsPenalizationForCancelledRun()
{
var parSorter = new SortByRuntime(factorPar: 7);
var result = ResultBase <RuntimeResult> .CreateCancelledResult(TimeSpan.FromSeconds(3.1));
Assert.Equal(
21.7,
parSorter.GetMetricRepresentation(result),
8);
}
/// <summary>
/// Creates a <see cref="RuntimeResult"/>.
/// </summary>
/// <param name="originalRunResult">Results of the target algorithm run as reported by its output.</param>
/// <returns>The created result.</returns>
protected override RuntimeResult CreateRunResult(OriginalRunResult originalRunResult)
{
// EPMs might report SAT instead of TIMEOUT if the runtime is exactly the cutoff.
// The specified timeout handling actually mirrors the one of SMAC 2.08 (without SMAC's own cutoff).
if (originalRunResult.Status == RunStatus.Timeout ||
originalRunResult.Status == RunStatus.Crashed ||
originalRunResult.Runtime >= this.Scenario.CutoffTime)
{
return(ResultBase <RuntimeResult> .CreateCancelledResult(this.Scenario.CutoffTime));
}
return(new RuntimeResult(originalRunResult.Runtime));
}
/// <summary>
/// Starts an evaluation of the current genome instance pair.
/// </summary>
private void StartEvaluation()
{
LoggingHelper.WriteLine(
VerbosityLevel.Trace,
$"Starting target algorithm run with configuration {this._currentEvaluation.GenomeInstancePair.Genome.ToFilteredGeneString(this._parameterTree)} on instance {this._currentEvaluation.GenomeInstancePair.Instance}.");
if (this._configuration.EnableDataRecording)
{
if (this._currentEvaluation.UseGrayBoxInEvaluation)
{
this.UpdateGrayBoxRandomForest();
}
this.ConfigureGrayBoxHandler();
}
// Set the CPU timeout.
this._evaluationCancellationTokenSource = new CancellationTokenSource(this._configuration.CpuTimeout);
// Start the target algorithm run.
this._configuredTargetAlgorithm.Run(this._currentEvaluation.GenomeInstancePair.Instance, this._evaluationCancellationTokenSource.Token)
.ContinueWith <object>(
finishedTask =>
{
if (finishedTask.IsFaulted)
{
return(new Faulted <TInstance>(this._currentEvaluation.GenomeInstancePair, finishedTask.Exception));
}
var result = finishedTask.IsCanceled
? ResultBase <TResult> .CreateCancelledResult(this._configuration.CpuTimeout)
: finishedTask.Result;
if (this._configuration.EnableDataRecording)
{
this._grayBoxHandler.WriteListOfDataRecordsToFile(result);
}
return(new EvaluationResult <TInstance, TResult>(this._currentEvaluation.GenomeInstancePair, result));
})
.PipeTo(this.Self, sender: this.Sender);
}
/// <summary>
/// Creates the runtime result.
/// </summary>
/// <param name="output">The output.</param>
/// <param name="timeout">The timeout.</param>
/// <returns>The result.</returns>
public static RuntimeResult CreateRuntimeResult(string output, TimeSpan timeout)
{
// If the process was not cancelled, first check the output for CPU time.
if (!SapsRunner.CpuTimeMatcher.IsMatch(output))
{
return(ResultBase <RuntimeResult> .CreateCancelledResult(timeout));
}
var printedCpuTime = SapsRunner.CpuTimeMatcher.Match(output);
var cpuTimeSeconds = double.Parse(printedCpuTime.Groups[1].Value, CultureInfo.InvariantCulture);
var isTimeout = cpuTimeSeconds + 1e-6 >= timeout.TotalSeconds;
// Finally return CPU time as result.
if (!isTimeout)
{
return(new RuntimeResult(TimeSpan.FromSeconds(cpuTimeSeconds)));
}
else
{
return(ResultBase <RuntimeResult> .CreateCancelledResult(timeout));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="LogWriterTest"/> class.
/// </summary>
public LogWriterTest()
{
Randomizer.Reset();
Randomizer.Configure(0);
this._parameterTree = LogWriterTest.CreateParameterTree();
this._configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetGenerations(24)
.SetGoalGeneration(17)
.SetLogFilePath(LogWriterTest.LogFilePath)
.Build(1);
this._testGenomeResults = new GenomeResults <InstanceFile, RuntimeResult>(
new ImmutableGenome(new GenomeBuilder(this._parameterTree, this._configuration).CreateRandomGenome(4)),
new SortedDictionary <InstanceFile, RuntimeResult>(
Comparer <InstanceFile> .Create((file1, file2) => string.CompareOrdinal(file1.ToString(), file2.ToString())))
{
{ new InstanceFile("a"), new RuntimeResult(TimeSpan.FromMilliseconds(42)) },
{ new InstanceFile("foo/bar"), ResultBase <RuntimeResult> .CreateCancelledResult(TimeSpan.FromMilliseconds(11)) },
});
this._writer = new LogWriter <InstanceFile, RuntimeResult>(this._parameterTree, this._configuration);
}
public void CreateCancelledResultThrowsIfTargetAlgorithmStatusIsRunningOrFinished(TargetAlgorithmStatus targetAlgorithmStatus)
{
Assert.Throws <ArgumentOutOfRangeException>(
() => { ResultBase <TestResult> .CreateCancelledResult(TimeSpan.FromSeconds(30), targetAlgorithmStatus); });
}
public void CancelledResultIsMarkedAsCancelled()
{
var result = ResultBase <TestResult> .CreateCancelledResult(TimeSpan.Zero);
Assert.True(result.IsCancelled, "Cancelled result not marked as cancelled.");
}
