public void ResultsAreLoggedCorrectly() { var results = new SortedDictionary <InstanceFile, RuntimeResult>( Comparer <InstanceFile> .Create((file1, file2) => string.Compare(file1.ToString(), file2.ToString()))) { { new InstanceFile("a"), new RuntimeResult(TimeSpan.FromMilliseconds(42)) }, { new InstanceFile("foo/bar"), ResultBase <RuntimeResult> .CreateCancelledResult(TimeSpan.FromMilliseconds(11)) }, }; var resultMessage = new GenomeResults <InstanceFile, RuntimeResult>(results); this._writer.LogFinishedGeneration(0, 1, this._genomeBuilder.CreateRandomGenome(0), resultMessage); // Check results were logged correclty. // Ignore first six lines as they do not describe the results. var relevantLinesInFile = File.ReadLines(LogWriterTest.LogFilePath).Skip(6).ToList(); Assert.True( relevantLinesInFile.Contains( FormattableString.Invariant($"\ta:\t{TimeSpan.FromMilliseconds(42):G}")), "Expected first result to be printed."); Assert.True( relevantLinesInFile.Contains( FormattableString.Invariant($"\tfoo/bar:\tCancelled after {TimeSpan.FromMilliseconds(11):G}")), "Expected second result to be printed."); }
public void CancelledResultReturnsCorrectRuntime() { var runtime = TimeSpan.FromMilliseconds(42); var result = ResultBase <TestResult> .CreateCancelledResult(runtime); Assert.Equal(runtime, result.Runtime); }
/// <summary> /// Starts an evaluation on the given instance. /// </summary> /// <param name="instance">The instance to run the configured target algorithm on.</param> private void StartEvaluation(TInstance instance) { LoggingHelper.WriteLine( VerbosityLevel.Trace, $"Starting target algorithm run with configuration {this._currentGenome.ToFilteredGeneString(this._parameterTree)} on instance {instance}."); // Set a CPU timeout. this._evaluationCancellationTokenSource = new CancellationTokenSource(this._configuration.CpuTimeout); // Start the target algorithm run. this._configuredTargetAlgorithm.Run(instance, this._evaluationCancellationTokenSource.Token) .ContinueWith <object>( finishedTask => { if (finishedTask.IsCanceled) { var cancellationResult = ResultBase <TResult> .CreateCancelledResult(this._configuration.CpuTimeout); return(new ResultMessage <TInstance, TResult>(this._currentGenome, instance, cancellationResult)); } if (finishedTask.IsFaulted) { return(new Faulted <TInstance>(this._currentGenome, instance, finishedTask.Exception)); } return(new ResultMessage <TInstance, TResult>(this._currentGenome, instance, finishedTask.Result)); }) .PipeTo(this.Self); }
/// <summary> /// Extracts the run statistics. /// </summary> /// <param name="consoleOutput">The console output.</param> /// <returns>The run statistics.</returns> public RuntimeResult ExtractRunStatistics(string consoleOutput) { var satMatches = new Regex(@"s SATISFIABLE").Match(consoleOutput); var runtimeMatches = new Regex(@"c[ \t]+\d+(?:\.\d+)? seconds,").Match(consoleOutput); if (!satMatches.Success || !runtimeMatches.Success) { return(ResultBase <RuntimeResult> .CreateCancelledResult(this._tunerConfig.CpuTimeout)); } var extractedRuntime = new Regex(@"\d+(?:\.\d+)?").Match(runtimeMatches.Value); if (!extractedRuntime.Success) { return(ResultBase <RuntimeResult> .CreateCancelledResult(this._tunerConfig.CpuTimeout)); } var extractedRuntimeSeconds = double.Parse(extractedRuntime.Value, CultureInfo.InvariantCulture); var reportedRuntime = TimeSpan.FromSeconds(extractedRuntimeSeconds); if (reportedRuntime <= this._tunerConfig.CpuTimeout) { return(new RuntimeResult(reportedRuntime)); } return(ResultBase <RuntimeResult> .CreateCancelledResult(this._tunerConfig.CpuTimeout)); }
/// <summary> /// Starts the desired single post tuning run. /// </summary> internal void ExecuteSinglePostTuningRun() { this._evaluationCancellationTokenSource = new CancellationTokenSource(this._tunerConfiguration.CpuTimeout); using var runnerTask = this._configuredTargetAlgorithm.Run(this._instance, this._evaluationCancellationTokenSource.Token); try { // ReSharper disable once MethodSupportsCancellation runnerTask.Wait(); } catch (Exception exception) { LoggingHelper.WriteLine(VerbosityLevel.Debug, exception.ToString()); } if (runnerTask.IsFaulted) { throw new TaskCanceledException($"{runnerTask.Exception?.Message ?? "Undefined exception!"}"); } var result = runnerTask.IsCanceled ? ResultBase <TResult> .CreateCancelledResult(this._tunerConfiguration.CpuTimeout) : runnerTask.Result; this._grayBoxHandler.WriteListOfDataRecordsToFile(result); this.DisposeTargetAlgorithmAndGrayBoxHandler(); }
public void GetMetricRepresentationAddsPenalizationForCancelledRun() { var parSorter = new SortByRuntime(factorPar: 7); var result = ResultBase <RuntimeResult> .CreateCancelledResult(TimeSpan.FromSeconds(3.1)); Assert.Equal( 21.7, parSorter.GetMetricRepresentation(result), 8); }
/// <summary> /// Creates a <see cref="RuntimeResult"/>. /// </summary> /// <param name="originalRunResult">Results of the target algorithm run as reported by its output.</param> /// <returns>The created result.</returns> protected override RuntimeResult CreateRunResult(OriginalRunResult originalRunResult) { // EPMs might report SAT instead of TIMEOUT if the runtime is exactly the cutoff. // The specified timeout handling actually mirrors the one of SMAC 2.08 (without SMAC's own cutoff). if (originalRunResult.Status == RunStatus.Timeout || originalRunResult.Status == RunStatus.Crashed || originalRunResult.Runtime >= this.Scenario.CutoffTime) { return(ResultBase <RuntimeResult> .CreateCancelledResult(this.Scenario.CutoffTime)); } return(new RuntimeResult(originalRunResult.Runtime)); }
/// <summary> /// Starts an evaluation of the current genome instance pair. /// </summary> private void StartEvaluation() { LoggingHelper.WriteLine( VerbosityLevel.Trace, $"Starting target algorithm run with configuration {this._currentEvaluation.GenomeInstancePair.Genome.ToFilteredGeneString(this._parameterTree)} on instance {this._currentEvaluation.GenomeInstancePair.Instance}."); if (this._configuration.EnableDataRecording) { if (this._currentEvaluation.UseGrayBoxInEvaluation) { this.UpdateGrayBoxRandomForest(); } this.ConfigureGrayBoxHandler(); } // Set the CPU timeout. this._evaluationCancellationTokenSource = new CancellationTokenSource(this._configuration.CpuTimeout); // Start the target algorithm run. this._configuredTargetAlgorithm.Run(this._currentEvaluation.GenomeInstancePair.Instance, this._evaluationCancellationTokenSource.Token) .ContinueWith <object>( finishedTask => { if (finishedTask.IsFaulted) { return(new Faulted <TInstance>(this._currentEvaluation.GenomeInstancePair, finishedTask.Exception)); } var result = finishedTask.IsCanceled ? ResultBase <TResult> .CreateCancelledResult(this._configuration.CpuTimeout) : finishedTask.Result; if (this._configuration.EnableDataRecording) { this._grayBoxHandler.WriteListOfDataRecordsToFile(result); } return(new EvaluationResult <TInstance, TResult>(this._currentEvaluation.GenomeInstancePair, result)); }) .PipeTo(this.Self, sender: this.Sender); }
/// <summary> /// Creates the runtime result. /// </summary> /// <param name="output">The output.</param> /// <param name="timeout">The timeout.</param> /// <returns>The result.</returns> public static RuntimeResult CreateRuntimeResult(string output, TimeSpan timeout) { // If the process was not cancelled, first check the output for CPU time. if (!SapsRunner.CpuTimeMatcher.IsMatch(output)) { return(ResultBase <RuntimeResult> .CreateCancelledResult(timeout)); } var printedCpuTime = SapsRunner.CpuTimeMatcher.Match(output); var cpuTimeSeconds = double.Parse(printedCpuTime.Groups[1].Value, CultureInfo.InvariantCulture); var isTimeout = cpuTimeSeconds + 1e-6 >= timeout.TotalSeconds; // Finally return CPU time as result. if (!isTimeout) { return(new RuntimeResult(TimeSpan.FromSeconds(cpuTimeSeconds))); } else { return(ResultBase <RuntimeResult> .CreateCancelledResult(timeout)); } }
/// <summary> /// Initializes a new instance of the <see cref="LogWriterTest"/> class. /// </summary> public LogWriterTest() { Randomizer.Reset(); Randomizer.Configure(0); this._parameterTree = LogWriterTest.CreateParameterTree(); this._configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetGenerations(24) .SetGoalGeneration(17) .SetLogFilePath(LogWriterTest.LogFilePath) .Build(1); this._testGenomeResults = new GenomeResults <InstanceFile, RuntimeResult>( new ImmutableGenome(new GenomeBuilder(this._parameterTree, this._configuration).CreateRandomGenome(4)), new SortedDictionary <InstanceFile, RuntimeResult>( Comparer <InstanceFile> .Create((file1, file2) => string.CompareOrdinal(file1.ToString(), file2.ToString()))) { { new InstanceFile("a"), new RuntimeResult(TimeSpan.FromMilliseconds(42)) }, { new InstanceFile("foo/bar"), ResultBase <RuntimeResult> .CreateCancelledResult(TimeSpan.FromMilliseconds(11)) }, }); this._writer = new LogWriter <InstanceFile, RuntimeResult>(this._parameterTree, this._configuration); }
public void CreateCancelledResultThrowsIfTargetAlgorithmStatusIsRunningOrFinished(TargetAlgorithmStatus targetAlgorithmStatus) { Assert.Throws <ArgumentOutOfRangeException>( () => { ResultBase <TestResult> .CreateCancelledResult(TimeSpan.FromSeconds(30), targetAlgorithmStatus); }); }
public void CancelledResultIsMarkedAsCancelled() { var result = ResultBase <TestResult> .CreateCancelledResult(TimeSpan.Zero); Assert.True(result.IsCancelled, "Cancelled result not marked as cancelled."); }