public ExecuteResult Execute(BuildResult buildResult, Benchmark benchmark, ILogger logger, IDiagnoser compositeDiagnoser = null) { var exePath = buildResult.ExecutablePath; var args = string.Empty; if (!File.Exists(exePath)) { return new ExecuteResult(false, new string[0]); } return Execute(benchmark, logger, exePath, null, args, compositeDiagnoser); }
public ExecuteResult Execute(BuildResult buildResult, Benchmark benchmark, ILogger logger, IResolver resolver, IDiagnoser compositeDiagnoser = null) { var exePath = buildResult.ArtifactsPaths.ExecutablePath; var args = compositeDiagnoser == null ? string.Empty : Engine.Signals.DiagnoserIsAttachedParam; if (!File.Exists(exePath)) { return new ExecuteResult(false, -1, new string[0], new string[0]); } return Execute(benchmark, logger, exePath, null, args, compositeDiagnoser, resolver); }
public BenchmarkReport( Benchmark benchmark, GenerateResult generateResult, BuildResult buildResult, IReadOnlyList<ExecuteResult> executeResults, IReadOnlyList<Measurement> allMeasurements) { Benchmark = benchmark; GenerateResult = generateResult; BuildResult = buildResult; ExecuteResults = executeResults ?? new ExecuteResult[0]; AllMeasurements = allMeasurements ?? new Measurement[0]; }
public ExecuteResult Execute(BuildResult buildResult, Benchmark benchmark, ILogger logger, IDiagnoser compositeDiagnoser = null) { if (consoleHandler == null) { consoleHandler = new ConsoleHandler(logger); Console.CancelKeyPress += consoleHandler.EventHandler; } var exeName = Path.Combine(buildResult.DirectoryPath, "Program.exe"); var args = string.Empty; if (File.Exists(exeName)) { try { var startInfo = CreateStartInfo(benchmark, exeName, args); using (var process = Process.Start(startInfo)) { if (process != null) { if (compositeDiagnoser != null) compositeDiagnoser.ProcessStarted(process); consoleHandler.SetProcess(process); var result = ExecuteImpl(process, exeName, benchmark, logger, compositeDiagnoser); if (compositeDiagnoser != null) compositeDiagnoser.ProcessStopped(process); return result; } } } finally { consoleHandler?.ClearProcess(); } } return new ExecuteResult(false, new string[0]); }
private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config) { var executeResults = new List<ExecuteResult>(); logger.WriteLineInfo("// *** Execute ***"); var launchCount = Math.Max(1, benchmark.Job.LaunchCount.IsAuto ? 2 : benchmark.Job.LaunchCount.Value); for (int processNumber = 0; processNumber < launchCount; processNumber++) { var printedProcessNumber = (benchmark.Job.LaunchCount.IsAuto && processNumber < 2) ? "" : " / " + launchCount.ToString(); logger.WriteLineInfo($"// Launch: {processNumber + 1}{printedProcessNumber}"); var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger); if (!executeResult.FoundExecutable) logger.WriteLineError("Executable not found"); executeResults.Add(executeResult); var measurements = executeResults .SelectMany(r => r.Data) .Select(line => Measurement.Parse(logger, line, 0)) .Where(r => r.IterationMode != IterationMode.Unknown). ToArray(); if (!measurements.Any()) { // Something went wrong during the benchmark, don't bother doing more runs logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!"); break; } if (benchmark.Job.LaunchCount.IsAuto && processNumber == 1) { var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median; var mainApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.MainTarget).Select(m => m.Nanoseconds)).Median; var percent = idleApprox / mainApprox * 100; launchCount = (int)Math.Round(Math.Max(2, 2 + (percent - 1) / 3)); // an empirical formula } } logger.WriteLine(); // Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results if (config.GetDiagnosers().Count() > 0) { logger.WriteLineInfo($"// Run, Diagnostic"); config.GetCompositeDiagnoser().Start(benchmark); var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, config.GetCompositeDiagnoser()); var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList(); var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns); config.GetCompositeDiagnoser().Stop(benchmark, report); if (!executeResult.FoundExecutable) logger.WriteLineError("Executable not found"); logger.WriteLine(); } return executeResults; }
public ExecuteResult Execute(BuildResult buildResult, Benchmark benchmark, ILogger logger, IResolver resolver, IDiagnoser diagnoser) { logger.WriteLine("Executing"); Done = true; return new ExecuteResult(true, 0, new string[0], new string[0]); }
private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config, IResolver resolver) { var executeResults = new List<ExecuteResult>(); logger.WriteLineInfo("// *** Execute ***"); bool analyzeRunToRunVariance = benchmark.Job.ResolveValue(AccuracyMode.AnalyzeLaunchVarianceCharacteristic, resolver); bool autoLaunchCount = !benchmark.Job.HasValue(RunMode.LaunchCountCharacteristic); int defaultValue = analyzeRunToRunVariance ? 2 : 1; int launchCount = Math.Max( 1, autoLaunchCount ? defaultValue: benchmark.Job.Run.LaunchCount); for (int launchIndex = 0; launchIndex < launchCount; launchIndex++) { string printedLaunchCount = (analyzeRunToRunVariance && autoLaunchCount && launchIndex < 2) ? "" : " / " + launchCount; logger.WriteLineInfo($"// Launch: {launchIndex + 1}{printedLaunchCount}"); var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, resolver); if (!executeResult.FoundExecutable) logger.WriteLineError("Executable not found"); if (executeResult.ExitCode != 0) logger.WriteLineError("ExitCode != 0"); executeResults.Add(executeResult); var measurements = executeResults .SelectMany(r => r.Data) .Select(line => Measurement.Parse(logger, line, 0)) .Where(r => r.IterationMode != IterationMode.Unknown). ToArray(); if (!measurements.Any()) { // Something went wrong during the benchmark, don't bother doing more runs logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!"); break; } if (autoLaunchCount && launchIndex == 1 && analyzeRunToRunVariance) { // TODO: improve this logic var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median; var mainApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.MainTarget).Select(m => m.Nanoseconds)).Median; var percent = idleApprox / mainApprox * 100; launchCount = (int)Math.Round(Math.Max(2, 2 + (percent - 1) / 3)); // an empirical formula } } logger.WriteLine(); // Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results if (config.GetDiagnosers().Any()) { logger.WriteLineInfo("// Run, Diagnostic"); var compositeDiagnoser = config.GetCompositeDiagnoser(); var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, resolver, compositeDiagnoser); var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList(); var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns); compositeDiagnoser.ProcessResults(benchmark, report); if (!executeResult.FoundExecutable) logger.WriteLineError("Executable not found"); logger.WriteLine(); } return executeResults; }