public ExecuteResult Execute(BuildResult buildResult, Benchmark benchmark, ILogger logger, IDiagnoser compositeDiagnoser = null)
{
var exePath = buildResult.ExecutablePath;
var args = string.Empty;
if (!File.Exists(exePath))
{
return new ExecuteResult(false, new string[0]);
}
return Execute(benchmark, logger, exePath, null, args, compositeDiagnoser);
}
public ExecuteResult Execute(BuildResult buildResult, Benchmark benchmark, ILogger logger, IResolver resolver, IDiagnoser compositeDiagnoser = null)
{
var exePath = buildResult.ArtifactsPaths.ExecutablePath;
var args = compositeDiagnoser == null ? string.Empty : Engine.Signals.DiagnoserIsAttachedParam;
if (!File.Exists(exePath))
{
return new ExecuteResult(false, -1, new string[0], new string[0]);
}
return Execute(benchmark, logger, exePath, null, args, compositeDiagnoser, resolver);
}
public BenchmarkReport(
Benchmark benchmark,
GenerateResult generateResult,
BuildResult buildResult,
IReadOnlyList<ExecuteResult> executeResults,
IReadOnlyList<Measurement> allMeasurements)
{
Benchmark = benchmark;
GenerateResult = generateResult;
BuildResult = buildResult;
ExecuteResults = executeResults ?? new ExecuteResult[0];
AllMeasurements = allMeasurements ?? new Measurement[0];
}
public ExecuteResult Execute(BuildResult buildResult, Benchmark benchmark, ILogger logger, IDiagnoser compositeDiagnoser = null)
{
if (consoleHandler == null)
{
consoleHandler = new ConsoleHandler(logger);
Console.CancelKeyPress += consoleHandler.EventHandler;
}
var exeName = Path.Combine(buildResult.DirectoryPath, "Program.exe");
var args = string.Empty;
if (File.Exists(exeName))
{
try
{
var startInfo = CreateStartInfo(benchmark, exeName, args);
using (var process = Process.Start(startInfo))
{
if (process != null)
{
if (compositeDiagnoser != null)
compositeDiagnoser.ProcessStarted(process);
consoleHandler.SetProcess(process);
var result = ExecuteImpl(process, exeName, benchmark, logger, compositeDiagnoser);
if (compositeDiagnoser != null)
compositeDiagnoser.ProcessStopped(process);
return result;
}
}
}
finally
{
consoleHandler?.ClearProcess();
}
}
return new ExecuteResult(false, new string[0]);
}
private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config)
{
var executeResults = new List<ExecuteResult>();
logger.WriteLineInfo("// *** Execute ***");
var launchCount = Math.Max(1, benchmark.Job.LaunchCount.IsAuto ? 2 : benchmark.Job.LaunchCount.Value);
for (int processNumber = 0; processNumber < launchCount; processNumber++)
{
var printedProcessNumber = (benchmark.Job.LaunchCount.IsAuto && processNumber < 2) ? "" : " / " + launchCount.ToString();
logger.WriteLineInfo($"// Launch: {processNumber + 1}{printedProcessNumber}");
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger);
if (!executeResult.FoundExecutable)
logger.WriteLineError("Executable not found");
executeResults.Add(executeResult);
var measurements = executeResults
.SelectMany(r => r.Data)
.Select(line => Measurement.Parse(logger, line, 0))
.Where(r => r.IterationMode != IterationMode.Unknown).
ToArray();
if (!measurements.Any())
{
// Something went wrong during the benchmark, don't bother doing more runs
logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!");
break;
}
if (benchmark.Job.LaunchCount.IsAuto && processNumber == 1)
{
var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median;
var mainApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.MainTarget).Select(m => m.Nanoseconds)).Median;
var percent = idleApprox / mainApprox * 100;
launchCount = (int)Math.Round(Math.Max(2, 2 + (percent - 1) / 3)); // an empirical formula
}
}
logger.WriteLine();
// Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results
if (config.GetDiagnosers().Count() > 0)
{
logger.WriteLineInfo($"// Run, Diagnostic");
config.GetCompositeDiagnoser().Start(benchmark);
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, config.GetCompositeDiagnoser());
var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList();
var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns);
config.GetCompositeDiagnoser().Stop(benchmark, report);
if (!executeResult.FoundExecutable)
logger.WriteLineError("Executable not found");
logger.WriteLine();
}
return executeResults;
}
public ExecuteResult Execute(BuildResult buildResult, Benchmark benchmark, ILogger logger, IResolver resolver, IDiagnoser diagnoser)
{
logger.WriteLine("Executing");
Done = true;
return new ExecuteResult(true, 0, new string[0], new string[0]);
}
private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config, IResolver resolver)
{
var executeResults = new List<ExecuteResult>();
logger.WriteLineInfo("// *** Execute ***");
bool analyzeRunToRunVariance = benchmark.Job.ResolveValue(AccuracyMode.AnalyzeLaunchVarianceCharacteristic, resolver);
bool autoLaunchCount = !benchmark.Job.HasValue(RunMode.LaunchCountCharacteristic);
int defaultValue = analyzeRunToRunVariance ? 2 : 1;
int launchCount = Math.Max(
1,
autoLaunchCount ? defaultValue: benchmark.Job.Run.LaunchCount);
for (int launchIndex = 0; launchIndex < launchCount; launchIndex++)
{
string printedLaunchCount = (analyzeRunToRunVariance &&
autoLaunchCount &&
launchIndex < 2)
? ""
: " / " + launchCount;
logger.WriteLineInfo($"// Launch: {launchIndex + 1}{printedLaunchCount}");
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, resolver);
if (!executeResult.FoundExecutable)
logger.WriteLineError("Executable not found");
if (executeResult.ExitCode != 0)
logger.WriteLineError("ExitCode != 0");
executeResults.Add(executeResult);
var measurements = executeResults
.SelectMany(r => r.Data)
.Select(line => Measurement.Parse(logger, line, 0))
.Where(r => r.IterationMode != IterationMode.Unknown).
ToArray();
if (!measurements.Any())
{
// Something went wrong during the benchmark, don't bother doing more runs
logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!");
break;
}
if (autoLaunchCount && launchIndex == 1 && analyzeRunToRunVariance)
{
// TODO: improve this logic
var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median;
var mainApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.MainTarget).Select(m => m.Nanoseconds)).Median;
var percent = idleApprox / mainApprox * 100;
launchCount = (int)Math.Round(Math.Max(2, 2 + (percent - 1) / 3)); // an empirical formula
}
}
logger.WriteLine();
// Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results
if (config.GetDiagnosers().Any())
{
logger.WriteLineInfo("// Run, Diagnostic");
var compositeDiagnoser = config.GetCompositeDiagnoser();
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, resolver, compositeDiagnoser);
var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList();
var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns);
compositeDiagnoser.ProcessResults(benchmark, report);
if (!executeResult.FoundExecutable)
logger.WriteLineError("Executable not found");
logger.WriteLine();
}
return executeResults;
}
