private BenchmarkReport Run(IBenchmarkLogger logger, Benchmark benchmark, IList <string> importantPropertyNames, BenchmarkParameters parameters = null)
{
var toolchain = Plugins.CreateToolchain(benchmark, logger);
logger.WriteLineHeader("// **************************");
logger.WriteLineHeader("// Benchmark: " + benchmark.Description);
var generateResult = Generate(logger, toolchain);
if (!generateResult.IsGenerateSuccess)
{
return(BenchmarkReport.CreateEmpty(benchmark, parameters));
}
var buildResult = Build(logger, toolchain, generateResult);
if (!buildResult.IsBuildSuccess)
{
return(BenchmarkReport.CreateEmpty(benchmark, parameters));
}
var runReports = Execute(logger, benchmark, importantPropertyNames, parameters, toolchain, buildResult);
return(new BenchmarkReport(benchmark, runReports, parameters));
}
private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
logger.WriteLineInfo($"// {benchmark.Description}");
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
}
logger.NewLine();
}
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
BenchmarkMarkdownExporter.Default.Export(reports, logger);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// *** Warnings *** ");
foreach (var warning in warnings)
logger.WriteLineError($"{warning.Message}");
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return reports;
}
public override void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null)
{
foreach (var report in reports)
{
var runs = report.Runs;
var modes = runs.Select(it => it.IterationMode).Distinct();
logger.WriteLineHeader($"*** {report.Benchmark.Description} ***");
logger.WriteLineHeader("* Raw *");
foreach (var run in runs)
logger.WriteLineResult(run.ToStr());
foreach (var mode in modes)
{
logger.NewLine();
logger.WriteLineHeader($"* Statistics for {mode}");
logger.WriteLineStatistic(runs.Where(it => it.IterationMode == mode).GetStats().ToTimeStr());
}
}
}
private List <BenchmarkReport> Run(List <Benchmark> benchmarks, IBenchmarkLogger logger)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
{
logger.WriteLineInfo($"// {benchmark.Description}");
}
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List <BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
}
logger.NewLine();
}
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
BenchmarkMarkdownExporter.Default.Export(reports, logger);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// *** Warnings *** ");
foreach (var warning in warnings)
{
logger.WriteLineError($"{warning.Message}");
}
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return(reports);
}
private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger, string competitionName)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
logger.WriteLineInfo($"// {benchmark.Description}");
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var globalStopwatch = Stopwatch.StartNew();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.GetTargetRuns().Any())
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.GetTargetRuns().Any())
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
}
}
logger.NewLine();
}
globalStopwatch.Stop();
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
logger.WriteLineHeader("// * Export *");
var files = Plugins.CompositeExporter.ExportToFile(reports, competitionName, Plugins.ResultExtenders);
foreach (var file in files)
logger.WriteLineInfo($" {file}");
logger.NewLine();
logger.WriteLineHeader("// * Detailed results *");
foreach (var report in reports)
{
logger.WriteLineInfo(report.Benchmark.Description);
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
logger.NewLine();
}
logger.WriteLineStatistic($"Total time: {globalStopwatch.Elapsed.TotalHours:00}:{globalStopwatch.Elapsed:mm\\:ss}");
logger.NewLine();
logger.WriteLineHeader("// * Summary *");
BenchmarkMarkdownExporter.Default.Export(reports, logger, Plugins.ResultExtenders);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// * Warnings * ");
foreach (var warning in warnings)
logger.WriteLineError($"{warning.Message}");
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return reports;
}
private BenchmarkReport Run(IBenchmarkLogger logger, Benchmark benchmark, IList<string> importantPropertyNames, BenchmarkParameters parameters = null)
{
var toolchain = Plugins.CreateToolchain(benchmark, logger);
logger.WriteLineHeader("// **************************");
logger.WriteLineHeader("// Benchmark: " + benchmark.Description);
var generateResult = Generate(logger, toolchain);
if (!generateResult.IsGenerateSuccess)
return BenchmarkReport.CreateEmpty(benchmark, parameters);
var buildResult = Build(logger, toolchain, generateResult);
if (!buildResult.IsBuildSuccess)
return BenchmarkReport.CreateEmpty(benchmark, parameters);
var runReports = Execute(logger, benchmark, importantPropertyNames, parameters, toolchain, buildResult);
return new BenchmarkReport(benchmark, runReports, EnvironmentInfo.GetCurrentInfo(), parameters);
}