private void RunCompetitions(string[] args)
{
for (int i = 0; i < Competitions.Length; i++)
{
var competition = Competitions[i];
if (args.Any(arg => competition.Name.ToLower().StartsWith(arg.ToLower())) || args.Contains("#" + i) || args.Contains("*"))
{
logger.WriteLineHeader("Target competition: " + competition.Name);
List<BenchmarkReport> reports;
using (var logStreamWriter = new StreamWriter(competition.Name + ".log"))
{
var loggers = new IBenchmarkLogger[] { new BenchmarkConsoleLogger(), new BenchmarkStreamLogger(logStreamWriter) };
var runner = new BenchmarkRunner(loggers);
reports = runner.RunCompetition(Activator.CreateInstance(competition), BenchmarkSettings.Parse(args)).ToList();
}
MarkdownReportExporter.Default.SaveToFile(reports, competition.Name + "-report.md");
CsvReportExporter.Default.SaveToFile(reports, competition.Name + "-report.csv");
logger.NewLine();
}
}
if (args.Length > 0 && (args[0].StartsWith("http://") || args[0].StartsWith("https://")))
{
var url = args[0];
Uri uri = new Uri(url);
var name = uri.IsFile ? Path.GetFileName(uri.LocalPath) : "URL";
using (var logStreamWriter = new StreamWriter(name + ".log"))
{
var loggers = new IBenchmarkLogger[] { new BenchmarkConsoleLogger(), new BenchmarkStreamLogger(logStreamWriter) };
var runner = new BenchmarkRunner(loggers);
runner.RunUrl(url, BenchmarkSettings.Parse(args));
}
}
}
public override void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null)
{
if(useCodeBlocks)
logger.WriteLine($"```{codeBlocksSyntax}");
logger = new BenchmarkLoggerWithPrefix(logger, prefix);
logger.WriteLineInfo(EnvironmentInfo.GetCurrentInfo().ToFormattedString("Host"));
logger.NewLine();
var table = BenchmarkExporterHelper.BuildTable(reports, resultExtenders);
// If we have Benchmarks with ParametersSets, force the "Method" columns to be displayed, otherwise it doesn't make as much sense
var columnsToAlwaysShow =
(reports.Any(r => r.Benchmark.Task.ParametersSets != null) ? new[] { "Method" } : new string[0]).
Concat((resultExtenders ?? new IBenchmarkResultExtender[0]).Select(e => e.ColumnName)).ToArray();
PrintTable(table, logger, columnsToAlwaysShow);
// TODO: move this logic to an analyser
var benchmarksWithTroubles = reports.Where(r => !r.GetTargetRuns().Any()).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
}
private void PrintTable(List<string[]> table, IBenchmarkLogger logger)
{
int rowCount = table.Count, colCount = table[0].Length;
int[] widths = new int[colCount];
bool[] areSame = new bool[colCount];
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
areSame[colIndex] = rowCount > 2 && colIndex < colCount - 3;
for (int rowIndex = 0; rowIndex < rowCount; rowIndex++)
{
widths[colIndex] = Math.Max(widths[colIndex], table[rowIndex][colIndex].Length + 1);
if (rowIndex > 1 && table[rowIndex][colIndex] != table[1][colIndex])
areSame[colIndex] = false;
}
}
if (areSame.Any(s => s))
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
if (areSame[colIndex])
logger.WriteInfo($"{table[0][colIndex]}={table[1][colIndex]} ");
logger.NewLine();
logger.WriteLineInfo("```");
logger.NewLine();
}
table.Insert(1, widths.Select(w => new string('-', w)).ToArray());
foreach (var row in table)
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
if (!areSame[colIndex])
logger.WriteStatistic(row[colIndex].PadLeft(widths[colIndex], ' ') + " |");
logger.NewLine();
}
}
public void Export(IList <BenchmarkReport> reports, IBenchmarkLogger logger)
{
foreach (var exporter in exporters)
{
exporter.Export(reports, logger);
}
}
public BenchmarkClassicFlow(Benchmark benchmark, IBenchmarkLogger logger)
{
this.benchmark = benchmark;
generator = new BenchmarkClassicGenerator(logger);
builder = new BenchmarkClassicBuilder(logger);
executor = new BenchmarkClassicExecutor(benchmark, logger);
}
public override void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null)
{
logger.WriteLine(string.Join(";", columns.Select(c => c.Title)));
foreach (var report in reports)
foreach (var run in report.Runs)
logger.WriteLine(string.Join(";", columns.Select(column => column.GetValue(report, run))));
}
private void PrintTable(List<string[]> table, IBenchmarkLogger logger, string[] columnsToAlwaysShow)
{
if (table.Count == 0)
{
logger.WriteLineError("There are no found benchmarks");
logger.NewLine();
return;
}
int rowCount = table.Count, colCount = table[0].Length;
var columnsToShowIndexes = columnsToAlwaysShow.Select(col => Array.IndexOf(table[0], col)).ToArray();
int[] widths = new int[colCount];
bool[] areSame = new bool[colCount];
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
areSame[colIndex] = rowCount > 2 && colIndex < colCount;
for (int rowIndex = 0; rowIndex < rowCount; rowIndex++)
{
widths[colIndex] = Math.Max(widths[colIndex], table[rowIndex][colIndex].Length + 1);
if (rowIndex > 1 && table[rowIndex][colIndex] != table[1][colIndex])
areSame[colIndex] = false;
}
}
if (areSame.Any(s => s))
{
var paramsOnLine = 0;
for (int colIndex = 0; colIndex < colCount; colIndex++)
if (areSame[colIndex] && columnsToShowIndexes.Contains(colIndex) == false)
{
logger.WriteInfo($"{table[0][colIndex]}={table[1][colIndex]} ");
paramsOnLine++;
if (paramsOnLine == 3)
{
logger.NewLine();
paramsOnLine = 0;
}
}
logger.NewLine();
}
if (useCodeBlocks)
{
logger.Write("```");
logger.NewLine();
}
table.Insert(1, widths.Select(w => new string('-', w)).ToArray());
foreach (var row in table)
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
if (!areSame[colIndex] || columnsToShowIndexes.Contains(colIndex))
logger.WriteStatistic(row[colIndex].PadLeft(widths[colIndex], ' ') + " |");
logger.NewLine();
}
if (areSame.Any(s => s))
{
logger.NewLine();
}
}
/// <summary>
/// Parses the benchmark statistics from the plain text line.
///
/// E.g. given the input <paramref name="line"/>:
///
/// Target 1: 10 op, 1005.8 ms, 1005842518 ns, 3332139 ticks, 100584251.7955 ns/op, 9.9 op/s
///
/// Will extract the number of <see cref="Operations"/> performed and the
/// total number of <see cref="Nanoseconds"/> it took to perform them.
/// </summary>
/// <param name="logger">The logger to write any diagnostic messages to.</param>
/// <param name="line">The line to parse.</param>
/// <returns>An instance of <see cref="BenchmarkRunReport"/> if parsed successfully. <c>Null</c> in case of any trouble.</returns>
public static BenchmarkRunReport Parse(IBenchmarkLogger logger, string line)
{
try
{
var op = 1L;
var ns = double.PositiveInfinity;
var items = line.
Split(new[] {':'}, StringSplitOptions.RemoveEmptyEntries)[1].
Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries);
foreach (var item in items)
{
var split = item.Split(new[] {' '}, StringSplitOptions.RemoveEmptyEntries);
var unit = split[1];
switch (unit)
{
case "ns":
ns = double.Parse(split[0], EnvironmentHelper.MainCultureInfo);
break;
case "op":
op = long.Parse(split[0]);
break;
}
}
return new BenchmarkRunReport(op, ns);
}
catch (Exception)
{
logger.WriteLineError("Parse error in the following line:");
logger.WriteLineError(line);
return null;
}
}
private BenchmarkReport Run(IBenchmarkLogger logger, Benchmark benchmark, IList <string> importantPropertyNames, BenchmarkParameters parameters = null)
{
var toolchain = Plugins.CreateToolchain(benchmark, logger);
logger.WriteLineHeader("// **************************");
logger.WriteLineHeader("// Benchmark: " + benchmark.Description);
var generateResult = Generate(logger, toolchain);
if (!generateResult.IsGenerateSuccess)
{
return(BenchmarkReport.CreateEmpty(benchmark, parameters));
}
var buildResult = Build(logger, toolchain, generateResult);
if (!buildResult.IsBuildSuccess)
{
return(BenchmarkReport.CreateEmpty(benchmark, parameters));
}
var runReports = Execute(logger, benchmark, importantPropertyNames, parameters, toolchain, buildResult);
return(new BenchmarkReport(benchmark, runReports, parameters));
}
/// <summary>
/// Parses the benchmark statistics from the plain text line.
///
/// E.g. given the input <paramref name="line"/>:
///
/// Target 1: 10 op, 1005.8 ms, 1005842518 ns, 3332139 ticks, 100584251.7955 ns/op, 9.9 op/s
///
/// Will extract the number of <see cref="Operations"/> performed and the
/// total number of <see cref="Nanoseconds"/> it took to perform them.
/// </summary>
/// <param name="logger">The logger to write any diagnostic messages to.</param>
/// <param name="line">The line to parse.</param>
/// <returns>An instance of <see cref="BenchmarkRunReport"/> if parsed successfully. <c>Null</c> in case of any trouble.</returns>
public static BenchmarkRunReport Parse(IBenchmarkLogger logger, string line)
{
try
{
var op = 1L;
var ns = double.PositiveInfinity;
var items = line.
Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries)[1].
Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var item in items)
{
var split = item.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
var unit = split[1];
switch (unit)
{
case "ns":
ns = double.Parse(split[0], EnvironmentHelper.MainCultureInfo);
break;
case "op":
op = long.Parse(split[0]);
break;
}
}
return(new BenchmarkRunReport(op, ns));
}
catch (Exception)
{
logger.WriteLineError("Parse error in the following line:");
logger.WriteLineError(line);
return(null);
}
}
public void Print(Benchmark benchmark, Process process, IBenchmarkLogger logger)
{
foreach (var diagnoster in diagnosers)
{
diagnoster.Print(benchmark, process, logger);
}
}
private void RunCompetitions(string[] args)
{
for (int i = 0; i < Competitions.Length; i++)
{
var competition = Competitions[i];
if (args.Any(arg => competition.Name.ToLower().StartsWith(arg.ToLower())) || args.Contains("#" + i) || args.Contains("*"))
{
logger.WriteLineHeader("Target competition: " + competition.Name);
using (var logStreamWriter = new StreamWriter(competition.Name + ".log"))
{
var loggers = new IBenchmarkLogger[] { new BenchmarkConsoleLogger(), new BenchmarkStreamLogger(logStreamWriter) };
var runner = new BenchmarkRunner(loggers);
runner.RunCompetition(Activator.CreateInstance(competition), BenchmarkSettings.Parse(args));
}
logger.NewLine();
}
}
if (args.Length > 0 && (args[0].StartsWith("http://") || args[0].StartsWith("https://")))
{
var url = args[0];
Uri uri = new Uri(url);
var name = uri.IsFile ? Path.GetFileName(uri.LocalPath) : "URL";
using (var logStreamWriter = new StreamWriter(name + ".log"))
{
var loggers = new IBenchmarkLogger[] { new BenchmarkConsoleLogger(), new BenchmarkStreamLogger(logStreamWriter) };
var runner = new BenchmarkRunner(loggers);
runner.RunUrl(url, BenchmarkSettings.Parse(args));
}
}
}
private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
logger.WriteLineInfo($"// {benchmark.Description}");
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
}
logger.NewLine();
}
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
BenchmarkMarkdownExporter.Default.Export(reports, logger);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// *** Warnings *** ");
foreach (var warning in warnings)
logger.WriteLineError($"{warning.Message}");
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return reports;
}
public static IBenchmarkToolchainFacade CreateToolchain(Benchmark benchmark, IBenchmarkLogger logger)
{
switch (benchmark.Task.Configuration.Toolchain)
{
case BenchmarkToolchain.Classic:
return new BenchmarkToolchainFacade(benchmark, new BenchmarkClassicGenerator(logger), new BenchmarkClassicBuilder(logger), new BenchmarkClassicExecutor(benchmark, logger));
default:
throw new NotSupportedException();
}
}
public static IBenchmarkFlow CreateFlow(Benchmark benchmark, IBenchmarkLogger logger)
{
switch (benchmark.Task.Configuration.Executor)
{
case BenchmarkExecutor.Classic:
return new BenchmarkClassicFlow(benchmark, logger);
default:
throw new NotSupportedException();
}
}
public BuildResult BuildProject(string directoryPath, IBenchmarkLogger logger)
{
string projectFileName = Path.Combine(directoryPath, MainClassName + ".csproj");
var consoleLogger = new MSBuildConsoleLogger(logger);
var globalProperties = new Dictionary<string, string>();
var buildRequest = new BuildRequestData(projectFileName, globalProperties, null, new[] { "Build" }, null);
var buildParameters = new BuildParameters(new ProjectCollection()) { DetailedSummary = false, Loggers = new ILogger[] { consoleLogger } };
var buildResult = BuildManager.DefaultBuildManager.Build(buildParameters, buildRequest);
return buildResult;
}
private void PrintTable(List <string[]> table, IBenchmarkLogger logger, string[] columnsToAlwaysShow)
{
if (table.Count == 0)
{
logger.WriteLineError("There are no found benchmarks");
logger.NewLine();
return;
}
int rowCount = table.Count, colCount = table[0].Length;
var columnsToShowIndexes = columnsToAlwaysShow.Select(col => Array.IndexOf(table[0], col));
int[] widths = new int[colCount];
bool[] areSame = new bool[colCount];
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
areSame[colIndex] = rowCount > 2 && colIndex < colCount - 3;
for (int rowIndex = 0; rowIndex < rowCount; rowIndex++)
{
widths[colIndex] = Math.Max(widths[colIndex], table[rowIndex][colIndex].Length + 1);
if (rowIndex > 1 && table[rowIndex][colIndex] != table[1][colIndex])
{
areSame[colIndex] = false;
}
}
}
if (areSame.Any(s => s))
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
if (areSame[colIndex] && columnsToShowIndexes.Contains(colIndex) == false)
{
logger.WriteInfo($"{table[0][colIndex]}={table[1][colIndex]} ");
}
}
logger.NewLine();
}
table.Insert(1, widths.Select(w => new string('-', w)).ToArray());
foreach (var row in table)
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
if (!areSame[colIndex] || columnsToShowIndexes.Contains(colIndex))
{
logger.WriteStatistic(row[colIndex].PadLeft(widths[colIndex], ' ') + " |");
}
}
logger.NewLine();
}
if (areSame.Any(s => s))
{
logger.NewLine();
}
}
public BenchmarkClassicExecutor(Benchmark benchmark, IBenchmarkLogger logger)
{
this.benchmark = benchmark;
this.logger = logger;
if (consoleHandler == null)
{
consoleHandler = new ConsoleHandler(logger);
Console.CancelKeyPress += consoleHandler.EventHandler;
}
}
public IBenchmarkToolchainFacade CreateToolchain(Benchmark benchmark, IBenchmarkLogger logger)
{
var toolchain = benchmark.Task.Configuration.Toolchain;
var targetToolchainBuilder = toolchains.FirstOrDefault(t => t.TargetToolchain == toolchain);
if (targetToolchainBuilder != null)
{
return(targetToolchainBuilder.Build(benchmark, logger));
}
throw new NotSupportedException($"There are no toolchain implementations for the '{toolchain}' toolchain");
}
public BenchmarkCodeExtractor(Benchmark benchmark, Process process, string codeExeName, IBenchmarkLogger logger)
{
this.process = process;
this.codeExeName = codeExeName;
this.logger = logger;
//Method name format: "BenchmarkDotNet.Samples.Infra.RunFast()" (NOTE: WITHOUT the return type)
var methodInfo = benchmark.Target.Method;
fullTypeName = methodInfo.DeclaringType.FullName;
var methodParams = string.Join(", ", methodInfo.GetParameters().Select(p => p.ParameterType.FullName));
fullMethodName = $"{fullTypeName}.{methodInfo.Name}({methodParams})";
}
public override void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null)
{
var table = BenchmarkExporterHelper.BuildTable(reports, resultExtenders, false);
foreach (var line in table)
{
for (int i = 0; i < line.Length; i++)
{
if (i != 0)
logger.Write(";");
logger.Write(line[i]);
}
logger.NewLine();
}
}
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger)
{
var table = ReportExporterHelper.BuildTable(reports, false);
foreach (var line in table)
{
for (int i = 0; i < line.Length; i++)
{
if (i != 0)
logger.Write(";");
logger.Write(line[i]);
}
logger.NewLine();
}
}
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger)
{
logger.WriteLineInfo("```ini");
logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
var table = ReportExporterHelper.BuildTable(reports);
PrintTable(table, logger);
var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
}
public void Export(IList <BenchmarkReport> reports, IBenchmarkLogger logger)
{
var table = BenchmarkExporterHelper.BuildTable(reports, false, true);
foreach (var line in table)
{
for (int i = 0; i < line.Length; i++)
{
if (i != 0)
{
logger.Write(";");
}
logger.Write(line[i]);
}
logger.NewLine();
}
}
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger)
{
logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
var table = BenchmarkExporterHelper.BuildTable(reports);
// If we have Benchmarks with ParametersSets, force the "Method" columns to be displayed, otherwise it doesn't make as much sense
var columnsToAlwaysShow = reports.Any(r => r.Benchmark.Task.ParametersSets != null) ? new[] { "Method" } : new string[0];
PrintTable(table, logger, columnsToAlwaysShow);
var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
}
private void RunCompetitions(string[] args)
{
for (int i = 0; i < Competitions.Length; i++)
{
var competition = Competitions[i];
if (args.Any(arg => competition.Name.ToLower().StartsWith(arg.ToLower())) || args.Contains("#" + i) || args.Contains("*"))
{
logger.WriteLineHeader("Target competition: " + competition.Name);
using (var logStreamWriter = new StreamWriter(competition.Name + ".log"))
{
var loggers = new IBenchmarkLogger[] {new BenchmarkConsoleLogger(), new BenchmarkStreamLogger(logStreamWriter)};
var runner = new BenchmarkRunner(loggers);
runner.RunCompetition(Activator.CreateInstance(competition), BenchmarkSettings.Parse(args));
}
logger.NewLine();
}
}
}
public override void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null)
{
foreach (var report in reports)
{
var runs = report.Runs;
var modes = runs.Select(it => it.IterationMode).Distinct();
logger.WriteLineHeader($"*** {report.Benchmark.Description} ***");
logger.WriteLineHeader("* Raw *");
foreach (var run in runs)
logger.WriteLineResult(run.ToStr());
foreach (var mode in modes)
{
logger.NewLine();
logger.WriteLineHeader($"* Statistics for {mode}");
logger.WriteLineStatistic(runs.Where(it => it.IterationMode == mode).GetStats().ToTimeStr());
}
}
}
private BenchmarkBuildResult Build(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain, BenchmarkGenerateResult generateResult)
{
logger.WriteLineInfo("// *** Build ***");
var buildResult = toolchain.Build(generateResult);
if (buildResult.IsBuildSuccess)
{
logger.WriteLineInfo("// Result = Success");
}
else
{
logger.WriteLineError("// Result = Failure");
if (buildResult.BuildException != null)
{
logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}");
}
}
logger.NewLine();
return(buildResult);
}
private BenchmarkGenerateResult Generate(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain)
{
logger.WriteLineInfo("// *** Generate *** ");
var generateResult = toolchain.Generate();
if (generateResult.IsGenerateSuccess)
{
logger.WriteLineInfo("// Result = Success");
logger.WriteLineInfo($"// {nameof(generateResult.DirectoryPath)} = {generateResult.DirectoryPath}");
}
else
{
logger.WriteLineError("// Result = Failure");
if (generateResult.GenerateException != null)
{
logger.WriteLineError($"// Exception: {generateResult.GenerateException.Message}");
}
}
logger.NewLine();
return(generateResult);
}
public void Export(IList <BenchmarkReport> reports, IBenchmarkLogger logger)
{
logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
var table = BenchmarkExporterHelper.BuildTable(reports);
// If we have Benchmarks with ParametersSets, force the "Method" columns to be displayed, otherwise it doesn't make as much sense
var columnsToAlwaysShow = reports.Any(r => r.Benchmark.Task.ParametersSets != null) ? new[] { "Method" } : new string[0];
PrintTable(table, logger, columnsToAlwaysShow);
var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
{
logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
}
}
/// <summary>
/// Parses the benchmark statistics from the plain text line.
///
/// E.g. given the input <paramref name="line"/>:
///
/// Target 1: 10 op, 1005842518 ns
///
/// Will extract the number of <see cref="Operations"/> performed and the
/// total number of <see cref="Nanoseconds"/> it took to perform them.
/// </summary>
/// <param name="logger">The logger to write any diagnostic messages to.</param>
/// <param name="line">The line to parse.</param>
/// <returns>An instance of <see cref="BenchmarkRunReport"/> if parsed successfully. <c>Null</c> in case of any trouble.</returns>
public static BenchmarkRunReport Parse(IBenchmarkLogger logger, string line)
{
try
{
var lineSplit = line.Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
var iterationInfo = lineSplit[0];
var iterationInfoSplit = iterationInfo.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
var iterationMode = ParseIterationMode(iterationInfoSplit[0]);
var iterationIndex = 0;
int.TryParse(iterationInfoSplit[1], out iterationIndex);
var measurementsInfo = lineSplit[1];
var measurementsInfoSplit = measurementsInfo.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
var op = 1L;
var ns = double.PositiveInfinity;
foreach (var item in measurementsInfoSplit)
{
var measurementSplit = item.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
var value = measurementSplit[0];
var unit = measurementSplit[1];
switch (unit)
{
case "ns":
ns = double.Parse(value, EnvironmentInfo.MainCultureInfo);
break;
case "op":
op = long.Parse(value);
break;
}
}
return new BenchmarkRunReport(iterationMode, iterationIndex, op, ns);
}
catch (Exception)
{
logger.WriteLineError("Parse error in the following line:");
logger.WriteLineError(line);
return null;
}
}
private List <BenchmarkRunReport> Execute(IBenchmarkLogger logger, Benchmark benchmark, IList <string> importantPropertyNames, BenchmarkParameters parameters, IBenchmarkToolchainFacade toolchain, BenchmarkBuildResult buildResult)
{
logger.WriteLineInfo("// *** Execute ***");
var processCount = Math.Max(1, benchmark.Task.ProcessCount);
var runReports = new List <BenchmarkRunReport>();
for (int processNumber = 0; processNumber < processCount; processNumber++)
{
logger.WriteLineInfo($"// Run, Process: {processNumber + 1} / {processCount}");
if (parameters != null)
{
logger.WriteLineInfo($"// {parameters.ToInfo()}");
}
if (importantPropertyNames.Any())
{
logger.WriteInfo("// ");
foreach (var name in importantPropertyNames)
{
logger.WriteInfo($"{name}={benchmark.Properties.GetValue(name)} ");
}
logger.NewLine();
}
var execResult = toolchain.Execute(buildResult, parameters, Plugins.CompositeDiagnoser);
if (execResult.FoundExecutable)
{
var iterRunReports = execResult.Data.Select(line => BenchmarkRunReport.Parse(logger, line)).Where(r => r != null).ToList();
runReports.AddRange(iterRunReports);
}
else
{
logger.WriteLineError("Executable not found");
}
}
logger.NewLine();
return(runReports);
}
public IBenchmarkToolchainFacade Build(Benchmark benchmark, IBenchmarkLogger logger)
{
return(new BenchmarkToolchainFacade(benchmark, createGenerator(benchmark, logger), createBuilder(benchmark, logger), createExecutor(benchmark, logger)));
}
public static void WriteLine(this IBenchmarkLogger logger)
{
logger.WriteLine(BenchmarkLogKind.Default, "");
}
public void Print(Benchmark benchmark, Process process, IBenchmarkLogger logger)
{
foreach (var diagnoster in diagnosers)
diagnoster.Print(benchmark, process, logger);
}
public IBenchmarkToolchainFacade Build(Benchmark benchmark, IBenchmarkLogger logger)
{
return new BenchmarkToolchainFacade(benchmark, createGenerator(benchmark, logger), createBuilder(benchmark, logger), createExecutor(benchmark, logger));
}
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null)
{
ExportResultExtenders = resultExtenders;
ExportCalled = true;
}
public ConsoleHandler( IBenchmarkLogger logger)
{
this.logger = logger;
EventHandler = new ConsoleCancelEventHandler(HandlerCallback);
}
public BenchmarkClassicBuilder(IBenchmarkLogger logger)
{
this.logger = logger;
}
public static void NewLine(this IBenchmarkLogger logger)
{
logger.Write(BenchmarkLogKind.Default, Environment.NewLine);
}
public static void WriteError(this IBenchmarkLogger logger, string format, params object[] args)
{
logger.Write(BenchmarkLogKind.Error, format, args);
}
public static void WriteLine(this IBenchmarkLogger logger, BenchmarkLogKind logKind, string format, params object[] args)
{
logger.Write(logKind, (format ?? string.Empty) + Environment.NewLine, args);
}
public BenchmarkClassicExecutor(Benchmark benchmark, IBenchmarkLogger logger)
{
this.benchmark = benchmark;
this.logger = logger;
}
public BenchmarkExecutor(IBenchmarkLogger logger = null, bool monoMode = false)
{
Logger = logger;
MonoMode = monoMode;
}
public IBenchmarkPluginBuilder AddLogger(IBenchmarkLogger logger)
{
loggers.Add(logger);
return(this);
}
public BenchmarkExecutor(IBenchmarkLogger logger = null, bool monoMode = false)
{
Logger = logger;
MonoMode = monoMode;
}
private List <BenchmarkReport> Run(List <Benchmark> benchmarks, IBenchmarkLogger logger)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
{
logger.WriteLineInfo($"// {benchmark.Description}");
}
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List <BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
}
logger.NewLine();
}
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
BenchmarkMarkdownExporter.Default.Export(reports, logger);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// *** Warnings *** ");
foreach (var warning in warnings)
{
logger.WriteLineError($"{warning.Message}");
}
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return(reports);
}
public void Print(Benchmark benchmark, Process process, IBenchmarkLogger logger)
{
PrintCodeForMethod(benchmark, process, logger, true, true, false);
}
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger)
{
foreach (var exporter in exporters)
exporter.Export(reports, logger);
}
/// <summary>
/// Code from http://stackoverflow.com/questions/2057781/is-there-a-way-to-get-the-stacktraces-for-all-threads-in-c-like-java-lang-thre/24315960#24315960
/// also see http://stackoverflow.com/questions/31633541/clrmd-throws-exception-when-creating-runtime/31745689#31745689
/// </summary>
public void PrintCodeForMethod(Benchmark benchmark, Process process, IBenchmarkLogger logger, bool printAssembly, bool printIL, bool printDiagnostics)
{
this.process = process;
this.logger = logger;
//Method name format: "BenchmarkDotNet.Samples.Infra.RunFast()" (NOTE: WITHOUT the return type)
var methodInfo = benchmark.Target.Method;
fullTypeName = methodInfo.DeclaringType.FullName;
var methodParams = string.Join(", ", methodInfo.GetParameters().Select(p => p.ParameterType.FullName));
fullMethodName = $"{fullTypeName}.{methodInfo.Name}({methodParams})";
logger?.WriteLine($"\nPrinting Code for Method: {fullMethodName}");
logger?.WriteLine($"\nPrintAssembly={printAssembly}, PrintIL={printIL}");
logger?.WriteLine($"Attaching to process {Path.GetFileName(process.MainModule.FileName)}, Pid={process.Id}");
logger?.WriteLine($"Path {process.MainModule.FileName}");
using (var dataTarget = DataTarget.AttachToProcess(process.Id, 5000, AttachFlag.NonInvasive))
{
var runtime = SetupClrRuntime(dataTarget);
if (printDiagnostics)
{
PrintRuntimeDiagnosticInfo(dataTarget, runtime);
}
if (printAssembly == false && printIL == false)
{
return;
}
ClrType @class = runtime.GetHeap().GetTypeByName(fullTypeName);
ClrMethod @method = @class.Methods.Single(m => m.GetFullSignature() == fullMethodName);
DesktopModule module = (DesktopModule)@method.Type.Module;
if (!module.IsPdbLoaded)
{
string pdbLocation = module.TryDownloadPdb(null);
if (pdbLocation != null)
{
module.LoadPdb(pdbLocation);
}
}
logger?.WriteLine($"Module: {Path.GetFileName(module.Name)}");
logger?.WriteLine($"Type: {method.Type.Name}");
logger?.WriteLine($"Method: {method.Name}");
// TODO work out why this returns locations inside OTHER methods, it's like it doesn't have an upper bound and just keeps going!?
var ilOffsetLocations = module.GetSourceLocationsForMethod(@method.MetadataToken);
string filePath = null;
string[] lines = null;
logger?.WriteLine("");
for (int i = 0; i < ilOffsetLocations.Count; i++)
{
var location = ilOffsetLocations[i];
var ilMaps = @method.ILOffsetMap.Where(il => il.ILOffset == location.ILOffset).ToList();
if (ilMaps.Any() == false)
{
continue;
}
if (lines == null || location.SourceLocation.FilePath != filePath)
{
filePath = location.SourceLocation.FilePath;
lines = File.ReadAllLines(filePath);
logger?.WriteLine($"Parsing file {Path.GetFileName(location.SourceLocation.FilePath)}");
}
PrintLocationAndILMapInfo(@method, location, ilMaps);
PrintSourceCode(lines, location);
if (printAssembly)
{
var debugControl = dataTarget.DebuggerInterface as IDebugControl;
PrintAssemblyCode(@method, ilMaps, runtime, debugControl);
}
}
}
}
public BenchmarkClassicGenerator(IBenchmarkLogger logger)
{
this.logger = logger;
}
public MSBuildConsoleLogger(IBenchmarkLogger logger)
{
Logger = logger;
}
public BenchmarkClassicExecutor(Benchmark benchmark, IBenchmarkLogger logger)
{
this.benchmark = benchmark;
this.logger = logger;
}
public BenchmarkClassicGenerator(IBenchmarkLogger logger)
{
this.logger = logger;
}
public static void WriteLineInfo(this IBenchmarkLogger logger, string format, params object[] args)
{
logger.WriteLine(BenchmarkLogKind.Info, format, args);
}
private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger, string competitionName)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
logger.WriteLineInfo($"// {benchmark.Description}");
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var globalStopwatch = Stopwatch.StartNew();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.GetTargetRuns().Any())
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.GetTargetRuns().Any())
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
}
}
logger.NewLine();
}
globalStopwatch.Stop();
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
logger.WriteLineHeader("// * Export *");
var files = Plugins.CompositeExporter.ExportToFile(reports, competitionName, Plugins.ResultExtenders);
foreach (var file in files)
logger.WriteLineInfo($" {file}");
logger.NewLine();
logger.WriteLineHeader("// * Detailed results *");
foreach (var report in reports)
{
logger.WriteLineInfo(report.Benchmark.Description);
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
logger.NewLine();
}
logger.WriteLineStatistic($"Total time: {globalStopwatch.Elapsed.TotalHours:00}:{globalStopwatch.Elapsed:mm\\:ss}");
logger.NewLine();
logger.WriteLineHeader("// * Summary *");
BenchmarkMarkdownExporter.Default.Export(reports, logger, Plugins.ResultExtenders);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// * Warnings * ");
foreach (var warning in warnings)
logger.WriteLineError($"{warning.Message}");
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return reports;
}
