public void Should_report_valid_markdown_format()
{
CleanPerfDir(_perfResultsPath);
SystemTime.UtcNow = () => new DateTime(2017, 3, 13);
var mdOutput = new MarkdownBenchmarkOutput(_perfResultsPath);
var fakeBenchmarkResults = new BenchmarkResults("NBench.FakeBenchmark",
new BenchmarkSettings(TestMode.Test, RunMode.Iterations, 30, 1000,
new List <IBenchmarkSetting>(),
new ConcurrentDictionary <MetricName, MetricsCollectorSelector>()),
new List <BenchmarkRunReport>()
{
new BenchmarkRunReport(TimeSpan.FromSeconds(3),
new List <MetricRunReport>()
{
new MetricRunReport(new CounterMetricName("FakeCounterMetric"), "bytes", 0d, Stopwatch.Frequency),
new MetricRunReport(new GcMetricName(GcMetric.TotalCollections, GcGeneration.Gen2), "collections", 0d, Stopwatch.Frequency),
new MetricRunReport(new MemoryMetricName(MemoryMetric.TotalBytesAllocated), "operations", 0d, Stopwatch.Frequency),
},
new List <Exception>())
});
var fakeBenchmarkFinalResults = new BenchmarkFinalResults(fakeBenchmarkResults, new List <AssertionResult>()
{
new AssertionResult(new CounterMetricName("FakeCounterMetric"), "[Counter] FakeCounterMetric Assertion Result", true),
new AssertionResult(new GcMetricName(GcMetric.TotalCollections, GcGeneration.Gen2), "TotalCollections [Gen2] Assertion Result", true),
new AssertionResult(new MemoryMetricName(MemoryMetric.TotalBytesAllocated), "TotalBytesAllocated Assertion Result", true)
});
mdOutput.WriteBenchmark(fakeBenchmarkFinalResults);
var fakePerfResultsFile = Directory.GetFiles(_perfResultsPath, "NBench.FakeBenchmark*",
SearchOption.AllDirectories);
Approvals.Verify(File.ReadAllText(fakePerfResultsFile.FirstOrDefault()), ScrubSysInfo);
}
public void WriteBenchmark(BenchmarkFinalResults results)
{
Console.WriteLine("--------------- RESULTS: {0} ---------------", results.BenchmarkName);
if (!string.IsNullOrEmpty(results.Data.Settings.Description))
{
Console.ForegroundColor = ConsoleColor.Gray;
Console.WriteLine(results.Data.Settings.Description);
Console.ResetColor();
}
Console.WriteLine("--------------- DATA ---------------");
foreach (var metric in results.Data.StatsByMetric.Values)
{
Console.WriteLine("{0}: Max: {2:n} {1}, Average: {3:n} {1}, Min: {4:n} {1}, StdDev: {5:n} {1}", metric.Name,
metric.Unit, metric.Stats.Max, metric.Stats.Average, metric.Stats.Min, metric.Stats.StandardDeviation);
Console.WriteLine("{0}: Max / s: {2:n} {1}, Average / s: {3:n} {1}, Min / s: {4:n} {1}, StdDev / s: {5:n} {1}", metric.Name,
metric.Unit, metric.PerSecondStats.Max, metric.PerSecondStats.Average, metric.PerSecondStats.Min, metric.PerSecondStats.StandardDeviation);
Console.WriteLine();
}
if (results.AssertionResults.Count > 0)
{
Console.WriteLine("--------------- ASSERTIONS ---------------");
foreach (var assertion in results.AssertionResults)
{
Console.ForegroundColor = assertion.Passed ? ConsoleColor.DarkGreen : ConsoleColor.DarkRed;
Console.WriteLine(assertion.Message);
Console.ResetColor();
}
}
if (results.Data.IsFaulted)
{
Console.WriteLine("--------------- EXCEPTIONS ---------------");
foreach (var exception in results.Data.Exceptions)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(exception);
Console.ResetColor();
}
}
Console.WriteLine();
}
public void WriteBenchmark(BenchmarkFinalResults results)
{
Console.WriteLine("--------------- RESULTS: {0} ---------------", results.BenchmarkName);
if (!string.IsNullOrEmpty(results.Data.Settings.Description))
{
Console.ForegroundColor = ConsoleColor.Gray;
Console.WriteLine(results.Data.Settings.Description);
Console.ResetColor();
}
Console.WriteLine("--------------- DATA ---------------");
foreach (var metric in results.Data.StatsByMetric.Values)
{
Console.WriteLine("{0}: Max: {2:n} {1}, Average: {3:n} {1}, Min: {4:n} {1}, StdDev: {5:n} {1}", metric.Name,
metric.Unit, metric.Stats.Max, metric.Stats.Average, metric.Stats.Min, metric.Stats.StandardDeviation);
Console.WriteLine("{0}: Max / s: {2:n} {1}, Average / s: {3:n} {1}, Min / s: {4:n} {1}, StdDev / s: {5:n} {1}", metric.Name,
metric.Unit, metric.PerSecondStats.Max, metric.PerSecondStats.Average, metric.PerSecondStats.Min, metric.PerSecondStats.StandardDeviation);
Console.WriteLine();
}
if (results.AssertionResults.Count > 0)
{
Console.WriteLine("--------------- ASSERTIONS ---------------");
foreach (var assertion in results.AssertionResults)
{
Console.ForegroundColor = assertion.Passed ? ConsoleColor.DarkGreen : ConsoleColor.DarkRed;
Console.WriteLine(assertion.Message);
Console.ResetColor();
}
}
if (results.Data.IsFaulted)
{
Console.WriteLine("--------------- EXCEPTIONS ---------------");
foreach (var exception in results.Data.Exceptions)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(exception);
Console.ResetColor();
}
}
Console.WriteLine();
}
public void WriteBenchmark(BenchmarkFinalResults results)
{
var filePath = FileNameGenerator.GenerateFileName(_outputDirectory, results.BenchmarkName,
MarkdownFileExtension, DateTime.UtcNow);
var sysInfo = SysInfo.Instance;
var sb = new StringBuilder();
sb.AppendLine($"# {results.BenchmarkName}");
if (!string.IsNullOrEmpty(results.Data.Settings.Description))
{
sb.AppendLine($"__{results.Data.Settings.Description}__");
}
sb.AppendLine($"_{DateTime.UtcNow}_");
sb.AppendLine("### System Info");
sb.AppendLine("```ini");
sb.AppendLine($"NBench={sysInfo.NBenchAssemblyVersion}");
sb.AppendLine($"OS={sysInfo.OS}");
sb.AppendLine($"ProcessorCount={sysInfo.ProcessorCount}");
sb.AppendLine(
$"CLR={sysInfo.ClrVersion},IsMono={sysInfo.IsMono},MaxGcGeneration={sysInfo.MaxGcGeneration}");
sb.AppendLine($"WorkerThreads={sysInfo.WorkerThreads}, IOThreads={sysInfo.IOThreads}");
sb.AppendLine("```");
sb.AppendLine();
sb.AppendLine("### NBench Settings");
sb.AppendLine("```ini");
sb.AppendLine($"RunMode={results.Data.Settings.RunMode}, TestMode={results.Data.Settings.TestMode}");
if (results.Data.Settings.SkipWarmups)
{
sb.AppendLine($"SkipWarmups={results.Data.Settings.SkipWarmups}");
}
sb.AppendLine(
$"NumberOfIterations={results.Data.Settings.NumberOfIterations}, MaximumRunTime={results.Data.Settings.RunTime}");
sb.AppendLine($"Concurrent={results.Data.Settings.ConcurrentMode}");
sb.AppendLine($"Tracing={results.Data.Settings.TracingEnabled}");
sb.AppendLine("```");
sb.AppendLine();
sb.AppendLine("## Data");
sb.AppendLine("-------------------");
sb.AppendLine();
sb.AppendLine("### Totals");
sb.AppendFormat(BuildStatTable(results.Data.StatsByMetric.Values));
sb.AppendLine();
sb.AppendLine("### Per-second Totals");
sb.AppendFormat(BuildPerSecondsStatTable(results.Data.StatsByMetric.Values));
sb.AppendLine();
sb.AppendLine("### Raw Data");
sb.AppendFormat(BuildRunTable(results.Data.Runs));
if (results.AssertionResults.Count > 0)
{
sb.AppendLine("## Benchmark Assertions");
sb.AppendLine();
foreach (var assertion in results.AssertionResults)
{
sb.AppendLine("* " + assertion.Message);
}
sb.AppendLine();
}
if (results.Data.IsFaulted)
{
Console.WriteLine("## Exceptions");
foreach (var exception in results.Data.Exceptions)
{
sb.AppendLine("```");
sb.AppendLine(exception.ToString());
sb.AppendLine("```");
sb.AppendLine();
}
}
if (_traceStringBuilder.IsValueCreated) // append trace values if the tracing system was used
{
sb.AppendLine("## Traces");
sb.AppendLine(_traceStringBuilder.Value.ToString());
sb.AppendLine();
// reset the string builder, since it may be re-used on the next benchmark
_traceStringBuilder.Value.Clear();
}
var report = sb.ToString();
// ensure directory exists
var directoryPath = Path.GetDirectoryName(filePath);
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
File.WriteAllText(filePath, report, Encoding.UTF8);
}
public void WriteBenchmark(BenchmarkFinalResults results)
{
var filePath = FileNameGenerator.GenerateFileName(_outputDirectory, results.BenchmarkName,
MarkdownFileExtension, DateTime.UtcNow);
var sysInfo = SysInfo.Instance;
var sb = new StringBuilder();
sb.AppendLine($"# {results.BenchmarkName}");
if (!string.IsNullOrEmpty(results.Data.Settings.Description))
{
sb.AppendLine($"__{results.Data.Settings.Description}__");
}
sb.AppendLine($"_{DateTime.UtcNow}_");
sb.AppendLine("### System Info");
sb.AppendLine("```ini");
sb.AppendLine($"NBench={sysInfo.NBenchAssemblyVersion}");
sb.AppendLine($"OS={sysInfo.OS}");
sb.AppendLine($"ProcessorCount={sysInfo.ProcessorCount}");
sb.AppendLine(
$"CLR={sysInfo.ClrVersion},IsMono={sysInfo.IsMono},MaxGcGeneration={sysInfo.MaxGcGeneration}");
sb.AppendLine($"WorkerThreads={sysInfo.WorkerThreads}, IOThreads={sysInfo.IOThreads}");
sb.AppendLine("```");
sb.AppendLine();
sb.AppendLine("### NBench Settings");
sb.AppendLine("```ini");
sb.AppendLine($"RunMode={results.Data.Settings.RunMode}, TestMode={results.Data.Settings.TestMode}");
sb.AppendLine(
$"NumberOfIterations={results.Data.Settings.NumberOfIterations}, MaximumRunTime={results.Data.Settings.RunTime}");
sb.AppendLine("```");
sb.AppendLine();
sb.AppendLine("## Data");
sb.AppendLine("-------------------");
sb.AppendLine();
sb.AppendLine("### Totals");
sb.AppendFormat(BuildStatTable(results.Data.StatsByMetric.Values));
sb.AppendLine();
sb.AppendLine("### Per-second Totals");
sb.AppendFormat(BuildPerSecondsStatTable(results.Data.StatsByMetric.Values));
sb.AppendLine();
sb.AppendLine("### Raw Data");
sb.AppendFormat(BuildRunTable(results.Data.Runs));
if (results.AssertionResults.Count > 0)
{
sb.AppendLine("## Assertions");
sb.AppendLine();
foreach (var assertion in results.AssertionResults)
{
sb.AppendLine("* " + assertion.Message);
}
sb.AppendLine();
}
if (results.Data.IsFaulted)
{
Console.WriteLine("## Exceptions");
foreach (var exception in results.Data.Exceptions)
{
sb.AppendLine("```");
sb.AppendLine(exception.ToString());
sb.AppendLine("```");
sb.AppendLine();
}
}
var report = sb.ToString();
File.WriteAllText(filePath, report, Encoding.UTF8);
}
public void WriteBenchmark(BenchmarkFinalResults results)
{
_benchmarkAction(results);
}
public void WriteBenchmark(BenchmarkFinalResults results)
{
var filePath = FileNameGenerator.GenerateFileName(_outputDirectory, results.BenchmarkName,
MarkdownFileExtension, DateTime.UtcNow);
var sysInfo = SysInfo.Instance;
var sb = new StringBuilder();
sb.AppendLine($"# {results.BenchmarkName}");
if (!string.IsNullOrEmpty(results.Data.Settings.Description))
{
sb.AppendLine($"__{results.Data.Settings.Description}__");
}
sb.AppendLine($"_{DateTime.UtcNow}_");
sb.AppendLine("### System Info");
sb.AppendLine("```ini");
sb.AppendLine($"NBench={sysInfo.NBenchAssemblyVersion}");
sb.AppendLine($"OS={sysInfo.OS}");
sb.AppendLine($"ProcessorCount={sysInfo.ProcessorCount}");
sb.AppendLine(
$"CLR={sysInfo.ClrVersion},IsMono={sysInfo.IsMono},MaxGcGeneration={sysInfo.MaxGcGeneration}");
sb.AppendLine($"WorkerThreads={sysInfo.WorkerThreads}, IOThreads={sysInfo.IOThreads}");
sb.AppendLine("```");
sb.AppendLine();
sb.AppendLine("### NBench Settings");
sb.AppendLine("```ini");
sb.AppendLine($"RunMode={results.Data.Settings.RunMode}, TestMode={results.Data.Settings.TestMode}");
if (results.Data.Settings.SkipWarmups)
{
sb.AppendLine($"SkipWarmups={results.Data.Settings.SkipWarmups}");
}
sb.AppendLine(
$"NumberOfIterations={results.Data.Settings.NumberOfIterations}, MaximumRunTime={results.Data.Settings.RunTime}");
sb.AppendLine($"Concurrent={results.Data.Settings.ConcurrentMode}");
sb.AppendLine($"Tracing={results.Data.Settings.TracingEnabled}");
sb.AppendLine("```");
sb.AppendLine();
sb.AppendLine("## Data");
sb.AppendLine("-------------------");
sb.AppendLine();
sb.AppendLine("### Totals");
sb.AppendFormat(BuildStatTable(results.Data.StatsByMetric.Values));
sb.AppendLine();
sb.AppendLine("### Per-second Totals");
sb.AppendFormat(BuildPerSecondsStatTable(results.Data.StatsByMetric.Values));
sb.AppendLine();
sb.AppendLine("### Raw Data");
sb.AppendFormat(BuildRunTable(results.Data.Runs));
if (results.AssertionResults.Count > 0)
{
sb.AppendLine("## Benchmark Assertions");
sb.AppendLine();
foreach (var assertion in results.AssertionResults)
{
sb.AppendLine("* " + assertion.Message);
}
sb.AppendLine();
}
if (results.Data.IsFaulted)
{
Console.WriteLine("## Exceptions");
foreach (var exception in results.Data.Exceptions)
{
sb.AppendLine("```");
sb.AppendLine(exception.ToString());
sb.AppendLine("```");
sb.AppendLine();
}
}
if (_traceStringBuilder.IsValueCreated) // append trace values if the tracing system was used
{
sb.AppendLine("## Traces");
sb.AppendLine(_traceStringBuilder.Value.ToString());
sb.AppendLine();
// reset the string builder, since it may be re-used on the next benchmark
_traceStringBuilder.Value.Clear();
}
var report = sb.ToString();
// ensure directory exists
var directoryPath = Path.GetDirectoryName(filePath);
if (!Directory.Exists(directoryPath))
Directory.CreateDirectory(directoryPath);
File.WriteAllText(filePath, report, Encoding.UTF8);
}
public void WriteBenchmark(BenchmarkFinalResults results)
{
_benchmarkAction(results);
}
public void WriteBenchmark(BenchmarkFinalResults results)
{
}
public void WriteBenchmark(BenchmarkFinalResults results)
{
var filePath = FileNameGenerator.GenerateFileName(_outputDirectory, results.BenchmarkName,
MarkdownFileExtension, DateTime.UtcNow);
var sysInfo = SysInfo.Instance;
var sb = new StringBuilder();
sb.AppendLine($"# {results.BenchmarkName}");
if (!string.IsNullOrEmpty(results.Data.Settings.Description))
{
sb.AppendLine($"__{results.Data.Settings.Description}__");
}
sb.AppendLine($"_{DateTime.UtcNow}_");
sb.AppendLine("### System Info");
sb.AppendLine("```ini");
sb.AppendLine($"NBench={sysInfo.NBenchAssemblyVersion}");
sb.AppendLine($"OS={sysInfo.OS}");
sb.AppendLine($"ProcessorCount={sysInfo.ProcessorCount}");
sb.AppendLine(
$"CLR={sysInfo.ClrVersion},IsMono={sysInfo.IsMono},MaxGcGeneration={sysInfo.MaxGcGeneration}");
sb.AppendLine($"WorkerThreads={sysInfo.WorkerThreads}, IOThreads={sysInfo.IOThreads}");
sb.AppendLine("```");
sb.AppendLine();
sb.AppendLine("### NBench Settings");
sb.AppendLine("```ini");
sb.AppendLine($"RunMode={results.Data.Settings.RunMode}, TestMode={results.Data.Settings.TestMode}");
sb.AppendLine(
$"NumberOfIterations={results.Data.Settings.NumberOfIterations}, MaximumRunTime={results.Data.Settings.RunTime}");
sb.AppendLine("```");
sb.AppendLine();
sb.AppendLine("## Data");
sb.AppendLine("-------------------");
sb.AppendLine();
sb.AppendLine("### Totals");
sb.AppendFormat(BuildStatTable(results.Data.StatsByMetric.Values));
sb.AppendLine();
sb.AppendLine("### Per-second Totals");
sb.AppendFormat(BuildPerSecondsStatTable(results.Data.StatsByMetric.Values));
sb.AppendLine();
sb.AppendLine("### Raw Data");
sb.AppendFormat(BuildRunTable(results.Data.Runs));
if (results.AssertionResults.Count > 0)
{
sb.AppendLine("## Assertions");
sb.AppendLine();
foreach (var assertion in results.AssertionResults)
{
sb.AppendLine("* " + assertion.Message);
}
sb.AppendLine();
}
if (results.Data.IsFaulted)
{
Console.WriteLine("## Exceptions");
foreach (var exception in results.Data.Exceptions)
{
sb.AppendLine("```");
sb.AppendLine(exception.ToString());
sb.AppendLine("```");
sb.AppendLine();
}
}
var report = sb.ToString();
File.WriteAllText(filePath, report, Encoding.UTF8);
}
