public static string ToTimeStr(this Statistics s, Encoding encoding, TimeUnit unit = null, bool calcHistogram = false)
{
if (s == null)
{
return(NullSummaryMessage);
}
if (unit == null)
{
unit = TimeUnit.GetBestTimeUnit(s.Mean);
}
var builder = new StringBuilder();
string errorPercent = (s.StandardError / s.Mean * 100).ToStr("0.00");
var ci = s.ConfidenceInterval;
string ciMarginPercent = (ci.Margin / s.Mean * 100).ToStr("0.00");
double mValue = MathHelper.CalculateMValue(s);
builder.AppendLine($"Mean = {s.Mean.ToTimeStr(unit, encoding)}, StdErr = {s.StandardError.ToTimeStr(unit, encoding)} ({errorPercent}%); N = {s.N}, StdDev = {s.StandardDeviation.ToTimeStr(unit, encoding)}");
builder.AppendLine($"Min = {s.Min.ToTimeStr(unit, encoding)}, Q1 = {s.Q1.ToTimeStr(unit, encoding)}, Median = {s.Median.ToTimeStr(unit, encoding)}, Q3 = {s.Q3.ToTimeStr(unit, encoding)}, Max = {s.Max.ToTimeStr(unit, encoding)}");
builder.AppendLine($"IQR = {s.InterquartileRange.ToTimeStr(unit, encoding)}, LowerFence = {s.LowerFence.ToTimeStr(unit, encoding)}, UpperFence = {s.UpperFence.ToTimeStr(unit, encoding)}");
builder.AppendLine($"ConfidenceInterval = {s.ConfidenceInterval.ToTimeStr(encoding, unit)}, Margin = {ci.Margin.ToTimeStr(unit, encoding)} ({ciMarginPercent}% of Mean)");
builder.AppendLine($"Skewness = {s.Skewness.ToStr()}, Kurtosis = {s.Kurtosis.ToStr()}, MValue = {mValue.ToStr()}");
if (calcHistogram)
{
var histogram = HistogramBuilder.Adaptive.Build(s);
builder.AppendLine("-------------------- Histogram --------------------");
builder.AppendLine(histogram.ToTimeStr(encoding: encoding));
builder.AppendLine("---------------------------------------------------");
}
return(builder.ToString().Trim());
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
var valueOfN = (int)benchmarkCase.Parameters.Items.Single(p => p.Name == "N").Value;
var timePerN = summary[benchmarkCase].ResultStatistics.Mean / valueOfN;
return(timePerN.ToTimeStr(benchmarkCase.Config.Encoding, TimeUnit.GetBestTimeUnit(timePerN)));
}
public static void Log(Summary summary, ITestOutputHelper output)
{
var logger = new OutputLogger(output);
// We construct HostEnvironmentInfo manually, so that we can have the HardwareTimerKind enum as text, rather than an integer
// SimpleJson serializer doesn't seem to have an enum String/Value option (to-be-fair, it is meant to be "Simple")
//var environmentInfo = new
//{
// HostEnvironmentInfo.BenchmarkDotNetCaption,
// summary.HostEnvironmentInfo.BenchmarkDotNetVersion,
// OsVersion = summary.HostEnvironmentInfo.OsVersion.Value,
// ProcessorName = ProcessorBrandStringHelper.Prettify(summary.HostEnvironmentInfo.CpuInfo.Value),
// summary.HostEnvironmentInfo.CpuInfo.Value?.PhysicalProcessorCount,
// summary.HostEnvironmentInfo.CpuInfo.Value?.PhysicalCoreCount,
// summary.HostEnvironmentInfo.CpuInfo.Value?.LogicalCoreCount,
// summary.HostEnvironmentInfo.RuntimeVersion,
// summary.HostEnvironmentInfo.Architecture,
// summary.HostEnvironmentInfo.HasAttachedDebugger,
// summary.HostEnvironmentInfo.HasRyuJit,
// summary.HostEnvironmentInfo.Configuration,
// DotNetCliVersion = summary.HostEnvironmentInfo.DotNetSdkVersion.Value,
// summary.HostEnvironmentInfo.ChronometerFrequency,
// HardwareTimerKind = summary.HostEnvironmentInfo.HardwareTimerKind.ToString()
//};
// If we just ask SimpleJson to serialize the entire "summary" object it throws several errors.
// So we are more specific in what we serialize (plus some fields/properties aren't relevant)
var benchmarks = summary.Reports.Select(report =>
{
var data = new Dictionary <string, object>
{
// We don't need Benchmark.ShortInfo, that info is available via Benchmark.Parameters below
{ "DisplayInfo", report.BenchmarkCase.DisplayInfo },
{ "Namespace", report.BenchmarkCase.Descriptor.Type.Namespace },
{ "Type", GetTypeName(report.BenchmarkCase.Descriptor.Type) },
{ "Method", report.BenchmarkCase.Descriptor.WorkloadMethod.Name },
{ "MethodTitle", report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo },
{ "Parameters", report.BenchmarkCase.Parameters.PrintInfo },
{ "FullName", FullNameProvider.GetBenchmarkName(report.BenchmarkCase) }, // do NOT remove this property, it is used for xunit-performance migration
{ "ExecutionTime", $"{report.ResultStatistics.Mean.ToTimeStr(TimeUnit.GetBestTimeUnit(report.ResultStatistics.Mean))}" },
};
// We show MemoryDiagnoser's results only if it is being used
if (report.BenchmarkCase.Config.HasMemoryDiagnoser())
{
data.Add("Memory", report.GcStats);
}
return(data);
});
logger.WriteLine(JsonConvert.SerializeObject(new Dictionary <string, object>
{
{ "Title", summary.Title },
// { "HostEnvironmentInfo", environmentInfo },
{ "Benchmarks", benchmarks }
}, Formatting.Indented));
}
internal SummaryTable(Summary summary, ISummaryStyle style = null)
{
Summary = summary;
if (summary.HasCriticalValidationErrors)
{
Columns = Array.Empty <SummaryTableColumn>();
ColumnCount = 0;
FullHeader = Array.Empty <string>();
FullContent = Array.Empty <string[]>();
FullContentStartOfGroup = Array.Empty <bool>();
FullContentWithHeader = Array.Empty <string[]>();
IsDefault = Array.Empty <bool>();
return;
}
// Ensure we have all required data for styling
style = style ?? SummaryStyle.Default;
if (style.TimeUnit == null)
{
style = style.WithTimeUnit(TimeUnit.GetBestTimeUnit(summary.Reports.Where(r => r.ResultStatistics != null).Select(r => r.ResultStatistics.Mean).ToArray()));
}
if (style.SizeUnit == null)
{
style = style.WithSizeUnit(SizeUnit.GetBestSizeUnit(summary.Reports.Select(r => r.GcStats.BytesAllocatedPerOperation).ToArray()));
}
var columns = summary.GetColumns();
ColumnCount = columns.Length;
FullHeader = columns.Select(c => c.GetColumnTitle(style)).ToArray();
var orderProvider = summary.Config.GetOrderProvider() ?? DefaultOrderProvider.Instance;
FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.Benchmark, style)).ToArray()).ToArray();
IsDefault = columns.Select(c => summary.Reports.All(r => c.IsDefault(summary, r.Benchmark))).ToArray();
var groupKeys = summary.Benchmarks.Select(b => orderProvider.GetGroupKey(b, summary)).ToArray();
FullContentStartOfGroup = new bool[summary.Reports.Length];
if (groupKeys.Distinct().Count() > 1 && FullContentStartOfGroup.Length > 0)
{
FullContentStartOfGroup[0] = true;
for (int i = 1; i < summary.Reports.Length; i++)
{
FullContentStartOfGroup[i] = groupKeys[i] != groupKeys[i - 1];
}
}
var full = new List <string[]> {
FullHeader
};
full.AddRange(FullContent);
FullContentWithHeader = full.ToArray();
Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i])).ToArray();
EffectiveSummaryStyle = style;
}
public static string ToTimeStr(this double value, TimeUnit unit = null, int unitNameWidth = 1)
{
unit = unit ?? TimeUnit.GetBestTimeUnit(value);
var unitValue = TimeUnit.Convert(value, TimeUnit.Nanoseconds, unit);
var unitName = unit.Name.PadLeft(unitNameWidth);
return($"{unitValue.ToStr("N4")} {unitName}");
}
public static string ToTimeStr(this double value, TimeUnit unit = null, int unitNameWidth = 1, bool showUnit = true, string format = "N4",
Encoding encoding = null)
{
unit = unit ?? TimeUnit.GetBestTimeUnit(value);
double unitValue = TimeUnit.Convert(value, TimeUnit.Nanosecond, unit);
if (showUnit)
{
string unitName = unit.Name.ToString(encoding ?? Encoding.ASCII).PadLeft(unitNameWidth);
return($"{unitValue.ToStr(format)} {unitName}");
}
return($"{unitValue.ToStr(format)}");
}
public static string ToTimeStr(this double value, TimeUnit unit = null, int unitNameWidth = 1, bool showUnit = true, string format = "N4")
{
unit = unit ?? TimeUnit.GetBestTimeUnit(value);
var unitValue = TimeUnit.Convert(value, TimeUnit.Nanosecond, unit);
if (showUnit)
{
var unitName = unit.Name.PadLeft(unitNameWidth);
return($"{unitValue.ToStr(format)} {unitName}");
}
else
{
return($"{unitValue.ToStr(format)}");
}
}
public Summary(string title, IList <BenchmarkReport> reports, EnvironmentHelper hostEnvironmentHelper, IConfig config, string currentDirectory, TimeSpan totalTime)
{
Title = title;
HostEnvironmentHelper = hostEnvironmentHelper;
Config = config;
CurrentDirectory = currentDirectory;
TotalTime = totalTime;
Reports = new Dictionary <Benchmark, BenchmarkReport>();
Benchmarks = reports.Select(r => r.Benchmark).ToArray();
foreach (var report in reports)
{
Reports[report.Benchmark] = report;
}
TimeUnit = TimeUnit.GetBestTimeUnit(reports.Where(r => r.ResultStatistics != null).Select(r => r.ResultStatistics.Mean).ToArray());
Table = new SummaryTable(this);
}
public static string ToTimeStr(this Statistics s, TimeUnit unit = null)
{
if (s == null)
{
return(NullSummaryMessage);
}
if (unit == null)
{
unit = TimeUnit.GetBestTimeUnit(s.Mean);
}
var builder = new StringBuilder();
var errorPercent = (s.StandardError / s.Mean * 100).ToStr();
builder.AppendLine($"Mean = {s.Mean.ToTimeStr(unit)}, StdError = {s.StandardError.ToTimeStr(unit)} ({errorPercent}%); N = {s.N}, StdDev = {s.StandardDeviation.ToTimeStr(unit)}");
builder.AppendLine($"Min = {s.Min.ToTimeStr(unit)}, Q1 = {s.Q1.ToTimeStr(unit)}, Median = {s.Median.ToTimeStr(unit)}, Q3 = {s.Q3.ToTimeStr(unit)}, Max = {s.Max.ToTimeStr(unit)}");
builder.AppendLine($"IQR = {s.InterquartileRange.ToTimeStr(unit)}, LowerFence = {s.LowerFence.ToTimeStr(unit)}, UpperFence = {s.UpperFence.ToTimeStr(unit)}");
builder.AppendLine($"ConfidenceInterval = {s.ConfidenceInterval.ToTimeStr(unit)}");
return(builder.ToString());
}
public Summary(string title, IList <BenchmarkReport> reports, HostEnvironmentInfo hostEnvironmentInfo, IConfig config, string resultsDirectoryPath, TimeSpan totalTime, ValidationError[] validationErrors)
: this(title, hostEnvironmentInfo, config, resultsDirectoryPath, totalTime, validationErrors)
{
Benchmarks = reports.Select(r => r.Benchmark).ToArray();
foreach (var report in reports)
{
reportMap[report.Benchmark] = report;
}
Reports = Benchmarks.Select(b => reportMap[b]).ToArray();
var orderProvider = config.GetOrderProvider() ?? DefaultOrderProvider.Instance;
Benchmarks = orderProvider.GetSummaryOrder(Benchmarks, this).ToArray();
Reports = Benchmarks.Select(b => reportMap[b]).ToArray();
TimeUnit = TimeUnit.GetBestTimeUnit(reports.Where(r => r.ResultStatistics != null).Select(r => r.ResultStatistics.Mean).ToArray());
Table = new SummaryTable(this);
shortInfos = new Dictionary <IJob, string>();
jobs = new Lazy <IJob[]>(() => Benchmarks.Select(b => b.Job).ToArray());
}
public static string ToTimeStr(this Histogram histogram, TimeUnit unit = null, char binSymbol = '@', bool full = false, Encoding encoding = null)
{
const string format = "0.000";
var bins = histogram.Bins;
int binCount = histogram.Bins.Length;
if (unit == null)
{
unit = TimeUnit.GetBestTimeUnit(bins.SelectMany(bin => bin.Values).ToArray());
}
if (encoding == null)
{
encoding = Encoding.ASCII;
}
var lower = new string[binCount];
var upper = new string[binCount];
for (int i = 0; i < binCount; i++)
{
lower[i] = bins[i].Lower.ToTimeStr(unit, encoding, format);
upper[i] = bins[i].Upper.ToTimeStr(unit, encoding, format);
}
int lowerWidth = lower.Max(it => it.Length);
int upperWidth = upper.Max(it => it.Length);
var builder = new StringBuilder();
for (int i = 0; i < binCount; i++)
{
string intervalStr = $"[{lower[i].PadLeft(lowerWidth)} ; {upper[i].PadLeft(upperWidth)})";
string barStr = full
? string.Join(", ", bins[i].Values.Select(it => it.ToTimeStr(unit, encoding, format)))
: new string(binSymbol, bins[i].Count);
builder.AppendLine($"{intervalStr} | {barStr}");
}
return(builder.ToString().Trim());
}
[PublicAPI] public string ToString(CultureInfo cultureInfo)
{
var unit = TimeUnit.GetBestTimeUnit(Values);
var builder = new StringBuilder();
builder.Append('[');
builder.Append(TimeInterval.FromNanoseconds(Lower).ToString(unit, cultureInfo));
builder.Append(';');
builder.Append(TimeInterval.FromNanoseconds(Upper).ToString(unit, cultureInfo));
builder.Append(' ');
builder.Append('{');
for (var i = 0; i < Values.Length; i++)
{
if (i != 0)
{
builder.Append("; ");
}
builder.Append(TimeInterval.FromNanoseconds(Values[i]).ToString(unit, cultureInfo));
}
builder.Append('}');
return(builder.ToString());
}
public static Func <double, string> CreateNanosecondFormatter(this Histogram histogram, CultureInfo cultureInfo = null, string format = "0.000")
{
var timeUnit = TimeUnit.GetBestTimeUnit(histogram.Bins.SelectMany(bin => bin.Values).ToArray());
return(value => TimeInterval.FromNanoseconds(value).ToString(timeUnit, cultureInfo, format));
}
internal SummaryTable(Summary summary, SummaryStyle style = null)
{
Summary = summary;
if (summary.HasCriticalValidationErrors)
{
Columns = Array.Empty <SummaryTableColumn>();
ColumnCount = 0;
FullHeader = Array.Empty <string>();
FullContent = Array.Empty <string[]>();
FullContentStartOfHighlightGroup = Array.Empty <bool>();
FullContentWithHeader = Array.Empty <string[]>();
IsDefault = Array.Empty <bool>();
return;
}
// Ensure we have all required data for styling
style = style ?? summary.Style ?? SummaryStyle.Default;
if (style.TimeUnit == null)
{
style = style.WithTimeUnit(TimeUnit.GetBestTimeUnit(summary.Reports.Where(r => r.ResultStatistics != null).Select(r => r.ResultStatistics.Mean)
.ToArray()));
}
if (style.SizeUnit == null)
{
style = style.WithSizeUnit(SizeUnit.GetBestSizeUnit(summary.Reports.Select(r => r.GcStats.GetBytesAllocatedPerOperation(r.BenchmarkCase)).ToArray()));
}
var columns = summary.GetColumns();
ColumnCount = columns.Length;
FullHeader = columns.Select(c => c.GetColumnTitle(style)).ToArray();
FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.BenchmarkCase, style)).ToArray()).ToArray();
IsDefault = columns.Select(c => summary.Reports.All(r => c.IsDefault(summary, r.BenchmarkCase))).ToArray();
var highlightGroupKeys = summary.BenchmarksCases.Select(b => b.Config.Orderer.GetHighlightGroupKey(b)).ToArray();
FullContentStartOfHighlightGroup = new bool[summary.Reports.Length];
if (highlightGroupKeys.Distinct().Count() > 1 && FullContentStartOfHighlightGroup.Length > 0)
{
FullContentStartOfHighlightGroup[0] = true;
for (int i = 1; i < summary.Reports.Length; i++)
{
FullContentStartOfHighlightGroup[i] = highlightGroupKeys[i] != highlightGroupKeys[i - 1];
}
}
var logicalGroupKeys = summary.BenchmarksCases
.Select(b => b.Config.Orderer.GetLogicalGroupKey(summary.BenchmarksCases, b))
.ToArray();
FullContentStartOfLogicalGroup = new bool[summary.Reports.Length];
if (logicalGroupKeys.Distinct().Count() > 1 && FullContentStartOfLogicalGroup.Length > 0)
{
FullContentStartOfLogicalGroup[0] = true;
for (int i = 1; i < summary.Reports.Length; i++)
{
FullContentStartOfLogicalGroup[i] = logicalGroupKeys[i] != logicalGroupKeys[i - 1];
}
}
SeparateLogicalGroups = summary.Orderer.SeparateLogicalGroups;
var full = new List <string[]> {
FullHeader
};
full.AddRange(FullContent);
FullContentWithHeader = full.ToArray();
Columns = new SummaryTableColumn[columns.Length];
for (int i = 0; i < columns.Length; i++)
{
var column = columns[i];
bool hide = summary.ColumnHidingRules.Any(rule => rule.NeedToHide(column));
Columns[i] = new SummaryTableColumn(this, i, column, hide);
}
EffectiveSummaryStyle = style;
}
[PublicAPI] public string ToString(Encoding encoding)
{
var unit = TimeUnit.GetBestTimeUnit(Values);
return($"[{Lower.ToTimeStr(unit, encoding)};{Upper.ToTimeStr(unit, encoding)}) {{{string.Join("; ", Values.Select(v => v.ToTimeStr(unit, encoding)))}}}");
}
public static Func <double, string> CreateNanosecondFormatter(this Statistics s, CultureInfo cultureInfo, string format = "N3")
{
var timeUnit = TimeUnit.GetBestTimeUnit(s.Mean);
return(x => TimeInterval.FromNanoseconds(x).ToString(timeUnit, cultureInfo, format));
}
private void CheckGetBestTimeUnit(TimeUnit timeUnit, params double[] values)
{
output.WriteLine($"Best TimeUnit for ({string.Join(";", values)})ns is {timeUnit.Description}");
Assert.Equal(timeUnit.Name, TimeUnit.GetBestTimeUnit(values).Name);
}
public override string ToString()
{
var unit = TimeUnit.GetBestTimeUnit(Values);
return($"[{Lower.ToTimeStr(unit)};{Upper.ToTimeStr(unit)}) {{{string.Join("; ", Values.Select(v => v.ToTimeStr(unit)))}}}");
}
