private static void DisplayDiff(IEnumerable <JobResults> allResults, IEnumerable <string> allNames)
{
// Use the first job results as the reference for metadata:
var firstJob = allResults.FirstOrDefault();
if (firstJob == null || firstJob.Jobs.Count < 1)
{
return;
}
foreach (var jobEntry in firstJob.Jobs)
{
var jobName = jobEntry.Key;
var jobResult = jobEntry.Value;
Console.WriteLine();
// description + baseline job (firstJob) + other jobs * 2 (value + percentage)
var table = new ResultTable(1 + 1 + (allNames.Count() - 1) * 2);
table.Headers.Add(jobName);
table.Headers.Add(allNames.First());
foreach (var name in allNames.Skip(1))
{
table.Headers.Add(name);
table.Headers.Add(""); // percentage
}
foreach (var metadata in jobResult.Metadata)
{
var metadataKey = metadata.Name.Split(';').First();
if (!jobResult.Results.ContainsKey(metadataKey))
{
continue;
}
// We don't render the result if it's a raw object
if (metadata.Format == "object")
{
continue;
}
var row = table.AddRow();
var cell = new Cell();
cell.Elements.Add(new CellElement()
{
Text = metadata.Description, Alignment = CellTextAlignment.Left
});
row.Add(cell);
foreach (var result in allResults)
{
// Skip jobs that have no data for this measure
if (!result.Jobs.ContainsKey(jobName))
{
foreach (var n in allNames)
{
row.Add(new Cell());
}
row.Add(new Cell());
continue;
}
var job = result.Jobs[jobName];
if (!String.IsNullOrEmpty(metadata.Format))
{
var measure = Convert.ToDouble(job.Results.ContainsKey(metadataKey) ? job.Results[metadataKey] : 0);
var previous = Convert.ToDouble(firstJob.Jobs[jobName].Results.ContainsKey(metadataKey) ? jobResult.Results[metadataKey] : 0);
var improvement = measure == 0
? 0
: (measure - previous) / previous * 100;
row.Add(cell = new Cell());
cell.Elements.Add(new CellElement {
Text = Convert.ToDouble(measure).ToString(metadata.Format), Alignment = CellTextAlignment.Right
});
// Don't render % on baseline job
if (firstJob != result)
{
row.Add(cell = new Cell());
if (measure != 0)
{
var sign = improvement > 0 ? "+" : "";
cell.Elements.Add(new CellElement {
Text = $"{sign}{improvement:n2}%", Alignment = CellTextAlignment.Right
});
}
}
}
else
{
var measure = job.Results.ContainsKey(metadataKey) ? job.Results[metadataKey] : 0;
row.Add(cell = new Cell());
cell.Elements.Add(new CellElement {
Text = measure.ToString(), Alignment = CellTextAlignment.Right
});
// Don't render % on baseline job
if (firstJob != result)
{
row.Add(new Cell());
}
}
}
}
table.Render(Console.Out);
Console.WriteLine();
}
}
private static void DisplayDiff(IEnumerable <Benchmark[]> allBenchmarks, IEnumerable <string> allNames)
{
// Use the first job's benchmarks as the reference for the rows:
var firstBenchmarks = allBenchmarks.FirstOrDefault();
if (firstBenchmarks == null || firstBenchmarks.Length < 1)
{
return;
}
var summaries = new Dictionary <string, List <BenchmarkSummary> >();
foreach (var benchmark in firstBenchmarks)
{
summaries[benchmark.FullName] = new List <BenchmarkSummary>();
}
foreach (var benchmarks in allBenchmarks)
{
foreach (var benchmark in benchmarks)
{
summaries[benchmark.FullName].Add(new BenchmarkSummary()
{
Name = benchmark.FullName,
MeanNanoseconds = benchmark.Statistics.Mean,
StandardErrorNanoseconds = benchmark.Statistics.StandardError,
StandardDeviationNanoseconds = benchmark.Statistics.StandardDeviation,
MedianNanoseconds = benchmark.Statistics.Median,
Gen0 = benchmark.Memory?.Gen0Collections ?? 0,
Gen1 = benchmark.Memory?.Gen1Collections ?? 0,
Gen2 = benchmark.Memory?.Gen2Collections ?? 0,
AllocatedBytes = benchmark.Memory?.BytesAllocatedPerOperation ?? 0
});
}
}
// Simplfy the benchmarks' names where possible to remove prefixes that
// are all the same to reduce the width of the first column of the table
// to the shortest unique string required across all benchmarks.
var nameSegments = summaries.Keys.ToDictionary(key => key, value => value.Split('.'));
while (true)
{
var areAllFirstSegmentsTheSame = nameSegments.Values
.Select(segments => segments[0])
.Distinct()
.Count() == 1;
if (!areAllFirstSegmentsTheSame)
{
// The names cannot be simplified further
break;
}
foreach (var pair in nameSegments)
{
nameSegments[pair.Key] = pair.Value.Skip(1).ToArray();
}
}
// Map the full names to their simplified name
var simplifiedNames = nameSegments.ToDictionary(key => key.Key, value => string.Join(".", value.Value));
var anyAllocations = summaries.Values
.SelectMany(list => list)
.Select(summary => summary.AllocatedBytes)
.Any(allocatedBytes => allocatedBytes > 0);
// Name + baseline mean (firstBenchmarks) + (other benchmarks' mean * 2 (value + ratio)) +
// baseline allocations + (other benchmarks' mean * 2 (value + ratio))
var otherCount = allNames.Count() - 1;
var table = new ResultTable(1 + 1 + (anyAllocations ? 1 : 0) + (otherCount * (anyAllocations ? 4 : 2)));
var firstName = allNames.First();
table.Headers.Add("benchmark");
table.Headers.Add($"mean ({firstName})");
foreach (var name in allNames.Skip(1))
{
table.Headers.Add($"mean ({name})");
table.Headers.Add("ratio");
}
if (anyAllocations)
{
table.Headers.Add($"allocated ({firstName})");
foreach (var name in allNames.Skip(1))
{
table.Headers.Add($"allocated ({name})");
table.Headers.Add("ratio");
}
}
foreach (var benchmark in summaries.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value))
{
var firstBenchmark = benchmark.First();
var simplifiedName = simplifiedNames[firstBenchmark.Name];
var benchmarks = summaries[firstBenchmark.Name];
var row = table.AddRow();
var cell = new Cell();
cell.Elements.Add(new CellElement()
{
Text = simplifiedName, Alignment = CellTextAlignment.Left
});
row.Add(cell);
AddCells(row, summary => summary.MeanNanoseconds, UnitType.Time, benchmarks);
if (anyAllocations)
{
AddCells(row, summary => summary.AllocatedBytes, UnitType.Size, benchmarks);
}
}
Console.WriteLine("```md"); // Format as a GitHub-flavored Markdown table
table.Render(Console.Out);
Console.WriteLine("```");
void AddCells(
List <Cell> row,
Func <BenchmarkSummary, object> valueFactory,
UnitType unitType,
List <BenchmarkSummary> summaries)
{
var rawValues = summaries
.Select(summary => valueFactory(summary))
.Select(value => Convert.ToDouble(value))
.ToArray();
var precision = PrecisionHelper.GetPrecision(rawValues);
var sizeUnit = unitType == UnitType.Size ? SizeUnit.GetBestSizeUnit(rawValues) : null;
var timeUnit = unitType == UnitType.Time ? TimeUnit.GetBestTimeUnit(rawValues) : null;
var units = unitType switch
{
UnitType.Size => sizeUnit.Name,
UnitType.Time => timeUnit.Name,
_ => string.Empty
};
var baseline = summaries[0];
for (var i = 0; i < summaries.Count; i++)
{
var measure = rawValues[i];
var previous = rawValues[0];
var ratio = measure == 0
? 0
: measure / previous;
var formattedValue = unitType switch
{
UnitType.Size => new SizeValue((long)measure).ToString(sizeUnit),
UnitType.Time => new TimeInterval(measure).ToString(timeUnit, precision),
_ => measure.ToString("N" + precision, CultureInfo.InvariantCulture)
};
var cell = new Cell();
cell.Elements.Add(new CellElement
{
Text = $"{formattedValue} {units}",
Alignment = CellTextAlignment.Right
});
row.Add(cell);
// Don't render the ratio on baseline benchmark
if (summaries[i] != baseline)
{
row.Add(cell = new Cell());
if (measure != 0)
{
cell.Elements.Add(new CellElement
{
Text = ratio.ToString("N2", CultureInfo.InvariantCulture),
Alignment = CellTextAlignment.Right
});
}
}
}
}
}