private static void PrintTable(ComparerResult[] notSame, EquivalenceTestConclusion conclusion, CommandLineOptions args)
{
var data = notSame
.Where(result => result.Conclusion == conclusion)
.OrderByDescending(result => GetRatio(conclusion, result.BaseResult, result.DiffResult))
.Take(args.TopCount ?? int.MaxValue)
.Select(result => new
{
Id = (result.Id.Length <= 80 || args.FullId) ? result.Id : result.Id.Substring(0, 80),
DisplayValue = GetRatio(conclusion, result.BaseResult, result.DiffResult),
BaseMedian = result.BaseResult.Statistics.Median,
DiffMedian = result.DiffResult.Statistics.Median,
Modality = GetModalInfo(result.BaseResult) ?? GetModalInfo(result.DiffResult)
})
.ToArray();
if (!data.Any())
{
Console.WriteLine($"No {conclusion} results for the provided threshold = {args.StatisticalTestThreshold} and noise filter = {args.NoiseThreshold}.");
Console.WriteLine();
return;
}
Table table = data
.ToMarkdownTable()
.WithHeaders(conclusion.ToString(), conclusion == EquivalenceTestConclusion.Faster ? "base/diff" : "diff/base", "Base Median (ns)", "Diff Median (ns)", "Modality");
foreach (string line in table.ToMarkdown().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries))
{
Console.WriteLine($"| {line.TrimStart()}|"); // the table starts with \t and does not end with '|' and it looks bad so we fix it
}
Console.WriteLine();
}
internal void Update(EquivalenceTestConclusion conclusion)
{
Total++;
switch (conclusion)
{
case EquivalenceTestConclusion.Base:
case EquivalenceTestConclusion.Same:
Same++;
break;
case EquivalenceTestConclusion.Faster:
Faster++;
break;
case EquivalenceTestConclusion.Slower:
Slower++;
break;
case EquivalenceTestConclusion.Unknown:
Unknown++;
break;
case Stats.Noise:
Noise++;
break;
default:
throw new NotSupportedException($"Invalid conclusion! {conclusion}");
}
}
public CompareResult(string id, string runtime, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)
{
Id = id;
Runtime = runtime;
BaseResult = baseResult;
DiffResult = diffResult;
Conclusion = conclusion;
}
private string GetRatio(EquivalenceTestConclusion conclusion, Benchmark baseResult, Benchmark diffResult)
{
if (conclusion == EquivalenceTestConclusion.Same)
{
return("0.0%");
}
if (conclusion == EquivalenceTestConclusion.Unknown)
{
return("----");
}
if (baseResult.Statistics.Median == 0)
{
return("???");
}
var diff = (diffResult.Statistics.Median - baseResult.Statistics.Median) / baseResult.Statistics.Median * 100;
return($"{diff:0.00} %");
}
public TostResult(Threshold threshold, EquivalenceTestConclusion conclusion, [CanBeNull] T slowerTestResult, [CanBeNull] T fasterTestResult)
: base(threshold, conclusion)
{
SlowerTestResult = slowerTestResult;
FasterTestResult = fasterTestResult;
}
public EquivalenceTestResult(Threshold threshold, EquivalenceTestConclusion conclusion)
{
Threshold = threshold;
Conclusion = conclusion;
}
private static double GetRatio(EquivalenceTestConclusion conclusion, Benchmark baseResult, Benchmark diffResult)
=> conclusion == EquivalenceTestConclusion.Faster
? baseResult.Statistics.Median / diffResult.Statistics.Median
: diffResult.Statistics.Median / baseResult.Statistics.Median;
private static ComparisonConclusion TransformConclusion(EquivalenceTestConclusion conclusion) =>
conclusion == EquivalenceTestConclusion.Same ? ComparisonConclusion.Same
: conclusion == EquivalenceTestConclusion.Faster ? ComparisonConclusion.Better
: conclusion == EquivalenceTestConclusion.Slower ? ComparisonConclusion.Worse
: ComparisonConclusion.Unknown;
