internal static void Compare(TwoInputsOptions args)
{
var notSame = GetNotSameResults(args).ToArray();
if (!notSame.Any())
{
Console.WriteLine($"No differences found between the benchmark results with threshold {args.StatisticalTestThreshold}.");
return;
}
PrintSummary(notSame);
PrintTable(notSame, EquivalenceTestConclusion.Slower, args);
PrintTable(notSame, EquivalenceTestConclusion.Faster, args);
}
private static IEnumerable <(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(TwoInputsOptions args)
{
foreach ((string id, Benchmark baseResult, Benchmark diffResult) in ReadResults(args)
.Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures
{
var baseValues = baseResult.Statistics.OriginalValues.ToArray();
var diffValues = diffResult.Statistics.OriginalValues.ToArray();
var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold);
if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same)
{
continue;
}
var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.NoiseThreshold);
if (noiseResult.Conclusion == EquivalenceTestConclusion.Same)
{
continue;
}
yield return(id, baseResult, diffResult, userTresholdResult.Conclusion);
}
}