private static bool AreSame(Statistics x, Statistics y)
{
if (x.N >= 3 && y.N >= 3)
{
var tost = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x.SortedValues.ToArray(), y.SortedValues.ToArray(), RelativeThreshold.Default);
if (tost.Conclusion != EquivalenceTestConclusion.Unknown)
{
return(tost.Conclusion == EquivalenceTestConclusion.Same);
}
}
return(Math.Abs(x.Mean - y.Mean) < Math.Abs(x.Mean + y.Mean) / 2 * 0.01);
}
protected override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, Statistics current, bool isBaseline)
{
var x = baseline.GetValues().ToArray();
var y = current.GetValues().ToArray();
switch (Kind)
{
case StatisticalTestKind.Welch:
return(StatisticalTestHelper.CalculateTost(WelchTest.Instance, x, y, Threshold).ToStr(ShowPValues));
case StatisticalTestKind.MannWhitney:
return(StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x, y, Threshold).ToStr(ShowPValues));
default:
throw new ArgumentOutOfRangeException();
}
}
public override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, IReadOnlyDictionary <string, Metric> baselineMetrics,
Statistics current, IReadOnlyDictionary <string, Metric> currentMetrics, bool isBaseline)
{
var x = baseline.OriginalValues.ToArray();
var y = current.OriginalValues.ToArray();
switch (Kind)
{
case StatisticalTestKind.Welch:
return(StatisticalTestHelper.CalculateTost(WelchTest.Instance, x, y, Threshold).ToString(ShowPValues));
case StatisticalTestKind.MannWhitney:
return(StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x, y, Threshold).ToString(ShowPValues));
default:
throw new ArgumentOutOfRangeException();
}
}
private static IEnumerable <(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(CommandLineOptions args, Threshold testThreshold, Threshold noiseThreshold)
{
foreach ((string id, Benchmark baseResult, Benchmark diffResult) in ReadResults(args)
.Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures
{
var baseValues = baseResult.GetOriginalValues();
var diffValues = diffResult.GetOriginalValues();
var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, testThreshold);
if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same)
{
continue;
}
var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, noiseThreshold);
if (noiseResult.Conclusion == EquivalenceTestConclusion.Same)
{
continue;
}
yield return(id, baseResult, diffResult, userTresholdResult.Conclusion);
}
}
private static IEnumerable <(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(string basePath, string diffPath, ComparerOptions options, Threshold testThreshold, Threshold noiseThreshold)
{
foreach ((string id, Benchmark baseResult, Benchmark diffResult) in ReadResults(basePath, diffPath, options)
.Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures
{
double[] baseValues = baseResult.GetOriginalValues();
double[] diffValues = diffResult.GetOriginalValues();
TostResult <MannWhitneyResult> userThresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, testThreshold);
if (userThresholdResult.Conclusion == EquivalenceTestConclusion.Same)
{
continue;
}
TostResult <MannWhitneyResult> noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, noiseThreshold);
if (noiseResult.Conclusion == EquivalenceTestConclusion.Same)
{
continue;
}
yield return(id, baseResult, diffResult, userThresholdResult.Conclusion);
}
}
public IEnumerable <CompareResult> Compare(string baseFile, string diffFile, IEnumerable <string> filters)
{
foreach (var pair in ReadResults(baseFile, diffFile, filters))
{
if (pair.baseResult.Statistics == null || pair.diffResult.Statistics == null)
{
yield return(new CompareResult(pair.id, pair.runtime, pair.baseResult, pair.diffResult, EquivalenceTestConclusion.Unknown));
}
var baseValues = pair.baseResult.GetOriginalValues();
var diffValues = pair.diffResult.GetOriginalValues();
var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, _testThreshold);
var conclusion = userTresholdResult.Conclusion;
if (userTresholdResult.Conclusion != EquivalenceTestConclusion.Same)
{
var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, _noiseThreshold);
conclusion = noiseResult.Conclusion;
}
yield return(new CompareResult(pair.id, pair.runtime, pair.baseResult, pair.diffResult, conclusion));
}
}