private static bool AreSame(Statistics x, Statistics y) { if (x.N >= 3 && y.N >= 3) { var tost = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x.SortedValues.ToArray(), y.SortedValues.ToArray(), RelativeThreshold.Default); if (tost.Conclusion != EquivalenceTestConclusion.Unknown) { return(tost.Conclusion == EquivalenceTestConclusion.Same); } } return(Math.Abs(x.Mean - y.Mean) < Math.Abs(x.Mean + y.Mean) / 2 * 0.01); }
protected override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, Statistics current, bool isBaseline) { var x = baseline.GetValues().ToArray(); var y = current.GetValues().ToArray(); switch (Kind) { case StatisticalTestKind.Welch: return(StatisticalTestHelper.CalculateTost(WelchTest.Instance, x, y, Threshold).ToStr(ShowPValues)); case StatisticalTestKind.MannWhitney: return(StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x, y, Threshold).ToStr(ShowPValues)); default: throw new ArgumentOutOfRangeException(); } }
public override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, IReadOnlyDictionary <string, Metric> baselineMetrics, Statistics current, IReadOnlyDictionary <string, Metric> currentMetrics, bool isBaseline) { var x = baseline.OriginalValues.ToArray(); var y = current.OriginalValues.ToArray(); switch (Kind) { case StatisticalTestKind.Welch: return(StatisticalTestHelper.CalculateTost(WelchTest.Instance, x, y, Threshold).ToString(ShowPValues)); case StatisticalTestKind.MannWhitney: return(StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x, y, Threshold).ToString(ShowPValues)); default: throw new ArgumentOutOfRangeException(); } }
private static IEnumerable <(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(CommandLineOptions args, Threshold testThreshold, Threshold noiseThreshold) { foreach ((string id, Benchmark baseResult, Benchmark diffResult) in ReadResults(args) .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures { var baseValues = baseResult.GetOriginalValues(); var diffValues = diffResult.GetOriginalValues(); var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, testThreshold); if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same) { continue; } var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, noiseThreshold); if (noiseResult.Conclusion == EquivalenceTestConclusion.Same) { continue; } yield return(id, baseResult, diffResult, userTresholdResult.Conclusion); } }
private static IEnumerable <(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(string basePath, string diffPath, ComparerOptions options, Threshold testThreshold, Threshold noiseThreshold) { foreach ((string id, Benchmark baseResult, Benchmark diffResult) in ReadResults(basePath, diffPath, options) .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures { double[] baseValues = baseResult.GetOriginalValues(); double[] diffValues = diffResult.GetOriginalValues(); TostResult <MannWhitneyResult> userThresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, testThreshold); if (userThresholdResult.Conclusion == EquivalenceTestConclusion.Same) { continue; } TostResult <MannWhitneyResult> noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, noiseThreshold); if (noiseResult.Conclusion == EquivalenceTestConclusion.Same) { continue; } yield return(id, baseResult, diffResult, userThresholdResult.Conclusion); } }
public IEnumerable <CompareResult> Compare(string baseFile, string diffFile, IEnumerable <string> filters) { foreach (var pair in ReadResults(baseFile, diffFile, filters)) { if (pair.baseResult.Statistics == null || pair.diffResult.Statistics == null) { yield return(new CompareResult(pair.id, pair.runtime, pair.baseResult, pair.diffResult, EquivalenceTestConclusion.Unknown)); } var baseValues = pair.baseResult.GetOriginalValues(); var diffValues = pair.diffResult.GetOriginalValues(); var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, _testThreshold); var conclusion = userTresholdResult.Conclusion; if (userTresholdResult.Conclusion != EquivalenceTestConclusion.Same) { var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, _noiseThreshold); conclusion = noiseResult.Conclusion; } yield return(new CompareResult(pair.id, pair.runtime, pair.baseResult, pair.diffResult, conclusion)); } }