Exemplo n.º 1
0
        private static bool AreSame(Statistics x, Statistics y)
        {
            if (x.N >= 3 && y.N >= 3)
            {
                var tost = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x.SortedValues.ToArray(), y.SortedValues.ToArray(), RelativeThreshold.Default);
                if (tost.Conclusion != EquivalenceTestConclusion.Unknown)
                {
                    return(tost.Conclusion == EquivalenceTestConclusion.Same);
                }
            }

            return(Math.Abs(x.Mean - y.Mean) < Math.Abs(x.Mean + y.Mean) / 2 * 0.01);
        }
Exemplo n.º 2
0
        protected override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, Statistics current, bool isBaseline)
        {
            var x = baseline.GetValues().ToArray();
            var y = current.GetValues().ToArray();

            switch (Kind)
            {
            case StatisticalTestKind.Welch:
                return(StatisticalTestHelper.CalculateTost(WelchTest.Instance, x, y, Threshold).ToStr(ShowPValues));

            case StatisticalTestKind.MannWhitney:
                return(StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x, y, Threshold).ToStr(ShowPValues));

            default:
                throw new ArgumentOutOfRangeException();
            }
        }
Exemplo n.º 3
0
        public override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, IReadOnlyDictionary <string, Metric> baselineMetrics,
                                        Statistics current, IReadOnlyDictionary <string, Metric> currentMetrics, bool isBaseline)
        {
            var x = baseline.OriginalValues.ToArray();
            var y = current.OriginalValues.ToArray();

            switch (Kind)
            {
            case StatisticalTestKind.Welch:
                return(StatisticalTestHelper.CalculateTost(WelchTest.Instance, x, y, Threshold).ToString(ShowPValues));

            case StatisticalTestKind.MannWhitney:
                return(StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x, y, Threshold).ToString(ShowPValues));

            default:
                throw new ArgumentOutOfRangeException();
            }
        }
Exemplo n.º 4
0
        private static IEnumerable <(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(CommandLineOptions args, Threshold testThreshold, Threshold noiseThreshold)
        {
            foreach ((string id, Benchmark baseResult, Benchmark diffResult) in ReadResults(args)
                     .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures
            {
                var baseValues = baseResult.GetOriginalValues();
                var diffValues = diffResult.GetOriginalValues();

                var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, testThreshold);
                if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same)
                {
                    continue;
                }

                var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, noiseThreshold);
                if (noiseResult.Conclusion == EquivalenceTestConclusion.Same)
                {
                    continue;
                }

                yield return(id, baseResult, diffResult, userTresholdResult.Conclusion);
            }
        }
Exemplo n.º 5
0
        private static IEnumerable <(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(string basePath, string diffPath, ComparerOptions options, Threshold testThreshold, Threshold noiseThreshold)
        {
            foreach ((string id, Benchmark baseResult, Benchmark diffResult) in ReadResults(basePath, diffPath, options)
                     .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures
            {
                double[] baseValues = baseResult.GetOriginalValues();
                double[] diffValues = diffResult.GetOriginalValues();

                TostResult <MannWhitneyResult> userThresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, testThreshold);
                if (userThresholdResult.Conclusion == EquivalenceTestConclusion.Same)
                {
                    continue;
                }

                TostResult <MannWhitneyResult> noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, noiseThreshold);
                if (noiseResult.Conclusion == EquivalenceTestConclusion.Same)
                {
                    continue;
                }

                yield return(id, baseResult, diffResult, userThresholdResult.Conclusion);
            }
        }
Exemplo n.º 6
0
        public IEnumerable <CompareResult> Compare(string baseFile, string diffFile, IEnumerable <string> filters)
        {
            foreach (var pair in ReadResults(baseFile, diffFile, filters))
            {
                if (pair.baseResult.Statistics == null || pair.diffResult.Statistics == null)
                {
                    yield return(new CompareResult(pair.id, pair.runtime, pair.baseResult, pair.diffResult, EquivalenceTestConclusion.Unknown));
                }

                var baseValues = pair.baseResult.GetOriginalValues();
                var diffValues = pair.diffResult.GetOriginalValues();

                var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, _testThreshold);
                var conclusion         = userTresholdResult.Conclusion;
                if (userTresholdResult.Conclusion != EquivalenceTestConclusion.Same)
                {
                    var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, _noiseThreshold);
                    conclusion = noiseResult.Conclusion;
                }

                yield return(new CompareResult(pair.id, pair.runtime, pair.baseResult, pair.diffResult, conclusion));
            }
        }