Пример #1
0
        internal static void Compare(TwoInputsOptions args)
        {
            var notSame = GetNotSameResults(args).ToArray();

            if (!notSame.Any())
            {
                Console.WriteLine($"No differences found between the benchmark results with threshold {args.StatisticalTestThreshold}.");
                return;
            }

            PrintSummary(notSame);

            PrintTable(notSame, EquivalenceTestConclusion.Slower, args);
            PrintTable(notSame, EquivalenceTestConclusion.Faster, args);
        }
Пример #2
0
        private static IEnumerable <(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)> GetNotSameResults(TwoInputsOptions args)
        {
            foreach ((string id, Benchmark baseResult, Benchmark diffResult) in ReadResults(args)
                     .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures
            {
                var baseValues = baseResult.Statistics.OriginalValues.ToArray();
                var diffValues = diffResult.Statistics.OriginalValues.ToArray();

                var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold);
                if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same)
                {
                    continue;
                }

                var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.NoiseThreshold);
                if (noiseResult.Conclusion == EquivalenceTestConclusion.Same)
                {
                    continue;
                }

                yield return(id, baseResult, diffResult, userTresholdResult.Conclusion);
            }
        }