private static void DummyNotStatisticalTest(AggregateMetrics first, AggregateMetrics second)
{
// TODO: Dummy NOT statistical test
var firstAvg = first.Runs.Average(run => run.ElapsedNanos);
var secondAvg = second.Runs.Average(run => run.ElapsedNanos);
PerfAssertContext.AssertIsTrue2(firstAvg < secondAvg, "Expected first to be faster (less time) than second");
}
private static void RealTest(AggregateMetrics first, AggregateMetrics second)
{
// Wilcoxon test to not assume normal distributions
var mannWhitneyWilcoxonTest = new MannWhitneyWilcoxonTest(
first.Runs.Select(run => run.ElapsedNanos).ToArray(),
second.Runs.Select(run => run.ElapsedNanos).ToArray(),
TwoSampleHypothesis.FirstValueIsSmallerThanSecond);
Trace.WriteLine(
"MannWWT s1 < s2: " + mannWhitneyWilcoxonTest.PValue +
" Stat1: " + mannWhitneyWilcoxonTest.Statistic1 + "; Stat2: " + mannWhitneyWilcoxonTest.Statistic2 +
" - Significant: " + mannWhitneyWilcoxonTest.Significant + " - Hyp: " + mannWhitneyWilcoxonTest.Hypothesis);
PerfAssertContext.AssertIsTrue(mannWhitneyWilcoxonTest.Significant);
}
public static void FirstDoesNotHaveHigherVariance(AggregateMetrics first, AggregateMetrics second)
{
double[] firstSamples = first.Runs.Select(run => run.ElapsedNanos).ToArray();
double[] secondSamples = second.Runs.Select(run => run.ElapsedNanos).ToArray();
// Fisher's F-test (also known as Snedecor)
var firstVariance = Measures.Variance(firstSamples);
var secondVariance = Measures.Variance(secondSamples);
var fishersFTest = new FTest(
firstVariance,
secondVariance,
firstSamples.Length - 1,
secondSamples.Length - 1,
TwoSampleHypothesis.FirstValueIsGreaterThanSecond);
Trace.WriteLine(
"FTest Var(s1) > Var(s2): " + fishersFTest.PValue +
" - Significant: " + fishersFTest.Significant + " - Hyp: " + fishersFTest.Hypothesis);
PerfAssertContext.AssertIsFalse(fishersFTest.Significant);
}
public static void FirstIsNotSlower(AggregateMetrics first, AggregateMetrics second)
{
double[] firstSamples = first.Runs.Select(run => run.ElapsedNanos).ToArray();
double[] secondSamples = second.Runs.Select(run => run.ElapsedNanos).ToArray();
// Wilcoxon test to not assume normal distributions
var mannWhitneyWilcoxonTest = new MannWhitneyWilcoxonTest(
firstSamples,
secondSamples,
TwoSampleHypothesis.FirstValueIsGreaterThanSecond);
Trace.WriteLine($"Sample 1 mean: {firstSamples.Average()} Sample 2 mean: {secondSamples.Average()}");
Trace.WriteLine(
"MannWWT s1 > s2: " + mannWhitneyWilcoxonTest.PValue +
" Stat1: " + mannWhitneyWilcoxonTest.Statistic1 + "; Stat2: " + mannWhitneyWilcoxonTest.Statistic2 +
" - Significant: " + mannWhitneyWilcoxonTest.Significant + " - Hyp: " + mannWhitneyWilcoxonTest.Hypothesis);
// We want to assert that we CANNOT statistically say that first > second (it can be less than, equal, or it might be greater than but not
// with statistical significance -- the only thing it CAN'T be, is KNOWN to be slower [>])
PerfAssertContext.AssertIsFalse(mannWhitneyWilcoxonTest.Significant);
}
