public void PowerTest1()
{
// Example from http://www.statpower.net/Content/310/Print%20Version%20--%20Power%20for%20the%202-Sample%20Z-Statistic.pdf
double mean1 = 0.5;
double mean2 = 0.0;
double var1 = 0.5;
double var2 = 0.5;
int samples = 25;
TwoSampleZTest test = new TwoSampleZTest(
mean1, var1, samples,
mean2, var2, samples);
TwoSampleZTestPowerAnalysis pa = (TwoSampleZTestPowerAnalysis)test.Analysis;
Assert.AreEqual(0.43, pa.Power, 0.01);
pa.Power = 0.8;
pa.ComputeSamples();
double expectedSamples = 63;
double actualSamples = Math.Ceiling(2 * pa.Samples1);
Assert.AreEqual(expectedSamples, actualSamples);
}
private void power(double var1, double var2, int n1, int n2)
{
double stdDev = Math.Sqrt(var1 + var2);
powerAnalysis = new TwoSampleZTestPowerAnalysis(Hypothesis)
{
Samples1 = n1,
Samples2 = n2,
Effect = (ObservedDifference - HypothesizedDifference) / stdDev,
Size = Size,
};
powerAnalysis.ComputePower();
}
public SamplesRequirement GetSampleSizeRequirement(BenchmarkResults.BeforeAndAfter basedOnPreliminaryResults)
{
if (basedOnPreliminaryResults.Baseline.ResultStatistics.N < 30 ||
basedOnPreliminaryResults.Treatment.ResultStatistics.N < 30)
{
throw new InvalidOperationException(
"Too few samples for Z test - please use T test");
}
var test = new TwoSampleZTest(
basedOnPreliminaryResults.Baseline.GetAverageNanosecondsForResultRuns(),
basedOnPreliminaryResults.Treatment.GetAverageNanosecondsForResultRuns(),
// TODO: P1 - Doing the tests separately like this and doing one tailed is not correct
// but achieving the call syntax we want with the semantics statistics needs is hard :(
// The specific problem is that the desired significance might not be achieved based on how this is done
alternate: TwoSampleHypothesis.ValuesAreDifferent);
Func <BaseTwoSamplePowerAnalysis, int> getSampleSizeForSample1 = analysis => (int)Math.Min(int.MaxValue, Math.Ceiling(analysis.Samples1));
// WORK AROUND FOR BUG IN ACCORD
{
// This was a weirdness in the Accord library - looks like a bug. We are going to work around it but validate it here in case it changes in the future.
var originalAnalysis = test.Analysis.Clone() as TwoSampleZTestPowerAnalysis;
var newAnalysis = test.Analysis as TwoSampleZTestPowerAnalysis;
newAnalysis.Power = 0.80;
newAnalysis.ComputeSamples();
var smallerPower = originalAnalysis.Power < newAnalysis.Power ? originalAnalysis : newAnalysis;
var largerPower = smallerPower == newAnalysis ? originalAnalysis : newAnalysis;
if (largerPower.Samples1 < smallerPower.Samples1)
{
// Not expected, but is the bug we are working around
if (largerPower.TotalSamples > smallerPower.Samples1)
{
// Bug validated, our work around is okay
getSampleSizeForSample1 = analysis => (int)Math.Min(int.MaxValue, Math.Ceiling(analysis.TotalSamples));
}
else
{
throw new InvalidOperationException(
"Larger power resulted in smaller sample size needed? Impossible.");
}
}
else
{
getSampleSizeForSample1 = analysis => (int)Math.Min(int.MaxValue, Math.Ceiling(analysis.TotalSamples));
var version = FileVersionInfo.GetVersionInfo(typeof(BaseTwoSamplePowerAnalysis).Assembly.Location);
if (version.FileMajorPart == 3 && version.FileMinorPart <= 8)
{
// Known version
}
else
{
throw new InvalidOperationException(
$"It's possible you just need a lot more samples, but it's also possible our work around for a bug in Accord is no longer needed. Gotta check this! {smallerPower.Samples1} {largerPower.Samples1}");
}
}
}
// WORK AROUND FOR BUG IN ACCORD
// The difference standard deviation
var standardDeviation = test.StandardError * Math.Sqrt(basedOnPreliminaryResults.Baseline.ResultStatistics.N);
var size4 = TwoSampleZTestPowerAnalysis.GetSampleSize(
// TODO: Does this delta need to be minimumDetectableDifferenceDesired, or do we use the observed difference?
delta: test.ObservedDifference,
power: this.testStatisticalPower,
alpha: this.alpha,
// TODO: P1 - Does the direction here matter?
hypothesis: TwoSampleHypothesis.ValuesAreDifferent,
standardDeviation: standardDeviation);
var n1 = getSampleSizeForSample1(size4);
return(new SamplesRequirement(
(int)Math.Min(int.MaxValue, n1),
(int)Math.Min(int.MaxValue, n1)));
}
