public void PowerTest1() { // Example from http://www.statpower.net/Content/310/Print%20Version%20--%20Power%20for%20the%202-Sample%20Z-Statistic.pdf double mean1 = 0.5; double mean2 = 0.0; double var1 = 0.5; double var2 = 0.5; int samples = 25; TwoSampleZTest test = new TwoSampleZTest( mean1, var1, samples, mean2, var2, samples); TwoSampleZTestPowerAnalysis pa = (TwoSampleZTestPowerAnalysis)test.Analysis; Assert.AreEqual(0.43, pa.Power, 0.01); pa.Power = 0.8; pa.ComputeSamples(); double expectedSamples = 63; double actualSamples = Math.Ceiling(2 * pa.Samples1); Assert.AreEqual(expectedSamples, actualSamples); }
private void power(double var1, double var2, int n1, int n2) { double stdDev = Math.Sqrt(var1 + var2); powerAnalysis = new TwoSampleZTestPowerAnalysis(Hypothesis) { Samples1 = n1, Samples2 = n2, Effect = (ObservedDifference - HypothesizedDifference) / stdDev, Size = Size, }; powerAnalysis.ComputePower(); }
public SamplesRequirement GetSampleSizeRequirement(BenchmarkResults.BeforeAndAfter basedOnPreliminaryResults) { if (basedOnPreliminaryResults.Baseline.ResultStatistics.N < 30 || basedOnPreliminaryResults.Treatment.ResultStatistics.N < 30) { throw new InvalidOperationException( "Too few samples for Z test - please use T test"); } var test = new TwoSampleZTest( basedOnPreliminaryResults.Baseline.GetAverageNanosecondsForResultRuns(), basedOnPreliminaryResults.Treatment.GetAverageNanosecondsForResultRuns(), // TODO: P1 - Doing the tests separately like this and doing one tailed is not correct // but achieving the call syntax we want with the semantics statistics needs is hard :( // The specific problem is that the desired significance might not be achieved based on how this is done alternate: TwoSampleHypothesis.ValuesAreDifferent); Func <BaseTwoSamplePowerAnalysis, int> getSampleSizeForSample1 = analysis => (int)Math.Min(int.MaxValue, Math.Ceiling(analysis.Samples1)); // WORK AROUND FOR BUG IN ACCORD { // This was a weirdness in the Accord library - looks like a bug. We are going to work around it but validate it here in case it changes in the future. var originalAnalysis = test.Analysis.Clone() as TwoSampleZTestPowerAnalysis; var newAnalysis = test.Analysis as TwoSampleZTestPowerAnalysis; newAnalysis.Power = 0.80; newAnalysis.ComputeSamples(); var smallerPower = originalAnalysis.Power < newAnalysis.Power ? originalAnalysis : newAnalysis; var largerPower = smallerPower == newAnalysis ? originalAnalysis : newAnalysis; if (largerPower.Samples1 < smallerPower.Samples1) { // Not expected, but is the bug we are working around if (largerPower.TotalSamples > smallerPower.Samples1) { // Bug validated, our work around is okay getSampleSizeForSample1 = analysis => (int)Math.Min(int.MaxValue, Math.Ceiling(analysis.TotalSamples)); } else { throw new InvalidOperationException( "Larger power resulted in smaller sample size needed? Impossible."); } } else { getSampleSizeForSample1 = analysis => (int)Math.Min(int.MaxValue, Math.Ceiling(analysis.TotalSamples)); var version = FileVersionInfo.GetVersionInfo(typeof(BaseTwoSamplePowerAnalysis).Assembly.Location); if (version.FileMajorPart == 3 && version.FileMinorPart <= 8) { // Known version } else { throw new InvalidOperationException( $"It's possible you just need a lot more samples, but it's also possible our work around for a bug in Accord is no longer needed. Gotta check this! {smallerPower.Samples1} {largerPower.Samples1}"); } } } // WORK AROUND FOR BUG IN ACCORD // The difference standard deviation var standardDeviation = test.StandardError * Math.Sqrt(basedOnPreliminaryResults.Baseline.ResultStatistics.N); var size4 = TwoSampleZTestPowerAnalysis.GetSampleSize( // TODO: Does this delta need to be minimumDetectableDifferenceDesired, or do we use the observed difference? delta: test.ObservedDifference, power: this.testStatisticalPower, alpha: this.alpha, // TODO: P1 - Does the direction here matter? hypothesis: TwoSampleHypothesis.ValuesAreDifferent, standardDeviation: standardDeviation); var n1 = getSampleSizeForSample1(size4); return(new SamplesRequirement( (int)Math.Min(int.MaxValue, n1), (int)Math.Min(int.MaxValue, n1))); }