public SamplesRequirement GetSampleSizeRequirement( BenchmarkResults.BeforeAndAfter basedOnPreliminaryResults) { if (basedOnPreliminaryResults.Baseline.ResultStatistics.N >= 30 && basedOnPreliminaryResults.Treatment.ResultStatistics.N >= 30) { Trace.WriteLine("In this scenario, you should use Z-test"); } { // Variances are determined by the preliminary results var size = TwoSampleTTestPowerAnalysis.GetSampleSize( variance1: basedOnPreliminaryResults.Baseline.ResultStatistics.Variance, variance2: basedOnPreliminaryResults.Treatment.ResultStatistics.Variance, alpha: this.alpha, delta: this.minimumDetectableDifferenceDesired, power: this.testStatisticalPower ); var n1 = (int)Math.Ceiling(size.Samples1); var n2 = (int)Math.Ceiling(size.Samples2); return(new SamplesRequirement( n1, n2)); } }
public void TTestPowerAnalysisConstructorTest4() { // Example from http://www.ats.ucla.edu/stat/stata/dae/t_test_power2.htm, // tested against G*Power results double meanA = 0; double meanB = 10; double sdA = 15; double sdB = 17; double varA = sdA * sdA; double varB = sdB * sdB; { var priori = TwoSampleTTestPowerAnalysis.GetSampleSize( 10, varA, varB, 1.0, 0.8, 0.05 ); Assert.AreEqual(41, Math.Truncate(priori.Samples1)); Assert.AreEqual(41, Math.Truncate(priori.Samples2)); } { TwoSampleTTest test = new TwoSampleTTest( meanA, varA, 30, meanB, varB, 30); Assert.AreEqual(0.661222, test.Analysis.Power, 1e-6); } { TwoSampleTTest test = new TwoSampleTTest( meanA, varA, 20, meanB, varB, 40); Assert.AreEqual(0.6102516, test.Analysis.Power, 1e-6); } { var priori = TwoSampleTTestPowerAnalysis.GetSampleSize( 10, varA, varB, 1.0, 0.8, 0.07 ); Assert.AreEqual(37, Math.Truncate(priori.Samples1)); Assert.AreEqual(37, Math.Truncate(priori.Samples2)); } }
public void TTestPowerAnalysisConstructorTest2() { // Example from R's graphical manual // http://rgm2.lab.nig.ac.jp/RGM2/func.php?rd_id=pwr:pwr.t2n.test double effectSize = 0.6; int n1 = 90; int n2 = 60; TwoSampleTTestPowerAnalysis target; double actual, expected; target = new TwoSampleTTestPowerAnalysis(TwoSampleHypothesis.FirstValueIsGreaterThanSecond) { Effect = effectSize, Samples1 = n1, Samples2 = n2, }; target.ComputePower(); expected = 0.9737262; actual = target.Power; Assert.AreEqual(expected, actual, 1e-6); target = new TwoSampleTTestPowerAnalysis(TwoSampleHypothesis.FirstValueIsSmallerThanSecond) { Effect = effectSize, Samples1 = n1, Samples2 = n2, }; target.ComputePower(); expected = 0.0; actual = target.Power; Assert.AreEqual(expected, actual, 1e-6); target = new TwoSampleTTestPowerAnalysis(TwoSampleHypothesis.ValuesAreDifferent) { Effect = effectSize, Samples1 = n1, Samples2 = n2, }; target.ComputePower(); expected = 0.9470154; actual = target.Power; Assert.AreEqual(expected, actual, 1e-6); }
public void TTestPowerAnalysisConstructorTest5() { var analysis = new TwoSampleTTestPowerAnalysis(TwoSampleHypothesis.ValuesAreDifferent); analysis.Effect = 5; analysis.Power = 0.95; analysis.ComputeSize(); Assert.AreEqual(0.12207248549844732, analysis.Size); Assert.AreEqual(2, analysis.Samples1); Assert.AreEqual(2, analysis.Samples2); }
public void TTestPowerAnalysisConstructorTest3() { // Examples from R's graphical manual // http://rgm2.lab.nig.ac.jp/RGM2/func.php?rd_id=pwr:pwr.t.test double actual, expected; { var target = new TTestPowerAnalysis(OneSampleHypothesis.ValueIsDifferentFromHypothesis) { Effect = 0.2, Samples = 60, Size = 0.10, }; target.ComputePower(); expected = 0.4555818; actual = target.Power; Assert.AreEqual(expected, actual, 1e-5); } { var target = new TwoSampleTTestPowerAnalysis(TwoSampleHypothesis.ValuesAreDifferent) { Effect = 2 / 2.8, Samples1 = 30, Samples2 = 30, }; target.ComputePower(); expected = 0.7764889; actual = target.Power; Assert.AreEqual(expected, actual, 1e-6); } { var target = new TwoSampleTTestPowerAnalysis(TwoSampleHypothesis.FirstValueIsGreaterThanSecond) { Effect = 0.3, Power = 0.75, }; target.ComputeSamples(); expected = 120.2232016; actual = target.Samples1; Assert.AreEqual(expected, actual, 1e-6); Assert.AreEqual(target.Samples1, target.Samples2); } }
private void power(double var1, double var2, int n1, int n2) { double stdDev = Math.Sqrt((var1 + var2) / 2.0); powerAnalysis = new TwoSampleTTestPowerAnalysis(Hypothesis) { Samples1 = n1, Samples2 = n2, Effect = (ObservedDifference - HypothesizedDifference) / stdDev, Size = Size, }; powerAnalysis.ComputePower(); }
public void TTestPowerAnalysisConstructorTest() { // Declare two samples double[] A = { 5.0, 6.0, 7.9, 6.95, 5.3, 10.0, 7.48, 9.4, 7.6, 8.0, 6.22 }; double[] B = { 5.0, 1.6, 5.75, 5.80, 2.9, 8.88, 4.56, 2.4, 5.0, 10.0 }; double meanA = A.Mean(); double meanB = B.Mean(); double varA = A.Variance(); double varB = B.Variance(); double sdA = A.StandardDeviation(); double sdB = B.StandardDeviation(); double sigma = Math.Sqrt((varA + varB) / 2.0); Assert.AreEqual(7.259, meanA, 1e-3); Assert.AreEqual(5.189, meanB, 1e-3); Assert.AreEqual(2.492289, varA, 1e-6); Assert.AreEqual(7.091476, varB, 1e-6); Assert.AreEqual(1.5786985, sdA, 1e-6); Assert.AreEqual(2.6629826, sdB, 1e-6); // Perform a hypothesis test TwoSampleTTest test = new TwoSampleTTest(A, B, assumeEqualVariances: false); Assert.AreEqual(14.351, test.DegreesOfFreedom, 1e-3); Assert.AreEqual(2.14, test.Statistic, 1e-3); Assert.AreEqual(0.04999, test.PValue, 1e-5); Assert.AreEqual(0.00013662, test.Confidence.Min, 1e-6); Assert.AreEqual(4.14004519, test.Confidence.Max, 1e-6); Assert.IsTrue(test.Significant); test = new TwoSampleTTest(A, B, assumeEqualVariances: true); Assert.AreEqual(19, test.DegreesOfFreedom, 1e-3); Assert.AreEqual(2.1921894, test.Statistic, 1e-3); Assert.AreEqual(0.0410, test.PValue, 1e-4); Assert.AreEqual(0.09364214, test.Confidence.Min, 1e-6); Assert.AreEqual(4.04653967, test.Confidence.Max, 1e-6); Assert.IsTrue(test.Significant); // Check the actual power of the test... Assert.AreEqual(0.5376260, test.Analysis.Power, 1e-6); // Check how much effect we are trying to detect Assert.AreEqual(0.9456628, test.Analysis.Effect, 1e-6); // So, what is the minimal difference we can detect? Assert.AreEqual(2.070090, test.Analysis.Effect * sigma, 1e-6); // Create an a posteriori analysis of the experiment var analysis = new TwoSampleTTestPowerAnalysis(test); analysis.Power = 0.8; // With 80% power, how much analysis.ComputeEffect(); // effect could we really detect? Assert.AreEqual(1.29051411, analysis.Effect, 1e-6); // Create an a priori power analysis so we can determine the sample // size needed to detect at least a difference of 2 points in the // student mean grades with at least 80% power: analysis = TwoSampleTTestPowerAnalysis.GetSampleSize(1, variance1: varA, variance2: varB, power: 0.8); Assert.AreEqual(0.4568219, analysis.Effect, 1e-6); // Check how many samples we would need to detect this effect with 80% power Assert.AreEqual(77, Math.Ceiling(analysis.Samples1)); Assert.AreEqual(77, Math.Ceiling(analysis.Samples2)); }
public void TTestPowerAnalysisConstructorTest() { // Let's say we have two samples, and we would like to know whether those // samples have the same mean. For this, we can perform a two sample T-Test: double[] A = { 5.0, 6.0, 7.9, 6.95, 5.3, 10.0, 7.48, 9.4, 7.6, 8.0, 6.22 }; double[] B = { 5.0, 1.6, 5.75, 5.80, 2.9, 8.88, 4.56, 2.4, 5.0, 10.0 }; // Perform the test, assuming the samples have unequal variances var test = new TwoSampleTTest(A, B, assumeEqualVariances: false); double df = test.DegreesOfFreedom; // d.f. = 14.351 double t = test.Statistic; // t = 2.14 double p = test.PValue; // p = 0.04999 bool significant = test.Significant; // true // The test gave us an indication that the samples may // indeed have come from different distributions (whose // mean value is actually distinct from each other). // Now, we would like to perform an _a posteriori_ analysis of the // test. When doing an a posteriori analysis, we can not change some // characteristics of the test (because it has been already done), // but we can measure some important features that may indicate // whether the test is trustworthy or not. // One of the first things would be to check for the test's power. // A test's power is 1 minus the probability of rejecting the null // hypothesis when the null hypothesis is actually false. It is // the other side of the coin when we consider that the P-value // is the probability of rejecting the null hypothesis when the // null hypothesis is actually true. // Ideally, this should be a high value: double power = test.Analysis.Power; // 0.5376260 // Check how much effect we are trying to detect double effect = test.Analysis.Effect; // 0.94566 // With this power, that is the minimal difference we can spot? double sigma = Math.Sqrt(test.Variance); double thres = test.Analysis.Effect * sigma; // 2.0700909090909 // This means that, using our test, the smallest difference that // we could detect with some confidence would be something around // 2 standard deviations. If we would like to say the samples are // different when they are less than 2 std. dev. apart, we would // need to do repeat our experiment differently. // Another way to create the power analysis is to pass // the test to the t-test power analysis constructor: // Create an a posteriori analysis of the experiment var analysis = new TwoSampleTTestPowerAnalysis(test); // When creating a power analysis, we have three things we can // change. We can always freely configure two of those things // and then ask the analysis to give us the third. // Those are: double e = analysis.Effect; // the test's minimum detectable effect size (0.94566) double n = analysis.TotalSamples; // the number of samples in the test (21 or (11 + 10)) double b = analysis.Power; // the probability of committing a type-2 error (0.53) // Let's say we would like to create a test with 80% power. analysis.Power = 0.8; analysis.ComputeEffect(); // what effect could we detect? double detectableEffect = analysis.Effect; // we would detect a difference of 1.290514 // However, to achieve this 80%, we would need to redo our experiment // more carefully. Assuming we are going to redo our experiment, we will // have more freedom about what we can change and what we can not. For // better addressing those points, we will create an a priori analysis // of the experiment: // We would like to know how many samples we would need to gather in // order to achieve a 80% power test which can detect an effect size // of one standard deviation: // analysis = TwoSampleTTestPowerAnalysis.GetSampleSize ( variance1: A.Variance(), variance2: B.Variance(), delta: 1.0, // the minimum detectable difference we want power: 0.8 // the test power that we want ); // How many samples would we need in order to see the effect we need? int n1 = (int)Math.Ceiling(analysis.Samples1); // 77 int n2 = (int)Math.Ceiling(analysis.Samples2); // 77 // According to our power analysis, we would need at least 77 // observations in each sample in order to see the effect we // need with the required 80% power. Assert.AreEqual(1.2905141186795861, detectableEffect); Assert.AreEqual(0.45682188621283815, analysis.Effect, 1e-6); Assert.AreEqual(2.0700909090909088, thres); Assert.AreEqual(0.53762605885988846, power); Assert.AreEqual(77, n1); Assert.AreEqual(77, n1); double meanA = A.Mean(); double meanB = B.Mean(); double varA = A.Variance(); double varB = B.Variance(); double sdA = A.StandardDeviation(); double sdB = B.StandardDeviation(); double sigma2 = Math.Sqrt((varA + varB) / 2.0); Assert.AreEqual(sigma2, sigma); Assert.AreEqual(7.259, meanA, 1e-3); Assert.AreEqual(5.189, meanB, 1e-3); Assert.AreEqual(2.492289, varA, 1e-6); Assert.AreEqual(7.091476, varB, 1e-6); Assert.AreEqual(1.5786985, sdA, 1e-6); Assert.AreEqual(2.6629826, sdB, 1e-6); Assert.AreEqual(14.351, df, 1e-3); Assert.AreEqual(2.14, t, 1e-3); Assert.AreEqual(0.04999, p, 1e-5); Assert.AreEqual(0.00013662, test.Confidence.Min, 1e-6); Assert.AreEqual(4.14004519, test.Confidence.Max, 1e-6); Assert.AreEqual(4.7918828787878791, test.Variance); Assert.IsTrue(test.Significant); Assert.AreEqual(0.5376260, test.Analysis.Power, 1e-6); Assert.AreEqual(0.9456628, test.Analysis.Effect, 1e-6); Assert.AreEqual(2.070090, test.Analysis.Effect * sigma, 1e-6); test = new TwoSampleTTest(A, B, assumeEqualVariances: true); Assert.AreEqual(19, test.DegreesOfFreedom, 1e-3); Assert.AreEqual(2.1921894, test.Statistic, 1e-3); Assert.AreEqual(0.0410, test.PValue, 1e-4); Assert.AreEqual(0.09364214, test.Confidence.Min, 1e-6); Assert.AreEqual(4.04653967, test.Confidence.Max, 1e-6); Assert.IsTrue(test.Significant); // Check the actual power of the test... Assert.AreEqual(0.5376260, test.Analysis.Power, 1e-6); // Check how much effect we are trying to detect Assert.AreEqual(0.9456628, test.Analysis.Effect, 1e-6); // So, what is the minimal difference we can detect? Assert.AreEqual(2.070090, test.Analysis.Effect * sigma, 1e-6); // Create an a posteriori analysis of the experiment analysis = new TwoSampleTTestPowerAnalysis(test); analysis.Power = 0.8; // With 80% power, how much analysis.ComputeEffect(); // effect could we really detect? Assert.AreEqual(1.29051411, analysis.Effect, 1e-6); // Create an a priori power analysis so we can determine the sample // size needed to detect at least a difference of 2 points in the // student mean grades with at least 80% power: analysis = TwoSampleTTestPowerAnalysis.GetSampleSize(1, variance1: varA, variance2: varB, power: 0.8); Assert.AreEqual(0.4568219, analysis.Effect, 1e-6); // Check how many samples we would need to detect this effect with 80% power Assert.AreEqual(77, Math.Ceiling(analysis.Samples1)); Assert.AreEqual(77, Math.Ceiling(analysis.Samples2)); }