public void TTestPowerAnalysisConstructorTest()
{
// Declare two samples
double[] A = { 5.0, 6.0, 7.9, 6.95, 5.3, 10.0, 7.48, 9.4, 7.6, 8.0, 6.22 };
double[] B = { 5.0, 1.6, 5.75, 5.80, 2.9, 8.88, 4.56, 2.4, 5.0, 10.0 };
double meanA = A.Mean();
double meanB = B.Mean();
double varA = A.Variance();
double varB = B.Variance();
double sdA = A.StandardDeviation();
double sdB = B.StandardDeviation();
double sigma = Math.Sqrt((varA + varB) / 2.0);
Assert.AreEqual(7.259, meanA, 1e-3);
Assert.AreEqual(5.189, meanB, 1e-3);
Assert.AreEqual(2.492289, varA, 1e-6);
Assert.AreEqual(7.091476, varB, 1e-6);
Assert.AreEqual(1.5786985, sdA, 1e-6);
Assert.AreEqual(2.6629826, sdB, 1e-6);
// Perform a hypothesis test
TwoSampleTTest test = new TwoSampleTTest(A, B, assumeEqualVariances: false);
Assert.AreEqual(14.351, test.DegreesOfFreedom, 1e-3);
Assert.AreEqual(2.14, test.Statistic, 1e-3);
Assert.AreEqual(0.04999, test.PValue, 1e-5);
Assert.AreEqual(0.00013662, test.Confidence.Min, 1e-6);
Assert.AreEqual(4.14004519, test.Confidence.Max, 1e-6);
Assert.IsTrue(test.Significant);
test = new TwoSampleTTest(A, B, assumeEqualVariances: true);
Assert.AreEqual(19, test.DegreesOfFreedom, 1e-3);
Assert.AreEqual(2.1921894, test.Statistic, 1e-3);
Assert.AreEqual(0.0410, test.PValue, 1e-4);
Assert.AreEqual(0.09364214, test.Confidence.Min, 1e-6);
Assert.AreEqual(4.04653967, test.Confidence.Max, 1e-6);
Assert.IsTrue(test.Significant);
// Check the actual power of the test...
Assert.AreEqual(0.5376260, test.Analysis.Power, 1e-6);
// Check how much effect we are trying to detect
Assert.AreEqual(0.9456628, test.Analysis.Effect, 1e-6);
// So, what is the minimal difference we can detect?
Assert.AreEqual(2.070090, test.Analysis.Effect * sigma, 1e-6);
// Create an a posteriori analysis of the experiment
var analysis = new TwoSampleTTestPowerAnalysis(test);
analysis.Power = 0.8; // With 80% power, how much
analysis.ComputeEffect(); // effect could we really detect?
Assert.AreEqual(1.29051411, analysis.Effect, 1e-6);
// Create an a priori power analysis so we can determine the sample
// size needed to detect at least a difference of 2 points in the
// student mean grades with at least 80% power:
analysis = TwoSampleTTestPowerAnalysis.GetSampleSize(1,
variance1: varA, variance2: varB, power: 0.8);
Assert.AreEqual(0.4568219, analysis.Effect, 1e-6);
// Check how many samples we would need to detect this effect with 80% power
Assert.AreEqual(77, Math.Ceiling(analysis.Samples1));
Assert.AreEqual(77, Math.Ceiling(analysis.Samples2));
}
public void TTestPowerAnalysisConstructorTest()
{
// Let's say we have two samples, and we would like to know whether those
// samples have the same mean. For this, we can perform a two sample T-Test:
double[] A = { 5.0, 6.0, 7.9, 6.95, 5.3, 10.0, 7.48, 9.4, 7.6, 8.0, 6.22 };
double[] B = { 5.0, 1.6, 5.75, 5.80, 2.9, 8.88, 4.56, 2.4, 5.0, 10.0 };
// Perform the test, assuming the samples have unequal variances
var test = new TwoSampleTTest(A, B, assumeEqualVariances: false);
double df = test.DegreesOfFreedom; // d.f. = 14.351
double t = test.Statistic; // t = 2.14
double p = test.PValue; // p = 0.04999
bool significant = test.Significant; // true
// The test gave us an indication that the samples may
// indeed have come from different distributions (whose
// mean value is actually distinct from each other).
// Now, we would like to perform an _a posteriori_ analysis of the
// test. When doing an a posteriori analysis, we can not change some
// characteristics of the test (because it has been already done),
// but we can measure some important features that may indicate
// whether the test is trustworthy or not.
// One of the first things would be to check for the test's power.
// A test's power is 1 minus the probability of rejecting the null
// hypothesis when the null hypothesis is actually false. It is
// the other side of the coin when we consider that the P-value
// is the probability of rejecting the null hypothesis when the
// null hypothesis is actually true.
// Ideally, this should be a high value:
double power = test.Analysis.Power; // 0.5376260
// Check how much effect we are trying to detect
double effect = test.Analysis.Effect; // 0.94566
// With this power, that is the minimal difference we can spot?
double sigma = Math.Sqrt(test.Variance);
double thres = test.Analysis.Effect * sigma; // 2.0700909090909
// This means that, using our test, the smallest difference that
// we could detect with some confidence would be something around
// 2 standard deviations. If we would like to say the samples are
// different when they are less than 2 std. dev. apart, we would
// need to do repeat our experiment differently.
// Another way to create the power analysis is to pass
// the test to the t-test power analysis constructor:
// Create an a posteriori analysis of the experiment
var analysis = new TwoSampleTTestPowerAnalysis(test);
// When creating a power analysis, we have three things we can
// change. We can always freely configure two of those things
// and then ask the analysis to give us the third.
// Those are:
double e = analysis.Effect; // the test's minimum detectable effect size (0.94566)
double n = analysis.TotalSamples; // the number of samples in the test (21 or (11 + 10))
double b = analysis.Power; // the probability of committing a type-2 error (0.53)
// Let's say we would like to create a test with 80% power.
analysis.Power = 0.8;
analysis.ComputeEffect(); // what effect could we detect?
double detectableEffect = analysis.Effect; // we would detect a difference of 1.290514
// However, to achieve this 80%, we would need to redo our experiment
// more carefully. Assuming we are going to redo our experiment, we will
// have more freedom about what we can change and what we can not. For
// better addressing those points, we will create an a priori analysis
// of the experiment:
// We would like to know how many samples we would need to gather in
// order to achieve a 80% power test which can detect an effect size
// of one standard deviation:
//
analysis = TwoSampleTTestPowerAnalysis.GetSampleSize
(
variance1: A.Variance(),
variance2: B.Variance(),
delta: 1.0, // the minimum detectable difference we want
power: 0.8 // the test power that we want
);
// How many samples would we need in order to see the effect we need?
int n1 = (int)Math.Ceiling(analysis.Samples1); // 77
int n2 = (int)Math.Ceiling(analysis.Samples2); // 77
// According to our power analysis, we would need at least 77
// observations in each sample in order to see the effect we
// need with the required 80% power.
Assert.AreEqual(1.2905141186795861, detectableEffect);
Assert.AreEqual(0.45682188621283815, analysis.Effect, 1e-6);
Assert.AreEqual(2.0700909090909088, thres);
Assert.AreEqual(0.53762605885988846, power);
Assert.AreEqual(77, n1);
Assert.AreEqual(77, n1);
double meanA = A.Mean();
double meanB = B.Mean();
double varA = A.Variance();
double varB = B.Variance();
double sdA = A.StandardDeviation();
double sdB = B.StandardDeviation();
double sigma2 = Math.Sqrt((varA + varB) / 2.0);
Assert.AreEqual(sigma2, sigma);
Assert.AreEqual(7.259, meanA, 1e-3);
Assert.AreEqual(5.189, meanB, 1e-3);
Assert.AreEqual(2.492289, varA, 1e-6);
Assert.AreEqual(7.091476, varB, 1e-6);
Assert.AreEqual(1.5786985, sdA, 1e-6);
Assert.AreEqual(2.6629826, sdB, 1e-6);
Assert.AreEqual(14.351, df, 1e-3);
Assert.AreEqual(2.14, t, 1e-3);
Assert.AreEqual(0.04999, p, 1e-5);
Assert.AreEqual(0.00013662, test.Confidence.Min, 1e-6);
Assert.AreEqual(4.14004519, test.Confidence.Max, 1e-6);
Assert.AreEqual(4.7918828787878791, test.Variance);
Assert.IsTrue(test.Significant);
Assert.AreEqual(0.5376260, test.Analysis.Power, 1e-6);
Assert.AreEqual(0.9456628, test.Analysis.Effect, 1e-6);
Assert.AreEqual(2.070090, test.Analysis.Effect * sigma, 1e-6);
test = new TwoSampleTTest(A, B, assumeEqualVariances: true);
Assert.AreEqual(19, test.DegreesOfFreedom, 1e-3);
Assert.AreEqual(2.1921894, test.Statistic, 1e-3);
Assert.AreEqual(0.0410, test.PValue, 1e-4);
Assert.AreEqual(0.09364214, test.Confidence.Min, 1e-6);
Assert.AreEqual(4.04653967, test.Confidence.Max, 1e-6);
Assert.IsTrue(test.Significant);
// Check the actual power of the test...
Assert.AreEqual(0.5376260, test.Analysis.Power, 1e-6);
// Check how much effect we are trying to detect
Assert.AreEqual(0.9456628, test.Analysis.Effect, 1e-6);
// So, what is the minimal difference we can detect?
Assert.AreEqual(2.070090, test.Analysis.Effect * sigma, 1e-6);
// Create an a posteriori analysis of the experiment
analysis = new TwoSampleTTestPowerAnalysis(test);
analysis.Power = 0.8; // With 80% power, how much
analysis.ComputeEffect(); // effect could we really detect?
Assert.AreEqual(1.29051411, analysis.Effect, 1e-6);
// Create an a priori power analysis so we can determine the sample
// size needed to detect at least a difference of 2 points in the
// student mean grades with at least 80% power:
analysis = TwoSampleTTestPowerAnalysis.GetSampleSize(1,
variance1: varA, variance2: varB, power: 0.8);
Assert.AreEqual(0.4568219, analysis.Effect, 1e-6);
// Check how many samples we would need to detect this effect with 80% power
Assert.AreEqual(77, Math.Ceiling(analysis.Samples1));
Assert.AreEqual(77, Math.Ceiling(analysis.Samples2));
}