public void PowerTest1() { // Example from http://www.statpower.net/Content/310/Print%20Version%20--%20Power%20for%20the%202-Sample%20Z-Statistic.pdf double mean1 = 0.5; double mean2 = 0.0; double var1 = 0.5; double var2 = 0.5; int samples = 25; TwoSampleZTest test = new TwoSampleZTest( mean1, var1, samples, mean2, var2, samples); TwoSampleZTestPowerAnalysis pa = (TwoSampleZTestPowerAnalysis)test.Analysis; Assert.AreEqual(0.43, pa.Power, 0.01); pa.Power = 0.8; pa.ComputeSamples(); double expectedSamples = 63; double actualSamples = Math.Ceiling(2 * pa.Samples1); Assert.AreEqual(expectedSamples, actualSamples); }
public void TestMethod1() { var concurrent = new ConcurrentPrimes(); concurrent.Init(cMinPrime, cMaxPrime, cDegreeConcurrency); PerformanceResult concResult = PerformancePatterns.RunPerformanceTest(cMinPerfIterations, (() => concurrent.Execute())); Console.WriteLine(cResultFormat + " with concurrent collection.", cMinPerfIterations, concResult.TotalMilliseconds, concResult.TotalSeconds); var plinqed = new PlinqPrimes(); plinqed.Init(cMinPrime, cMaxPrime, cDegreeConcurrency); PerformanceResult pResult = PerformancePatterns.RunPerformanceTest(cMinPerfIterations, (() => plinqed.Execute())); Console.WriteLine(cResultFormat + " with plinq.", cMinPerfIterations, pResult.TotalMilliseconds, pResult.TotalSeconds); var comparison = new TwoSampleZTest(pResult.DescriptiveResult, concResult.DescriptiveResult, 0.0, TwoSampleHypothesis.FirstValueIsSmallerThanSecond); Console.WriteLine(concResult.ToString()); Console.WriteLine(pResult.ToString()); Assert.IsTrue(comparison.Significant); Assert.AreEqual(concurrent.Primes.Count,plinqed.Primes.Count); }
public void CompareMethods() { PerformanceResult resultWith = PerformancePatterns.RunPerformanceTest(cMinPerfIterations, (() => FuncWithLookup())); Console.WriteLine(cResultFormat + " with lookup.", cMinPerfIterations, resultWith.TotalMilliseconds, resultWith.TotalSeconds); PerformanceResult resultWithout = PerformancePatterns.RunPerformanceTest(cMinPerfIterations, (() => { var payids = (from payment in _bigList select payment.PayId).Distinct(); var myStuff = (from payid in payids.AsParallel() select new { PayId = payid, coverages = string.Join(", ", _bigList.Where(x => x.PayId == payid).Select(x => x.CategoryCode)), total = _bigList.Where(x => x.PayId == payid).Select(x => x.PaymentAmount).Sum() }).ToList(); var myList = myStuff.ToList(); })); Console.WriteLine(cResultFormat + " without lookup.", cMinPerfIterations, resultWithout.TotalMilliseconds, resultWithout.TotalSeconds); var comparison = new TwoSampleZTest(resultWith.DescriptiveResult, resultWithout.DescriptiveResult, 0.0, TwoSampleHypothesis.FirstValueIsSmallerThanSecond); Assert.IsTrue(comparison.Significant); }
public void TwoSampleZTestConstructorTest() { Accord.Math.Tools.SetupGenerator(0); double[] samples1 = new Accord.Statistics.Distributions.Univariate .NormalDistribution(29.8, 4.0).Generate(200); double[] samples2 = new Accord.Statistics.Distributions.Univariate .NormalDistribution(34.7, 5.0).Generate(250); TwoSampleZTest actual = new TwoSampleZTest(samples1, samples2); double mean1 = Accord.Statistics.Tools.Mean(samples1); double mean2 = Accord.Statistics.Tools.Mean(samples2); double var1 = Accord.Statistics.Tools.Variance(samples1); double var2 = Accord.Statistics.Tools.Variance(samples2); int n1 = samples1.Length; int n2 = samples2.Length; TwoSampleZTest expected = new TwoSampleZTest(mean1, var1, n1, mean2, var2, n2); Assert.AreEqual(expected.EstimatedValue1, actual.EstimatedValue1); Assert.AreEqual(expected.EstimatedValue2, actual.EstimatedValue2); Assert.AreEqual(expected.StandardError, actual.StandardError); Assert.AreEqual(expected.Statistic, actual.Statistic); Assert.IsTrue(actual.Significant); }
public void TwoSampleZTestConstructorTest2() { // Example from http://www.stat.purdue.edu/~tlzhang/stat511/chapter9_1.pdf double mean1 = 29.8; double var1 = System.Math.Pow(4.0, 2); int samples1 = 20; double mean2 = 34.7; double var2 = System.Math.Pow(5.0, 2); int samples2 = 25; TwoSampleZTest target = new TwoSampleZTest( mean1, var1, samples1, mean2, var2, samples2); Assert.AreEqual(mean1, target.EstimatedValue1); Assert.AreEqual(mean2, target.EstimatedValue2); Assert.AreEqual(-3.66, target.Statistic, 0.01); var range = target.GetConfidenceInterval(0.99); Assert.AreEqual(-8.36, range.Min, 0.01); Assert.AreEqual(-1.44, range.Max, 0.01); Assert.IsTrue(target.Significant); target.Size = 0.01; Assert.IsTrue(target.Significant); }
public void SampleSizeTest1() { // Example from http://udel.edu/~mcdonald/statttest.html double mean1 = 3.2; double mean2 = 0; double var1 = System.Math.Pow(4.0, 2); double var2 = System.Math.Pow(4.3, 2); double alpha = 0.05; double power = 0.80; TwoSampleZTest test = new TwoSampleZTest(mean1, var1, 10, mean2, var2, 10, alternate: TwoSampleHypothesis.ValuesAreDifferent); var target = (TwoSampleZTestPowerAnalysis)test.Analysis.Clone(); target.Power = power; target.Size = alpha; target.ComputeSamples(1); double actual = Math.Ceiling(target.Samples1); double expected = 27; Assert.AreEqual(expected, actual, 1e-3); }
/// <summary> /// Creates a new <see cref="ZTestPowerAnalysis"/>. /// </summary> /// /// <param name="test">The test to create the analysis for.</param> /// public ZTestPowerAnalysis(TwoSampleZTest test) : base(test.Tail) { this.Power = test.Analysis.Power; this.Size = test.Analysis.Size; this.Effect = test.Analysis.Effect; this.Samples = test.Analysis.Samples; }
public void TwoSampleZTestConstructorTest3() { // Example from Larser & Farber, Elementary Statistics: Picturing the world /* * A high school math teacher claims that students in her class * will score higher on the math portion of the ACT then students * in a colleague’s math class. The mean ACT math score for 49 * students in her class is 22.1 and the standard deviation is 4.8. * The mean ACT math score for 44 of the colleague’s students is * 19.8 and the standard deviation is 5.4. At a = 0.10, can the * teacher’s claim be supported? */ double mean1 = 22.1; double var1 = System.Math.Pow(4.8, 2); int samples1 = 49; double mean2 = 19.8; double var2 = System.Math.Pow(5.4, 2); int samples2 = 44; { TwoSampleZTest target = new TwoSampleZTest( mean1, var1, samples1, mean2, var2, samples2, alternate: TwoSampleHypothesis.FirstValueIsGreaterThanSecond); target.Size = 0.10; Assert.AreEqual(mean1, target.EstimatedValue1); Assert.AreEqual(mean2, target.EstimatedValue2); Assert.AreEqual(1.0644, target.StandardError, 0.0001); Assert.AreEqual(2.161, target.Statistic, 0.001); Assert.IsTrue(target.Significant); } { TwoSampleZTest target = new TwoSampleZTest( mean2, var2, samples2, mean1, var1, samples1, alternate: TwoSampleHypothesis.FirstValueIsSmallerThanSecond); target.Size = 0.10; Assert.AreEqual(mean2, target.EstimatedValue1); Assert.AreEqual(mean1, target.EstimatedValue2); Assert.AreEqual(1.0644, target.StandardError, 0.0001); Assert.AreEqual(-2.161, target.Statistic, 0.001); Assert.IsTrue(target.Significant); } }
public GeneralHypothesisTest(string firstLabel, DescriptiveResult sample1, string secondLabel, DescriptiveResult sample2, double hypothesizedDifference = 0, TwoSampleHypothesis alternate = TwoSampleHypothesis.ValuesAreDifferent) { int samples1 = sample1.Count; int samples2 = sample2.Count; HypothesizedDifference = Math.Abs(hypothesizedDifference); var s1 = new SampleInfo { Name = firstLabel, Count = sample1.Count, Mean = sample1.Mean, StdDev = sample1.StdDev }; var s2 = new SampleInfo { Name = secondLabel, Count = sample2.Count, Mean = sample2.Mean, StdDev = sample2.StdDev }; Result = new ComparisonResult { FirstSample = s1, SecondSample = s2, Hypothesis = alternate, HypothesizedDifference = HypothesizedDifference }; if (samples1 < 30 || samples2 < 30) { _tTest = new TwoSampleTTest(sample1, sample2, false, HypothesizedDifference, alternate); Result.Confidence = _tTest.Confidence; Result.ObservedDifference = _tTest.ObservedDifference; Result.Significant = _tTest.Significant; Result.Size = _tTest.Size; Result.StandardError = _tTest.StandardError; } else { _zTest = new TwoSampleZTest(sample1, sample2, HypothesizedDifference, alternate); Result.Confidence = _zTest.Confidence; Result.ObservedDifference = _zTest.ObservedDifference; Result.Significant = _zTest.Significant; Result.Size = _zTest.Size; Result.StandardError = _zTest.StandardError; } }
public void ExtraLargeIntKey() { InitIntData(cXLargeTableSize); RunIntTests(); var comparison = new TwoSampleZTest(_dictResult.DescriptiveResult, _tableResult.DescriptiveResult, 0.0, TwoSampleHypothesis.FirstValueIsSmallerThanSecond); Console.WriteLine(_dictResult.ToString()); Console.WriteLine(_tableResult.ToString()); Assert.IsTrue(comparison.Significant); }
public TwoSampleHypothesisTestResult TestHypothesis(IEnumerable <double> sample1, IEnumerable <double> sample2, double hypothesizedDifference, TwoSampleHypothesis alternateHypothesis, double alpha) { var test = new TwoSampleZTest( sample1.ToArray(), sample2.ToArray(), hypothesizedDifference: hypothesizedDifference, alternate: alternateHypothesis); test.Size = alpha; return(new TwoSampleHypothesisTestResult( test.Significant, test.GetConfidenceInterval(1 - alpha), test.ObservedDifference)); }
public void ExtraLargeIntKeyvStringKey() { InitSIData(cXLargeTableSize); RunSITests(); var comparison = new TwoSampleZTest(_dictResult.DescriptiveResult, _tableResult.DescriptiveResult, 0.0, TwoSampleHypothesis.ValuesAreDifferent); Console.WriteLine(_dictResult.ToString()); Console.WriteLine(_tableResult.ToString()); // Interesting view of "significant" here. A test with this many runs is almost always significant // statistically. But another read of "significant" is that sometimes this shows as a significant diff // and sometimes not. Depends on the run. Assert.IsFalse(comparison.Significant); }
public void TestMethod1() { PerformanceResult boxResult = PerformancePatterns.RunPerformanceTest(cMinPerfIterations, (() => { if (_boxList.Count < (cMinPerfIterations >> 2)) { _boxList.Add(_rng.Next()); } else { int i = _rng.Next(0, _boxList.Count - 1); var r = (int) _boxList[i]; _boxList.RemoveAt(i); } })); Console.WriteLine(cResultFormat + " with boxing.", cMinPerfIterations, boxResult.TotalMilliseconds, boxResult.TotalSeconds); PerformanceResult noboxResult = PerformancePatterns.RunPerformanceTest(cMinPerfIterations, (() => { if (_noboxList.Count < (cMinPerfIterations >> 2)) { _noboxList.Add(_rng.Next()); } else { int i = _rng.Next(0, _noboxList.Count - 1); int r = _noboxList[i]; _noboxList.RemoveAt(i); } })); Console.WriteLine(cResultFormat + " with generic collection.", cMinPerfIterations, noboxResult.TotalMilliseconds, noboxResult.TotalSeconds); var comparison = new TwoSampleZTest(noboxResult.DescriptiveResult, boxResult.DescriptiveResult, 0.0, TwoSampleHypothesis.FirstValueIsSmallerThanSecond); Console.WriteLine(noboxResult.ToString()); Console.WriteLine(boxResult.ToString()); Assert.IsTrue(comparison.Significant); }
public void Test1() { PerformanceResult contentionResult = PerformancePatterns.RunConcurrentPerformanceTest(cMinPerfIterations, cDegreeConcurrency, (() => { lock (_collectionLock) { int v = _rng.Next(1, cMinPerfIterations); if (_bigList.Count > (cMinPerfIterations >> 2)) { var s = _bigList[v%_bigList.Count]; _bigList.RemoveAt(v % _bigList.Count); } else { _bigList.Add(v); } } })); Console.WriteLine(cResultFormat + " with contention.", cMinPerfIterations, contentionResult.TotalMilliseconds, contentionResult.TotalSeconds); PerformanceResult concurrentResult = PerformancePatterns.RunConcurrentPerformanceTest(cMinPerfIterations, cDegreeConcurrency, (() => { int v = _rng.Next(1, cMinPerfIterations); if (_concurrentQ.Count > (cMinPerfIterations >> 2)) { var s = _concurrentQ.Take(); } else { _concurrentQ.Add(v); } })); Console.WriteLine(cResultFormat + " with concurrent collection.", cMinPerfIterations, concurrentResult.TotalMilliseconds, concurrentResult.TotalSeconds); var comparison = new TwoSampleZTest(concurrentResult.DescriptiveResult, contentionResult.DescriptiveResult, 0.0, TwoSampleHypothesis.FirstValueIsSmallerThanSecond); Console.WriteLine(concurrentResult.ToString()); Console.WriteLine(contentionResult.ToString()); Assert.IsTrue(comparison.Significant); }
public void TwoSampleZTestConstructorTest1() { // Example from http://www.stat.ucla.edu/~cochran/stat10/winter/lectures/lect21.html double mean1 = 9.78; double var1 = System.Math.Pow(4.05, 2); int samples1 = 900; double mean2 = 15.10; double var2 = System.Math.Pow(4.28, 2); int samples2 = 1000; TwoSampleZTest target = new TwoSampleZTest(mean1, var1, samples1, mean2, var2, samples2); Assert.AreEqual(mean1, target.EstimatedValue1); Assert.AreEqual(mean2, target.EstimatedValue2); Assert.AreEqual(0.19, target.StandardError, 0.005); Assert.AreEqual(-28, target.Statistic, 0.5); Assert.IsTrue(target.Significant); }
public SamplesRequirement GetSampleSizeRequirement(BenchmarkResults.BeforeAndAfter basedOnPreliminaryResults) { if (basedOnPreliminaryResults.Baseline.ResultStatistics.N < 30 || basedOnPreliminaryResults.Treatment.ResultStatistics.N < 30) { throw new InvalidOperationException( "Too few samples for Z test - please use T test"); } var test = new TwoSampleZTest( basedOnPreliminaryResults.Baseline.GetAverageNanosecondsForResultRuns(), basedOnPreliminaryResults.Treatment.GetAverageNanosecondsForResultRuns(), // TODO: P1 - Doing the tests separately like this and doing one tailed is not correct // but achieving the call syntax we want with the semantics statistics needs is hard :( // The specific problem is that the desired significance might not be achieved based on how this is done alternate: TwoSampleHypothesis.ValuesAreDifferent); Func <BaseTwoSamplePowerAnalysis, int> getSampleSizeForSample1 = analysis => (int)Math.Min(int.MaxValue, Math.Ceiling(analysis.Samples1)); // WORK AROUND FOR BUG IN ACCORD { // This was a weirdness in the Accord library - looks like a bug. We are going to work around it but validate it here in case it changes in the future. var originalAnalysis = test.Analysis.Clone() as TwoSampleZTestPowerAnalysis; var newAnalysis = test.Analysis as TwoSampleZTestPowerAnalysis; newAnalysis.Power = 0.80; newAnalysis.ComputeSamples(); var smallerPower = originalAnalysis.Power < newAnalysis.Power ? originalAnalysis : newAnalysis; var largerPower = smallerPower == newAnalysis ? originalAnalysis : newAnalysis; if (largerPower.Samples1 < smallerPower.Samples1) { // Not expected, but is the bug we are working around if (largerPower.TotalSamples > smallerPower.Samples1) { // Bug validated, our work around is okay getSampleSizeForSample1 = analysis => (int)Math.Min(int.MaxValue, Math.Ceiling(analysis.TotalSamples)); } else { throw new InvalidOperationException( "Larger power resulted in smaller sample size needed? Impossible."); } } else { getSampleSizeForSample1 = analysis => (int)Math.Min(int.MaxValue, Math.Ceiling(analysis.TotalSamples)); var version = FileVersionInfo.GetVersionInfo(typeof(BaseTwoSamplePowerAnalysis).Assembly.Location); if (version.FileMajorPart == 3 && version.FileMinorPart <= 8) { // Known version } else { throw new InvalidOperationException( $"It's possible you just need a lot more samples, but it's also possible our work around for a bug in Accord is no longer needed. Gotta check this! {smallerPower.Samples1} {largerPower.Samples1}"); } } } // WORK AROUND FOR BUG IN ACCORD // The difference standard deviation var standardDeviation = test.StandardError * Math.Sqrt(basedOnPreliminaryResults.Baseline.ResultStatistics.N); var size4 = TwoSampleZTestPowerAnalysis.GetSampleSize( // TODO: Does this delta need to be minimumDetectableDifferenceDesired, or do we use the observed difference? delta: test.ObservedDifference, power: this.testStatisticalPower, alpha: this.alpha, // TODO: P1 - Does the direction here matter? hypothesis: TwoSampleHypothesis.ValuesAreDifferent, standardDeviation: standardDeviation); var n1 = getSampleSizeForSample1(size4); return(new SamplesRequirement( (int)Math.Min(int.MaxValue, n1), (int)Math.Min(int.MaxValue, n1))); }