/// <summary>
/// Create a new running statistics over the combined samples of two existing running statistics.
/// </summary>
public static RunningStatistics Combine(RunningStatistics a, RunningStatistics b)
{
if (a._n == 0)
{
return(b);
}
else if (b._n == 0)
{
return(a);
}
long n = a._n + b._n;
double d = b._m1 - a._m1;
double d2 = d * d;
double d3 = d2 * d;
double d4 = d2 * d2;
double m1 = (a._n * a._m1 + b._n * b._m1) / n;
double m2 = a._m2 + b._m2 + d2 * a._n * b._n / n;
double m3 = a._m3 + b._m3 + d3 * a._n * b._n * (a._n - b._n) / (n * n)
+ 3 * d * (a._n * b._m2 - b._n * a._m2) / n;
double m4 = a._m4 + b._m4 + d4 * a._n * b._n * (a._n * a._n - a._n * b._n + b._n * b._n) / (n * n * n)
+ 6 * d2 * (a._n * a._n * b._m2 + b._n * b._n * a._m2) / (n * n) + 4 * d * (a._n * b._m3 - b._n * a._m3) / n;
double min = Math.Min(a._min, b._min);
double max = Math.Max(a._max, b._max);
return(new RunningStatistics {
_n = n, _m1 = m1, _m2 = m2, _m3 = m3, _m4 = m4, _min = min, _max = max
});
}
public void RunningStatisticsWithInfinityNaNDataContractSerializationTest()
{
var expected = new RunningStatistics(new[] { 1.0, 2.0, 3.0, double.PositiveInfinity, double.NaN });
var serializer = new DataContractSerializer(typeof(RunningStatistics));
var stream = new MemoryStream();
serializer.WriteObject(stream, expected);
stream.Position = 0;
var actual = (RunningStatistics)serializer.ReadObject(stream);
Assert.That(actual.Count, Is.EqualTo(expected.Count));
Assert.That(actual.Maximum, Is.EqualTo(expected.Maximum));
Assert.That(actual.Mean, Is.EqualTo(expected.Mean));
}
/// <summary>
/// Create a new running statistics over the combined samples of two existing running statistics.
/// </summary>
public static RunningStatistics Combine(RunningStatistics a, RunningStatistics b)
{
long n = a._n + b._n;
double d = b._m1 - a._m1;
double d2 = d * d;
double d3 = d2 * d;
double d4 = d2 * d2;
double m1 = (a._n * a._m1 + b._n * b._m1) / n;
double m2 = a._m2 + b._m2 + d2 * a._n * b._n / n;
double m3 = a._m3 + b._m3 + d3 * a._n * b._n * (a._n - b._n) / (n * n)
+ 3 * d * (a._n * b._m2 - b._n * a._m2) / n;
double m4 = a._m4 + b._m4 + d4 * a._n * b._n * (a._n * a._n - a._n * b._n + b._n * b._n) / (n * n * n)
+ 6 * d2 * (a._n * a._n * b._m2 + b._n * b._n * a._m2) / (n * n) + 4 * d * (a._n * b._m3 - b._n * a._m3) / n;
return(new RunningStatistics {
_n = n, _m1 = m1, _m2 = m2, _m3 = m3, _m4 = m4
});
}
/// <summary>
/// Evaluates the skewness and kurtosis from the full population.
/// Does not use a normalizer and would thus be biased if applied to a subset (type 1).
/// </summary>
/// <param name="population">The full population data.</param>
public static Tuple <double, double> PopulationSkewnessKurtosis(this IEnumerable <double> population)
{
var stats = new RunningStatistics(population);
return(new Tuple <double, double>(stats.PopulationSkewness, stats.PopulationKurtosis));
}
/// <summary>
/// Estimates the unbiased population skewness and kurtosis from the provided samples in a single pass.
/// Uses a normalizer (Bessel's correction; type 2).
/// </summary>
/// <param name="samples">A subset of samples, sampled from the full population.</param>
public static Tuple <double, double> SkewnessKurtosis(this IEnumerable <double> samples)
{
var stats = new RunningStatistics(samples);
return(new Tuple <double, double>(stats.Skewness, stats.Kurtosis));
}
public void ConsistentWithNist(string dataSet, int digits, double skewness, double kurtosis, double median, double min, double max, int count)
{
var data = _data[dataSet];
var stats = new RunningStatistics(data.Data);
AssertHelpers.AlmostEqualRelative(data.Mean, stats.Mean, 10);
AssertHelpers.AlmostEqualRelative(data.StandardDeviation, stats.StandardDeviation, digits);
AssertHelpers.AlmostEqualRelative(skewness, stats.Skewness, 8);
AssertHelpers.AlmostEqualRelative(kurtosis, stats.Kurtosis, 8);
Assert.AreEqual(stats.Minimum, min);
Assert.AreEqual(stats.Maximum, max);
Assert.AreEqual(stats.Count, count);
}
public void Combine()
{
var rnd = new SystemRandomSource(10);
var a = Generate.Random(200, new Erlang(2, 0.2, rnd));
var b = Generate.Random(100, new Beta(1.2, 1.4, rnd));
var c = Generate.Random(150, new Rayleigh(0.8, rnd));
var d = a.Concat(b).Concat(c).ToArray();
var x = new RunningStatistics(d);
var y = new RunningStatistics(a);
y.PushRange(b);
y.PushRange(c);
var za = new RunningStatistics(a);
var zb = new RunningStatistics(b);
var zc = new RunningStatistics(c);
var z = za + zb + zc;
Assert.That(x.Mean, Is.EqualTo(d.Mean()).Within(1e-12), "Mean Reference");
Assert.That(y.Mean, Is.EqualTo(x.Mean).Within(1e-12), "Mean y");
Assert.That(z.Mean, Is.EqualTo(x.Mean).Within(1e-12), "Mean z");
Assert.That(x.Variance, Is.EqualTo(d.Variance()).Within(1e-12), "Variance Reference");
Assert.That(y.Variance, Is.EqualTo(x.Variance).Within(1e-12), "Variance y");
Assert.That(z.Variance, Is.EqualTo(x.Variance).Within(1e-12), "Variance z");
Assert.That(x.PopulationVariance, Is.EqualTo(d.PopulationVariance()).Within(1e-12), "PopulationVariance Reference");
Assert.That(y.PopulationVariance, Is.EqualTo(x.PopulationVariance).Within(1e-12), "PopulationVariance y");
Assert.That(z.PopulationVariance, Is.EqualTo(x.PopulationVariance).Within(1e-12), "PopulationVariance z");
Assert.That(x.StandardDeviation, Is.EqualTo(d.StandardDeviation()).Within(1e-12), "StandardDeviation Reference");
Assert.That(y.StandardDeviation, Is.EqualTo(x.StandardDeviation).Within(1e-12), "StandardDeviation y");
Assert.That(z.StandardDeviation, Is.EqualTo(x.StandardDeviation).Within(1e-12), "StandardDeviation z");
Assert.That(x.PopulationStandardDeviation, Is.EqualTo(d.PopulationStandardDeviation()).Within(1e-12), "PopulationStandardDeviation Reference");
Assert.That(y.PopulationStandardDeviation, Is.EqualTo(x.PopulationStandardDeviation).Within(1e-12), "PopulationStandardDeviation y");
Assert.That(z.PopulationStandardDeviation, Is.EqualTo(x.PopulationStandardDeviation).Within(1e-12), "PopulationStandardDeviation z");
Assert.That(x.Skewness, Is.EqualTo(d.Skewness()).Within(1e-12), "Skewness Reference (not independent!)");
Assert.That(y.Skewness, Is.EqualTo(x.Skewness).Within(1e-12), "Skewness y");
Assert.That(z.Skewness, Is.EqualTo(x.Skewness).Within(1e-12), "Skewness z");
Assert.That(x.PopulationSkewness, Is.EqualTo(d.PopulationSkewness()).Within(1e-12), "PopulationSkewness Reference (not independent!)");
Assert.That(y.PopulationSkewness, Is.EqualTo(x.PopulationSkewness).Within(1e-12), "PopulationSkewness y");
Assert.That(z.PopulationSkewness, Is.EqualTo(x.PopulationSkewness).Within(1e-12), "PopulationSkewness z");
Assert.That(x.Kurtosis, Is.EqualTo(d.Kurtosis()).Within(1e-12), "Kurtosis Reference (not independent!)");
Assert.That(y.Kurtosis, Is.EqualTo(x.Kurtosis).Within(1e-12), "Kurtosis y");
Assert.That(z.Kurtosis, Is.EqualTo(x.Kurtosis).Within(1e-12), "Kurtosis z");
Assert.That(x.PopulationKurtosis, Is.EqualTo(d.PopulationKurtosis()).Within(1e-12), "PopulationKurtosis Reference (not independent!)");
Assert.That(y.PopulationKurtosis, Is.EqualTo(x.PopulationKurtosis).Within(1e-12), "PopulationKurtosis y");
Assert.That(z.PopulationKurtosis, Is.EqualTo(x.PopulationKurtosis).Within(1e-12), "PopulationKurtosis z");
}
public void ZeroVarianceSequence()
{
var stats = new RunningStatistics(new[] { 2.0, 2.0, 2.0, 2.0 });
Assert.That(stats.Skewness, Is.NaN);
Assert.That(stats.Kurtosis, Is.NaN);
}
public void ShortSequences()
{
var stats0 = new RunningStatistics(new double[0]);
Assert.That(stats0.Skewness, Is.NaN);
Assert.That(stats0.Kurtosis, Is.NaN);
var stats1 = new RunningStatistics(new[] { 1.0 });
Assert.That(stats1.Skewness, Is.NaN);
Assert.That(stats1.Kurtosis, Is.NaN);
var stats2 = new RunningStatistics(new[] { 1.0, 2.0 });
Assert.That(stats2.Skewness, Is.NaN);
Assert.That(stats2.Kurtosis, Is.NaN);
var stats3 = new RunningStatistics(new[] { 1.0, 2.0, -3.0 });
Assert.That(stats3.Skewness, Is.Not.NaN);
Assert.That(stats3.Kurtosis, Is.NaN);
var stats4 = new RunningStatistics(new[] { 1.0, 2.0, -3.0, -4.0 });
Assert.That(stats4.Skewness, Is.Not.NaN);
Assert.That(stats4.Kurtosis, Is.Not.NaN);
}
public void KurtosisConsistentWithR_e1071(string dataSet, double kurtosisType1, double kurtosisType2)
{
var data = _data[dataSet];
var stats = new RunningStatistics(data.Data);
Assert.That(stats.Kurtosis, Is.EqualTo(kurtosisType2).Within(1e-6), "Kurtosis");
Assert.That(stats.PopulationKurtosis, Is.EqualTo(kurtosisType1).Within(1e-6), "PopulationKurtosis");
}
public void SkewnessConsistentWithR_e1071(string dataSet, double delta, double skewnessType1, double skewnessType2)
{
var data = _data[dataSet];
var stats = new RunningStatistics(data.Data);
Assert.That(stats.Skewness, Is.EqualTo(skewnessType2).Within(delta), "Skewness");
Assert.That(stats.PopulationSkewness, Is.EqualTo(skewnessType1).Within(delta), "PopulationSkewness");
}
/// <summary>
/// Create a new running statistics over the combined samples of two existing running statistics.
/// </summary>
public static RunningStatistics Combine(RunningStatistics a, RunningStatistics b)
{
long n = a._n + b._n;
double d = b._m1 - a._m1;
double d2 = d*d;
double d3 = d2*d;
double d4 = d2*d2;
double m1 = (a._n*a._m1 + b._n*b._m1)/n;
double m2 = a._m2 + b._m2 + d2*a._n*b._n/n;
double m3 = a._m3 + b._m3 + d3*a._n*b._n*(a._n - b._n)/(n*n)
+ 3*d*(a._n*b._m2 - b._n*a._m2)/n;
double m4 = a._m4 + b._m4 + d4*a._n*b._n*(a._n*a._n - a._n*b._n + b._n*b._n)/(n*n*n)
+ 6*d2*(a._n*a._n*b._m2 + b._n*b._n*a._m2)/(n*n) + 4*d*(a._n*b._m3 - b._n*a._m3)/n;
return new RunningStatistics { _n = n, _m1 = m1, _m2 = m2, _m3 = m3, _m4 = m4 };
}
public void UniformDistributionTest(double[] sampleArr, double lowerBound, double upperBound)
{
Array.Sort(sampleArr);
RunningStatistics runningStats = new RunningStatistics(sampleArr);
// Skewness should be pretty close to zero (evenly distributed samples)
if (Math.Abs(runningStats.Skewness) > 0.01) Assert.Fail();
// Mean test.
double range = upperBound - lowerBound;
double expectedMean = lowerBound + (range / 2.0);
double meanErr = expectedMean - runningStats.Mean;
double maxExpectedErr = range / 1000.0;
if (Math.Abs(meanErr) > maxExpectedErr) Assert.Fail();
for (double tau = 0; tau <= 1.0; tau += 0.01)
{
double quantile = SortedArrayStatistics.Quantile(sampleArr, tau);
double expectedQuantile = lowerBound + (tau * range);
double quantileError = expectedQuantile - quantile;
if (Math.Abs(quantileError) > maxExpectedErr) Assert.Fail();
}
ShapiroWilkTest test = new ShapiroWilkTest(sampleArr);
Assert.True(test.Significant);
}
/// <summary>
/// Estimates the unbiased population skewness and kurtosis from the provided samples in a single pass.
/// Uses a normalizer (Bessel's correction; type 2).
/// </summary>
/// <param name="samples">A subset of samples, sampled from the full population.</param>
public static Tuple<double, double> SkewnessKurtosis(this IEnumerable<double> samples)
{
var stats = new RunningStatistics(samples);
return new Tuple<double, double>(stats.Skewness, stats.Kurtosis);
}
/// <summary>
/// Evaluates the skewness and kurtosis from the full population.
/// Does not use a normalizer and would thus be biased if applied to a subset (type 1).
/// </summary>
/// <param name="population">The full population data.</param>
public static Tuple<double, double> PopulationSkewnessKurtosis(this IEnumerable<double> population)
{
var stats = new RunningStatistics(population);
return new Tuple<double, double>(stats.PopulationSkewness, stats.PopulationKurtosis);
}
