public void SampleMaximumLikelihoodFit()
{
// normal distriubtion
Console.WriteLine("normal");
double mu = -1.0;
double sigma = 2.0;
Distribution nd = new NormalDistribution(mu, sigma);
Sample ns = CreateSample(nd, 500);
//FitResult nr = ns.MaximumLikelihoodFit(new NormalDistribution(mu + 1.0, sigma + 1.0));
FitResult nr = ns.MaximumLikelihoodFit((IList<double> p) => new NormalDistribution(p[0], p[1]), new double[] { mu + 1.0, sigma + 1.0 });
Console.WriteLine(nr.Parameter(0));
Console.WriteLine(nr.Parameter(1));
Assert.IsTrue(nr.Dimension == 2);
Assert.IsTrue(nr.Parameter(0).ConfidenceInterval(0.95).ClosedContains(mu));
Assert.IsTrue(nr.Parameter(1).ConfidenceInterval(0.95).ClosedContains(sigma));
FitResult nr2 = NormalDistribution.FitToSample(ns);
Console.WriteLine(nr.Covariance(0,1));
// test analytic expression
Assert.IsTrue(TestUtilities.IsNearlyEqual(nr.Parameter(0).Value, ns.Mean, Math.Sqrt(TestUtilities.TargetPrecision)));
// we don't expect to be able to test sigma against analytic expression because ML value has known bias for finite sample size
// exponential distribution
Console.WriteLine("exponential");
double em = 3.0;
Distribution ed = new ExponentialDistribution(em);
Sample es = CreateSample(ed, 100);
//FitResult er = es.MaximumLikelihoodFit(new ExponentialDistribution(em + 1.0));
FitResult er = es.MaximumLikelihoodFit((IList<double> p) => new ExponentialDistribution(p[0]), new double[] { em + 1.0 });
Console.WriteLine(er.Parameter(0));
Assert.IsTrue(er.Dimension == 1);
Assert.IsTrue(er.Parameter(0).ConfidenceInterval(0.95).ClosedContains(em));
// test against analytic expression
Assert.IsTrue(TestUtilities.IsNearlyEqual(er.Parameter(0).Value, es.Mean, Math.Sqrt(TestUtilities.TargetPrecision)));
Assert.IsTrue(TestUtilities.IsNearlyEqual(er.Parameter(0).Uncertainty, es.Mean / Math.Sqrt(es.Count), Math.Sqrt(Math.Sqrt(TestUtilities.TargetPrecision))));
// lognormal distribution
Console.WriteLine("lognormal");
double l1 = -4.0;
double l2 = 5.0;
Distribution ld = new LognormalDistribution(l1, l2);
Sample ls = CreateSample(ld, 100);
//FitResult lr = ls.MaximumLikelihoodFit(new LognormalDistribution(l1 + 1.0, l2 + 1.0));
FitResult lr = ls.MaximumLikelihoodFit((IList<double> p) => new LognormalDistribution(p[0], p[1]), new double[] { l1 + 1.0, l2 + 1.0 });
Console.WriteLine(lr.Parameter(0));
Console.WriteLine(lr.Parameter(1));
Console.WriteLine(lr.Covariance(0, 1));
Assert.IsTrue(lr.Dimension == 2);
Assert.IsTrue(lr.Parameter(0).ConfidenceInterval(0.99).ClosedContains(l1));
Assert.IsTrue(lr.Parameter(1).ConfidenceInterval(0.99).ClosedContains(l2));
// weibull distribution
Console.WriteLine("weibull");
double w_scale = 4.0;
double w_shape = 2.0;
WeibullDistribution w_d = new WeibullDistribution(w_scale, w_shape);
Sample w_s = CreateSample(w_d, 20);
//FitResult w_r = w_s.MaximumLikelihoodFit(new WeibullDistribution(1.0, 0.5));
FitResult w_r = w_s.MaximumLikelihoodFit((IList<double> p) => new WeibullDistribution(p[0], p[1]), new double[] { 2.0, 2.0 });
Console.WriteLine(w_r.Parameter(0));
Console.WriteLine(w_r.Parameter(1));
Console.WriteLine(w_r.Covariance(0, 1));
Assert.IsTrue(w_r.Parameter(0).ConfidenceInterval(0.95).ClosedContains(w_d.ScaleParameter));
Assert.IsTrue(w_r.Parameter(1).ConfidenceInterval(0.95).ClosedContains(w_d.ShapeParameter));
// logistic distribution
Console.WriteLine("logistic");
double logistic_m = -3.0;
double logistic_s = 2.0;
Distribution logistic_distribution = new LogisticDistribution(logistic_m, logistic_s);
Sample logistic_sample = CreateSample(logistic_distribution, 100);
//FitResult logistic_result = logistic_sample.MaximumLikelihoodFit(new LogisticDistribution());
FitResult logistic_result = logistic_sample.MaximumLikelihoodFit((IList<double> p) => new LogisticDistribution(p[0], p[1]), new double[] { 2.0, 3.0 });
Console.WriteLine(logistic_result.Parameter(0));
Console.WriteLine(logistic_result.Parameter(1));
Assert.IsTrue(logistic_result.Dimension == 2);
Assert.IsTrue(logistic_result.Parameter(0).ConfidenceInterval(0.95).ClosedContains(logistic_m));
Assert.IsTrue(logistic_result.Parameter(1).ConfidenceInterval(0.95).ClosedContains(logistic_s));
// beta distribution
// not yet!
/*
double beta_alpha = 0.5;
double beta_beta = 2.0;
Distribution beta_distribution = new BetaDistribution(beta_alpha, beta_beta);
Sample beta_sample = CreateSample(beta_distribution, 100);
FitResult beta_result = beta_sample.MaximumLikelihoodFit(new BetaDistribution(1.0, 1.0));
Console.WriteLine("Beta:");
Console.WriteLine(beta_result.Parameter(0));
Console.WriteLine(beta_result.Parameter(1));
Assert.IsTrue(beta_result.Dimension == 2);
Assert.IsTrue(beta_result.Parameter(0).ConfidenceInterval(0.95).ClosedContains(beta_alpha));
Assert.IsTrue(beta_result.Parameter(1).ConfidenceInterval(0.95).ClosedContains(beta_beta));
*/
}
public void WeibullFitUncertainties()
{
// check that the uncertainty in reported fit parameters is actually meaningful
// it should be the standard deviation of fit parameter values in a sample of many fits
// define a population distribution
Distribution distribution = new WeibullDistribution(2.5, 1.5);
// draw a lot of samples from it; fit each sample and
// record the reported parameter value and error of each
BivariateSample values = new BivariateSample();
MultivariateSample uncertainties = new MultivariateSample(3);
for (int i = 0; i < 50; i++) {
Sample sample = CreateSample(distribution, 10, i);
FitResult fit = WeibullDistribution.FitToSample(sample);
UncertainValue a = fit.Parameter(0);
UncertainValue b = fit.Parameter(1);
values.Add(a.Value, b.Value);
uncertainties.Add(a.Uncertainty, b.Uncertainty, fit.Covariance(0,1));
}
// the reported errors should agree with the standard deviation of the reported parameters
Assert.IsTrue(values.X.PopulationStandardDeviation.ConfidenceInterval(0.95).ClosedContains(uncertainties.Column(0).Mean));
Assert.IsTrue(values.Y.PopulationStandardDeviation.ConfidenceInterval(0.95).ClosedContains(uncertainties.Column(1).Mean));
//Console.WriteLine("{0} {1}", values.PopulationCovariance, uncertainties.Column(2).Mean);
//Assert.IsTrue(values.PopulationCovariance.ConfidenceInterval(0.95).ClosedContains(uncertainties.Column(2).Mean));
}
/// <summary>
/// Computes the Weibull distribution that best fits the given sample.
/// </summary>
/// <param name="sample">The sample to fit.</param>
/// <returns>The best fit parameters.</returns>
/// <remarks>
/// <para>The returned fit parameters are the <see cref="ShapeParameter"/> and <see cref="ScaleParameter"/>, in that order.
/// These are the same parameters, in the same order, that are required by the <see cref="WeibullDistribution(double,double)"/> constructor to
/// specify a new Weibull distribution.</para>
/// </remarks>
/// <exception cref="ArgumentNullException"><paramref name="sample"/> is null.</exception>
/// <exception cref="InvalidOperationException"><paramref name="sample"/> contains non-positive values.</exception>
/// <exception cref="InsufficientDataException"><paramref name="sample"/> contains fewer than three values.</exception>
public static FitResult FitToSample(Sample sample)
{
if (sample == null)
{
throw new ArgumentNullException("sample");
}
if (sample.Count < 3)
{
throw new InsufficientDataException();
}
if (sample.Minimum <= 0.0)
{
throw new InvalidOperationException();
}
// The log likelyhood function is
// \log L = N \log k + (k-1) \sum_i \log x_i - N K \log \lambda - \sum_i \left(\frac{x_i}{\lambda}\right)^k
// Taking derivatives, we get
// \frac{\partial \log L}{\partial \lambda} = - \frac{N k}{\lambda} + \sum_i \frac{k}{\lambda} \left(\frac{x_i}{\lambda}\right)^k
// \frac{\partial \log L}{\partial k} =\frac{N}{k} + \sum_i \left[ 1 - \left(\frac{x_i}{\lambda}\right)^k \right] \log \left(\frac{x_i}{\lambda}\right)
// Setting the first expression to zero and solving for \lambda gives
// \lambda = \left( N^{-1} \sum_i x_i^k \right)^{1/k} = ( < x^k > )^{1/k}
// which allows us to reduce the problem from 2D to 1D.
// By the way, using the expression for the moment < x^k > of the Weibull distribution, you can show there is
// no bias to this result even for finite samples.
// Setting the second expression to zero gives
// \frac{1}{k} = \frac{1}{N} \sum_i \left[ \left( \frac{x_i}{\lambda} \right)^k - 1 \right] \log \left(\frac{x_i}{\lambda}\right)
// which, given the equation for \lambda as a function of k derived from the first expression, is an implicit equation for k.
// It cannot be solved in closed form, but we have now reduced our problem to finding a root in one-dimension.
// We need a starting guess for k.
// The method of moments equations are not solvable for the parameters in closed form
// but the scale parameter drops out of the ratio of the 1/3 and 2/3 quantile points
// and the result is easily solved for the shape parameter
// k = \frac{\log 2}{\log\left(\frac{x_{2/3}}{x_{1/3}}\right)}
double x1 = sample.InverseLeftProbability(1.0 / 3.0);
double x2 = sample.InverseLeftProbability(2.0 / 3.0);
double k0 = Global.LogTwo / Math.Log(x2 / x1);
// Given the shape paramter, we could invert the expression for the mean to get
// the scale parameter, but since we have an expression for \lambda from k, we
// dont' need it.
//double s0 = sample.Mean / AdvancedMath.Gamma(1.0 + 1.0 / k0);
// Simply handing our 1D function to a root-finder works fine until we start to encounter large k. For large k,
// even just computing \lambda goes wrong because we are taking x_i^k which overflows. Horst Rinne, "The Weibull
// Distribution: A Handbook" describes a way out. Basically, we first move to variables z_i = \log(x_i) and
// then w_i = z_i - \bar{z}. Then lots of factors of e^{k \bar{z}} cancel out and, even though we still do
// have some e^{k w_i}, the w_i are small and centered around 0 instead of large and centered around \lambda.
Sample transformedSample = sample.Copy();
transformedSample.Transform(x => Math.Log(x));
double zbar = transformedSample.Mean;
transformedSample.Transform(z => z - zbar);
// After this change of variable the 1D function to zero becomes
// g(k) = \sum_i ( 1 - k w_i ) e^{k w_i}
// It's easy to show that g(0) = n and g(\infinity) = -\infinity, so it must cross zero. It's also easy to take
// a derivative
// g'(k) = - k \sum_i w_i^2 e^{k w_i}
// so we can apply Newton's method.
int i = 0;
double k1 = k0;
while (true)
{
i++;
double g = 0.0;
double gp = 0.0;
foreach (double w in transformedSample)
{
double e = Math.Exp(k1 * w);
g += (1.0 - k1 * w) * e;
gp -= k1 * w * w * e;
}
double dk = -g / gp;
k1 += dk;
if (Math.Abs(dk) <= Global.Accuracy * Math.Abs(k1))
{
break;
}
if (i >= Global.SeriesMax)
{
throw new NonconvergenceException();
}
}
// The corresponding lambda can also be expressed in terms of zbar and w's.
double t = 0.0;
foreach (double w in transformedSample)
{
t += Math.Exp(k1 * w);
}
t /= transformedSample.Count;
double lambda1 = Math.Exp(zbar) * Math.Pow(t, 1.0 / k1);
// We need the curvature matrix at the minimum of our log likelyhood function
// to determine the covariance matrix. Taking more derivatives...
// \frac{\partial^2 \log L} = \frac{N k}{\lambda^2} - \sum_i \frac{k(k+1) x_i^k}{\lambda^{k+2}}
// = - \frac{N k^2}{\lambda^2}
// The second expression follows by inserting the first-derivative-equal-zero relation into the first.
// For k=1, this agrees with the variance formula for the mean of the best-fit exponential.
// Derivatives involving k are less simple.
// We end up needing the means < (x/lambda)^k log(x/lambda) > and < (x/lambda)^k log^2(x/lambda) >
double mpl = 0.0; double mpl2 = 0.0;
foreach (double x in sample)
{
double r = x / lambda1;
double p = Math.Pow(r, k1);
double l = Math.Log(r);
double pl = p * l;
double pl2 = pl * l;
mpl += pl;
mpl2 += pl2;
}
mpl = mpl / sample.Count;
mpl2 = mpl2 / sample.Count;
// See if we can't do any better here. Transforming to zbar and w's looked ugly, but perhaps it
// can be simplified? One interesting observation: if we take expectation values (which gives
// the Fisher information matrix) the entries become simple:
// B_{\lambda \lambda} = \frac{N k^2}{\lambda^2}
// B_{\lambda k} = -\Gamma'(2) \frac{N}{\lambda}
// B_{k k } = [1 + \Gamma''(2)] \frac{N}{k^2}
// Would it be bad to just use these directly?
// Construct the curvature matrix and invert it.
SymmetricMatrix B = new SymmetricMatrix(2);
B[0, 0] = sample.Count * MoreMath.Sqr(k1 / lambda1);
B[0, 1] = -sample.Count * k1 / lambda1 * mpl;
B[1, 1] = sample.Count * (1.0 / MoreMath.Pow2(k1) + mpl2);
SymmetricMatrix C = B.CholeskyDecomposition().Inverse();
// Do a KS test to compare sample to best-fit distribution
Distribution distribution = new WeibullDistribution(lambda1, k1);
TestResult test = sample.KolmogorovSmirnovTest(distribution);
// return the result
return(new FitResult(new double[] { lambda1, k1 }, C, test));
}
public void WeibullFit()
{
foreach (double alpha in new double[] { 0.031, 0.13, 1.3, 3.1, 13.1, 33.1, 313.1, 1313.1 }) {
WeibullDistribution W = new WeibullDistribution(2.2, alpha);
Sample S = CreateSample(W, 50);
FitResult R = WeibullDistribution.FitToSample(S);
Console.WriteLine("{0} ?= {1}, {2} ?= {3}", R.Parameter(0), W.ScaleParameter, R.Parameter(1), W.ShapeParameter);
Assert.IsTrue(R.Parameter(0).ConfidenceInterval(0.99).ClosedContains(W.ScaleParameter));
Assert.IsTrue(R.Parameter(1).ConfidenceInterval(0.99).ClosedContains(W.ShapeParameter));
}
}
/// <summary>
/// Computes the Weibull distribution that best fits the given sample.
/// </summary>
/// <param name="sample">The sample to fit.</param>
/// <returns>The best fit parameters.</returns>
/// <remarks>
/// <para>The returned fit parameters are the <see cref="ShapeParameter"/> and <see cref="ScaleParameter"/>, in that order.
/// These are the same parameters, in the same order, that are required by the <see cref="WeibullDistribution(double,double)"/> constructor to
/// specify a new Weibull distribution.</para>
/// </remarks>
/// <exception cref="ArgumentNullException"><paramref name="sample"/> is null.</exception>
/// <exception cref="InvalidOperationException"><paramref name="sample"/> contains non-positive values.</exception>
/// <exception cref="InsufficientDataException"><paramref name="sample"/> contains fewer than three values.</exception>
public static FitResult FitToSample(Sample sample)
{
if (sample == null) throw new ArgumentNullException("sample");
if (sample.Count < 3) throw new InsufficientDataException();
if (sample.Minimum <= 0.0) throw new InvalidOperationException();
// The log likelyhood function is
// \log L = N \log k + (k-1) \sum_i \log x_i - N K \log \lambda - \sum_i \left(\frac{x_i}{\lambda}\right)^k
// Taking derivatives, we get
// \frac{\partial \log L}{\partial \lambda} = - \frac{N k}{\lambda} + \sum_i \frac{k}{\lambda} \left(\frac{x_i}{\lambda}\right)^k
// \frac{\partial \log L}{\partial k} =\frac{N}{k} + \sum_i \left[ 1 - \left(\frac{x_i}{\lambda}\right)^k \right] \log \left(\frac{x_i}{\lambda}\right)
// Setting the first expression to zero and solving for \lambda gives
// \lambda = \left( N^{-1} \sum_i x_i^k \right)^{1/k} = ( < x^k > )^{1/k}
// which allows us to reduce the problem from 2D to 1D.
// By the way, using the expression for the moment < x^k > of the Weibull distribution, you can show there is
// no bias to this result even for finite samples.
// Setting the second expression to zero gives
// \frac{1}{k} = \frac{1}{N} \sum_i \left[ \left( \frac{x_i}{\lambda} \right)^k - 1 \right] \log \left(\frac{x_i}{\lambda}\right)
// which, given the equation for \lambda as a function of k derived from the first expression, is an implicit equation for k.
// It cannot be solved in closed form, but we have now reduced our problem to finding a root in one-dimension.
// We need a starting guess for k.
// The method of moments equations are not solvable for the parameters in closed form
// but the scale parameter drops out of the ratio of the 1/3 and 2/3 quantile points
// and the result is easily solved for the shape parameter
// k = \frac{\log 2}{\log\left(\frac{x_{2/3}}{x_{1/3}}\right)}
double x1 = sample.InverseLeftProbability(1.0 / 3.0);
double x2 = sample.InverseLeftProbability(2.0 / 3.0);
double k0 = Global.LogTwo / Math.Log(x2 / x1);
// Given the shape paramter, we could invert the expression for the mean to get
// the scale parameter, but since we have an expression for \lambda from k, we
// dont' need it.
//double s0 = sample.Mean / AdvancedMath.Gamma(1.0 + 1.0 / k0);
// Simply handing our 1D function to a root-finder works fine until we start to encounter large k. For large k,
// even just computing \lambda goes wrong because we are taking x_i^k which overflows. Horst Rinne, "The Weibull
// Distribution: A Handbook" describes a way out. Basically, we first move to variables z_i = \log(x_i) and
// then w_i = z_i - \bar{z}. Then lots of factors of e^{k \bar{z}} cancel out and, even though we still do
// have some e^{k w_i}, the w_i are small and centered around 0 instead of large and centered around \lambda.
Sample transformedSample = sample.Copy();
transformedSample.Transform(x => Math.Log(x));
double zbar = transformedSample.Mean;
transformedSample.Transform(z => z - zbar);
// After this change of variable the 1D function to zero becomes
// g(k) = \sum_i ( 1 - k w_i ) e^{k w_i}
// It's easy to show that g(0) = n and g(\infinity) = -\infinity, so it must cross zero. It's also easy to take
// a derivative
// g'(k) = - k \sum_i w_i^2 e^{k w_i}
// so we can apply Newton's method.
int i = 0;
double k1 = k0;
while (true) {
i++;
double g = 0.0;
double gp = 0.0;
foreach (double w in transformedSample) {
double e = Math.Exp(k1 * w);
g += (1.0 - k1 * w) * e;
gp -= k1 * w * w * e;
}
double dk = -g / gp;
k1 += dk;
if (Math.Abs(dk) <= Global.Accuracy * Math.Abs(k1)) break;
if (i >= Global.SeriesMax) throw new NonconvergenceException();
}
// The corresponding lambda can also be expressed in terms of zbar and w's.
double t = 0.0;
foreach (double w in transformedSample) {
t += Math.Exp(k1 * w);
}
t /= transformedSample.Count;
double lambda1 = Math.Exp(zbar) * Math.Pow(t, 1.0 / k1);
// We need the curvature matrix at the minimum of our log likelyhood function
// to determine the covariance matrix. Taking more derivatives...
// \frac{\partial^2 \log L} = \frac{N k}{\lambda^2} - \sum_i \frac{k(k+1) x_i^k}{\lambda^{k+2}}
// = - \frac{N k^2}{\lambda^2}
// The second expression follows by inserting the first-derivative-equal-zero relation into the first.
// For k=1, this agrees with the variance formula for the mean of the best-fit exponential.
// Derivatives involving k are less simple.
// We end up needing the means < (x/lambda)^k log(x/lambda) > and < (x/lambda)^k log^2(x/lambda) >
double mpl = 0.0; double mpl2 = 0.0;
foreach (double x in sample) {
double r = x / lambda1;
double p = Math.Pow(r, k1);
double l = Math.Log(r);
double pl = p * l;
double pl2 = pl * l;
mpl += pl;
mpl2 += pl2;
}
mpl = mpl / sample.Count;
mpl2 = mpl2 / sample.Count;
// See if we can't do any better here. Transforming to zbar and w's looked ugly, but perhaps it
// can be simplified? One interesting observation: if we take expectation values (which gives
// the Fisher information matrix) the entries become simple:
// B_{\lambda \lambda} = \frac{N k^2}{\lambda^2}
// B_{\lambda k} = -\Gamma'(2) \frac{N}{\lambda}
// B_{k k } = [1 + \Gamma''(2)] \frac{N}{k^2}
// Would it be bad to just use these directly?
// Construct the curvature matrix and invert it.
SymmetricMatrix B = new SymmetricMatrix(2);
B[0, 0] = sample.Count * MoreMath.Sqr(k1 / lambda1);
B[0, 1] = -sample.Count * k1 / lambda1 * mpl;
B[1, 1] = sample.Count * (1.0 / MoreMath.Pow2(k1) + mpl2);
SymmetricMatrix C = B.CholeskyDecomposition().Inverse();
// Do a KS test to compare sample to best-fit distribution
Distribution distribution = new WeibullDistribution(lambda1, k1);
TestResult test = sample.KolmogorovSmirnovTest(distribution);
// return the result
return (new FitResult(new double[] {lambda1, k1}, C, test));
}