public void MultiRootExtendedRosenbock()
{
Random rng = new Random(1);
// dimension must be even for this one
for (int d = 2; d <= 10; d += 2)
{
Func <IReadOnlyList <double>, IReadOnlyList <double> > f = delegate(IReadOnlyList <double> u) {
double[] v = new double[d];
for (int k = 0; k < d / 2; k++)
{
v[2 * k] = 1.0 - u[2 * k + 1];
v[2 * k + 1] = 10.0 * (u[2 * k] - MoreMath.Pow(u[2 * k + 1], 2));
}
return(v);
};
ColumnVector x = MultiFunctionMath.FindZero(f, GetRandomStartingVector(rng, d));
// the solution is (1, 1, ..., 1)
for (int i = 0; i < d; i++)
{
Assert.IsTrue(TestUtilities.IsNearlyEqual(x[i], 1.0));
}
// and of course the function should evaluate to zero
IReadOnlyList <double> y = f(x);
for (int i = 0; i < d; i++)
{
Assert.IsTrue(Math.Abs(y[i]) < TestUtilities.TargetPrecision);
}
}
}
public void MultiRootMathworksExample()
{
Func <IReadOnlyList <double>, IReadOnlyList <double> > f = delegate(IReadOnlyList <double> u) {
double x = u[0]; double y = u[1];
return(new double[] { 2.0 * x - y - Math.Exp(-x), 2.0 * y - x - Math.Exp(-y) });
};
double[] x0 = GetRandomStartingVector(new Random(1), 2);
IReadOnlyList <double> xz = MultiFunctionMath.FindZero(f, x0);
IReadOnlyList <double> fz = f(xz);
Assert.IsTrue(Math.Abs(fz[0]) < TestUtilities.TargetPrecision);
Assert.IsTrue(Math.Abs(fz[1]) < TestUtilities.TargetPrecision);
}
public void MultiRootTrigExp()
{
Random rng = new Random(1);
foreach (int d in new int[] { 3, 5, 8 })
{
Func <IReadOnlyList <double>, IReadOnlyList <double> > f = delegate(IReadOnlyList <double> x) {
double[] y = new double[d];
y[0] = 3.0 * MoreMath.Pow(x[0], 3) + 2.0 * x[1] - 5.0 + Math.Sin(x[0] - x[1]) * Math.Sin(x[0] + x[1]);
for (int k = 1; k < d - 1; k++)
{
y[k] = -x[k - 1] * Math.Exp(x[k - 1] - x[k]) + x[k] * (4.0 + 3.0 * MoreMath.Pow(x[k], 2)) + 2.0 * x[k + 1] +
Math.Sin(x[k] - x[k + 1]) * Math.Sin(x[k] + x[k + 1]) - 8.0;
}
y[d - 1] = -x[d - 2] * Math.Exp(x[d - 2] - x[d - 1]) + 4.0 * x[d - 1] - 3.0;
return(y);
};
double[] x0 = GetRandomStartingVector(rng, d);
IReadOnlyList <double> xz = MultiFunctionMath.FindZero(f, x0);
// solution is (1, 1, ..., 1)
for (int i = 0; i < d; i++)
{
Assert.IsTrue(TestUtilities.IsNearlyEqual(xz[i], 1.0));
}
// and function there is of course zero
IReadOnlyList <double> fz = f(xz);
for (int i = 0; i < d; i++)
{
Assert.IsTrue(Math.Abs(fz[i]) < TestUtilities.TargetPrecision);
}
}
}
// We need a goodness-of-fit measurement
internal LinearLogisticRegressionResult(IReadOnlyList <double> x, IReadOnlyList <bool> y)
{
Debug.Assert(x != null);
Debug.Assert(y != null);
Debug.Assert(x.Count == y.Count);
// check size of data set
int n = x.Count;
if (n < 3)
{
throw new InsufficientDataException();
}
// The linear logistic model is:
// p_i = \sigma(t_i) \quad t_i = a + b x_i
// So the log likelihood of the data set under the model is:
// \ln L = \sum_{{\rm true} i} \ln p_i + \sum_{{\rm false} i} \ln (1 - p_i)
// = \sum_{{\rm true} i} \ln \sigma(t_i) + \sum_{{\rm false} i} \ln (1 - \sigma(t_i))
// Taking derivatives:
// \frac{\partial L}{\partial a} = \sum_{{\rm true} i} \frac{\sigma'(t_i)}{\sigma(t_i)}
// + \sum_{{\rm false} i} \frac{-\sigma'(t_i)}{1 - \sigma(t_i)}
// \frac{\partial L}{\partial b} = \sum_{{\rm true} i} \frac{\sigma'(t_i)}{\sigma(t_i)} x_i
// + \sum_{{\rm false} i} \frac{-\sigma'(t_i)}{1 - \sigma(t_i)} x_i
// Using \sigma(t) = \frac{1}{1 + e^{-t}}, we can derive:
// \frac{\sigma'(t)}{\sigma(t)} = \sigma(-t)
// \frac{\sigma'(t)}{1 - \sigma(t)} = \sigma(t)
// So this becomes
// \frac{\partial L}{\partial a} = \sum_i \pm \sigma(\mp t_i)
// \frac{\partial L}{\partial b} = \sum_i \pm \sigma(\mp t_i) x_i
// where the upper sign is for true values and the lower sign is for false values.
// Find the simultaneous zeros of these equations to obtain the likelihood-maximizing a, b.
// To get the curvature matrix, we need the second derivatives.
// \frac{\partial^2 L}{\partial a^2} = - \sum_i \sigma'(\mp t_i)
// \frac{\partial^2 L}{\partial a \partial b} = - \sum_i \sigma'(\mp t_i) x_i
// \frac{\partial^2 L}{\partial b^2} = - \sum_i \sigma'(\mp t_i) x_i^2
// We need an initial guess at the parameters. Begin with the Ansatz of the logistic model:
// \frac{p}{1-p} = e^{\alpha + \beta x}
// Differentiate and do some algebra to get:
// \frac{\partial p}{\partial x} = \beta p ( 1 - p)
// Evaluating at means, and noting that p (1 - p) = var(y) and that, in a development around the means,
// cov(p, x) = \frac{\partial p}{\partial x} var(x)
// we get
// \beta = \frac{cov(y, x)}{var(x) var(y)}
// This approximation gets the sign right, but it looks like it usually gets the magnitude quite wrong.
// The problem with the approach is that var(y) = p (1 - p) assumes y are chosen with fixed p, but they aren't.
// We need to re-visit this analysis.
double xMean, yMean, xxSum, yySum, xySum;
Bivariate.ComputeBivariateMomentsUpToTwo(x, y.Select(z => z ? 1.0 : 0.0), out n, out xMean, out yMean, out xxSum, out yySum, out xySum);
double p = yMean;
double b0 = xySum / xxSum / yySum * n;
double a0 = Math.Log(p / (1.0 - p)) - b0 * xMean;
Func <IReadOnlyList <double>, IReadOnlyList <double> > J = (IReadOnlyList <double> a) => {
double dLda = 0.0;
double dLdb = 0.0;
for (int i = 0; i < n; i++)
{
double t = a[0] + a[1] * x[i];
if (y[i])
{
double s = Sigma(-t);
dLda += s;
dLdb += s * x[i];
}
else
{
double s = Sigma(t);
dLda -= s;
dLdb -= s * x[i];
}
}
return(new double[] { dLda, dLdb });
};
ColumnVector b = MultiFunctionMath.FindZero(J, new double[] { a0, b0 });
SymmetricMatrix C = new SymmetricMatrix(2);
for (int i = 0; i < n; i++)
{
double t = b[0] + b[1] * x[i];
if (y[i])
{
t = -t;
}
double e = Math.Exp(-t);
double sp = e / MoreMath.Sqr(1.0 + e);
C[0, 0] += sp;
C[0, 1] += sp * x[i];
C[1, 1] += sp * x[i] * x[i];
}
CholeskyDecomposition CD = C.CholeskyDecomposition();
if (CD == null)
{
throw new DivideByZeroException();
}
C = CD.Inverse();
best = b;
covariance = C;
}
/// <summary>
/// Finds the Beta distribution that best fits the given sample.
/// </summary>
/// <param name="sample">The sample to fit.</param>
/// <returns>The best fit parameters.</returns>
/// <exception cref="ArgumentNullException"><paramref name="sample"/> is <see langword="null"/>.</exception>
/// <exception cref="InsufficientDataException"><paramref name="sample"/> contains fewer than three values.</exception>
/// <exception cref="InvalidOperationException">Not all the entries in <paramref name="sample" /> lie between zero and one.</exception>
public static BetaFitResult FitToBeta(this IReadOnlyList <double> sample)
{
if (sample == null)
{
throw new ArgumentNullException(nameof(sample));
}
if (sample.Count < 3)
{
throw new InsufficientDataException();
}
// maximum likelihood calculation
// \log L = \sum_i \left[ (\alpha-1) \log x_i + (\beta-1) \log (1-x_i) - \log B(\alpha,\beta) \right]
// using \frac{\partial B(a,b)}{\partial a} = \psi(a) - \psi(a+b), we have
// \frac{\partial \log L}{\partial \alpha} = \sum_i \log x_i - N \left[ \psi(\alpha) - \psi(\alpha+\beta) \right]
// \frac{\partial \log L}{\partial \beta} = \sum_i \log (1-x_i) - N \left[ \psi(\beta) - \psi(\alpha+\beta) \right]
// set equal to zero to get equations for \alpha, \beta
// \psi(\alpha) - \psi(\alpha+\beta) = <\log x>
// \psi(\beta) - \psi(\alpha+\beta) = <\log (1-x)>
// compute the mean log of x and (1-x)
// these are the (logs of) the geometric means
double ga = 0.0; double gb = 0.0;
foreach (double value in sample)
{
if ((value <= 0.0) || (value >= 1.0))
{
throw new InvalidOperationException();
}
ga += Math.Log(value); gb += Math.Log(1.0 - value);
}
ga /= sample.Count; gb /= sample.Count;
// define the function to zero
Func <IReadOnlyList <double>, IReadOnlyList <double> > f = delegate(IReadOnlyList <double> x) {
double pab = AdvancedMath.Psi(x[0] + x[1]);
return(new double[] {
AdvancedMath.Psi(x[0]) - pab - ga,
AdvancedMath.Psi(x[1]) - pab - gb
});
};
// guess initial values using the method of moments
// M1 = \frac{\alpha}{\alpha+\beta} C2 = \frac{\alpha\beta}{(\alpha+\beta)^2 (\alpha+\beta+1)}
// implies
// \alpha = M1 \left( \frac{M1 (1-M1)}{C2} - 1 \right)
// \beta = (1 - M1) \left( \frac{M1 (1-M1)}{C2} -1 \right)
int n;
double m, v;
ComputeMomentsUpToSecond(sample, out n, out m, out v);
v = v / n;
double mm = 1.0 - m;
double q = m * mm / v - 1.0;
double[] x0 = new double[] { m *q, mm *q };
// find the parameter values that zero the two equations
ColumnVector ab = MultiFunctionMath.FindZero(f, x0);
double a = ab[0]; double b = ab[1];
// take more derivatives of \log L to get curvature matrix
// \frac{\partial^2 \log L}{\partial\alpha^2} = - N \left[ \psi'(\alpha) - \psi'(\alpha+\beta) \right]
// \frac{\partial^2 \log L}{\partial\beta^2} = - N \left[ \psi'(\beta) - \psi'(\alpha+\beta) \right]
// \frac{\partial^2 \log L}{\partial \alpha \partial \beta} = - N \psi'(\alpha+\beta)
// covariance matrix is inverse of curvature matrix
SymmetricMatrix C = new SymmetricMatrix(2);
C[0, 0] = sample.Count * (AdvancedMath.Psi(1, a) - AdvancedMath.Psi(1, a + b));
C[1, 1] = sample.Count * (AdvancedMath.Psi(1, b) - AdvancedMath.Psi(1, a + b));
C[0, 1] = sample.Count * AdvancedMath.Psi(1, a + b);
CholeskyDecomposition CD = C.CholeskyDecomposition();
if (CD == null)
{
throw new DivideByZeroException();
}
C = CD.Inverse();
// do a KS test on the result
BetaDistribution distribution = new BetaDistribution(a, b);
TestResult test = sample.KolmogorovSmirnovTest(distribution);
return(new BetaFitResult(ab, C, distribution, test));
}
