/// <summary>
/// Evidence message for EP.
/// </summary>
/// <param name="cases">Incoming message from 'cases'.</param>
/// <param name="b">Incoming message from 'b'.</param>
public static double LogEvidenceRatio(IList <Bernoulli> cases, Bernoulli b)
{
// result = log (p(data|b=true) p(b=true) + p(data|b=false) p(b=false))
// log (p(data|b=true) p(b=true) + p(data|b=false) (1-p(b=true))
// log ((p(data|b=true) - p(data|b=false)) p(b=true) + p(data|b=false))
// log ((p(data|b=true)/p(data|b=false) - 1) p(b=true) + 1) + log p(data|b=false)
// where cases[0].LogOdds = log p(data|b=true)
// cases[1].LogOdds = log p(data|b=false)
if (b.IsPointMass)
{
return(b.Point ? cases[0].LogOdds : cases[1].LogOdds);
}
//else return MMath.LogSumExp(cases[0].LogOdds + b.GetLogProbTrue(), cases[1].LogOdds + b.GetLogProbFalse());
else
{
// the common case is when cases[0].LogOdds == cases[1].LogOdds. we must not introduce rounding error in that case.
if (cases[0].LogOdds >= cases[1].LogOdds)
{
if (Double.IsNegativeInfinity(cases[1].LogOdds))
{
return(cases[0].LogOdds + b.GetLogProbTrue());
}
else
{
return(cases[1].LogOdds + MMath.Log1Plus(b.GetProbTrue() * MMath.ExpMinus1(cases[0].LogOdds - cases[1].LogOdds)));
}
}
else
{
if (Double.IsNegativeInfinity(cases[0].LogOdds))
{
return(cases[1].LogOdds + b.GetLogProbFalse());
}
else
{
return(cases[0].LogOdds + MMath.Log1Plus(b.GetProbFalse() * MMath.ExpMinus1(cases[1].LogOdds - cases[0].LogOdds)));
}
}
}
}
/// <summary>
/// Find a Beta distribution with given integral and mean times a Beta weight function.
/// </summary>
/// <param name="mean">The desired value of the mean</param>
/// <param name="logZ">The desired value of the integral</param>
/// <param name="a">trueCount-1 of the weight function</param>
/// <param name="b">falseCount-1 of the weight function</param>
/// <returns></returns>
private static Beta BetaFromMeanAndIntegral(double mean, double logZ, double a, double b)
{
// The constraints are:
// 1. int_p to_p(p) p^a (1-p)^b dp = exp(logZ)
// 2. int_p to_p(p) p p^a (1-p)^b dp = mean*exp(logZ)
// Let to_p(p) = Beta(p; af, bf)
// The LHS of (1) is gamma(af+bf)/gamma(af+bf+a+b) gamma(af+a)/gamma(af) gamma(bf+b)/gamma(bf)
// The LHS of (2) is gamma(af+bf)/gamma(af+bf+a+b+1) gamma(af+a+1)/gamma(af) gamma(bf+b)/gamma(bf)
// The ratio of (2)/(1) is gamma(af+a+1)/gamma(af+a) gamma(af+bf+a+b)/gamma(af+bf+a+b+1) = (af+a)/(af+bf+a+b) = mean
// Solving for bf gives bf = (af+a)/mean - (af+a+b).
// To solve for af, we apply a generalized Newton algorithm to solve equation (1) with bf substituted.
// af0 is the smallest value of af that ensures (af >= 0, bf >= 0).
if (mean <= 0)
{
throw new ArgumentException("mean <= 0");
}
if (mean >= 1)
{
throw new ArgumentException("mean >= 1");
}
if (double.IsNaN(mean))
{
throw new ArgumentException("mean is NaN");
}
// bf = (af+bx)*(1-m)/m
double bx = -(mean * (a + b) - a) / (1 - mean);
// af0 is the lower bound for af
// we need both af>0 and bf>0
double af0 = Math.Max(0, -bx);
double x = Math.Max(0, bx);
double af = af0 + 1; // initial guess for af
double invMean = 1 / mean;
double bf = (af + a) * invMean - (af + a + b);
for (int iter = 0; iter < 20; iter++)
{
double old_af = af;
double f = (MMath.GammaLn(af + bf) - MMath.GammaLn(af + bf + a + b)) + (MMath.GammaLn(af + a) - MMath.GammaLn(af)) + (MMath.GammaLn(bf + b) - MMath.GammaLn(bf));
double g = (MMath.Digamma(af + bf) - MMath.Digamma(af + bf + a + b)) * invMean + (MMath.Digamma(af + a) - MMath.Digamma(af)) + (MMath.Digamma(bf + b) - MMath.Digamma(bf)) * (invMean - 1);
// fit a fcn of the form: s*log((af-af0)/(af+x)) + c
// whose deriv is s/(af-af0) - s/(af+x)
double s = g / (1 / (af - af0) - 1 / (af + x));
double c = f - s * Math.Log((af - af0) / (af + x));
bool isIncreasing = (x > -af0);
if ((!isIncreasing && c >= logZ) || (isIncreasing && c <= logZ))
{
// the approximation doesn't fit; use Gauss-Newton instead
af += (logZ - f) / g;
}
else
{
// now solve s*log((af-af0)/(af+x))+c = logz
// af-af0 = exp((logz-c)/s) (af+x)
af = af0 + (x + af0) / MMath.ExpMinus1((c - logZ) / s);
if (af == af0)
{
throw new ArgumentException("logZ is out of range");
}
}
if (double.IsNaN(af))
{
throw new ApplicationException("af is nan");
}
bf = (af + a) / mean - (af + a + b);
if (Math.Abs(af - old_af) < 1e-8)
{
break;
}
}
if (false)
{
// check that integrals are correct
double f = (MMath.GammaLn(af + bf) - MMath.GammaLn(af + bf + a + b)) + (MMath.GammaLn(af + a) - MMath.GammaLn(af)) + (MMath.GammaLn(bf + b) - MMath.GammaLn(bf));
if (Math.Abs(f - logZ) > 1e-6)
{
throw new ApplicationException("wrong f");
}
double f2 = (MMath.GammaLn(af + bf) - MMath.GammaLn(af + bf + a + b + 1)) + (MMath.GammaLn(af + a + 1) - MMath.GammaLn(af)) + (MMath.GammaLn(bf + b) - MMath.GammaLn(bf));
if (Math.Abs(f2 - (Math.Log(mean) + logZ)) > 1e-6)
{
throw new ApplicationException("wrong f2");
}
}
return(new Beta(af, bf));
}
/// <summary>
/// Sample from a Gaussian(0,1) truncated at the given upper and lower bounds
/// </summary>
/// <param name="lowerBound">Can be -Infinity.</param>
/// <param name="upperBound">Must be >= <paramref name="lowerBound"/>. Can be Infinity.</param>
/// <returns>A real number >= <paramref name="lowerBound"/> and < <paramref name="upperBound"/></returns>
public static double NormalBetween(double lowerBound, double upperBound)
{
if (double.IsNaN(lowerBound))
{
throw new ArgumentException("lowerBound is NaN");
}
if (double.IsNaN(upperBound))
{
throw new ArgumentException("upperBound is NaN");
}
double delta = upperBound - lowerBound;
if (delta == 0)
{
return(lowerBound);
}
if (delta < 0)
{
throw new ArgumentException("upperBound (" + upperBound + ") < lowerBound (" + lowerBound + ")");
}
// Switch between the following 3 options:
// 1. Gaussian rejection, with acceptance rate Z = NormalCdf(upperBound) - NormalCdf(lowerBound)
// 2. Uniform rejection, with acceptance rate sqrt(2*pi)*Z/delta if the interval contains 0
// 3. Truncated exponential rejection, with acceptance rate
// = sqrt(2*pi)*Z*lambda*exp(-lambda^2/2)/(exp(-lambda*lowerBound)-exp(-lambda*upperBound))
// = sqrt(2*pi)*Z*lowerBound*exp(lowerBound^2/2)/(1-exp(-lowerBound*(upperBound-lowerBound)))
// (3) has the highest acceptance rate under the following conditions:
// lowerBound > 0.5 or (lowerBound > 0 and delta < 2.5)
// (2) has the highest acceptance rate if the interval contains 0 and delta < sqrt(2*pi)
// (1) has the highest acceptance rate otherwise
if (lowerBound > 0.5 || (lowerBound > 0 && delta < 2.5))
{
// Rejection sampler using truncated exponential proposal
double lambda = lowerBound;
double s = MMath.ExpMinus1(-lambda * delta);
double c = 2 * lambda * lambda;
while (true)
{
double x = -MMath.Log1Plus(s * Rand.Double());
double u = -System.Math.Log(Rand.Double());
if (c * u > x * x)
{
return(x / lambda + lowerBound);
}
}
throw new InferRuntimeException("failed to sample");
}
else if (upperBound < -0.5 || (upperBound < 0 && delta < 2.5))
{
return(-NormalBetween(-upperBound, -lowerBound));
}
else if (lowerBound <= 0 && upperBound >= 0 && delta < MMath.Sqrt2PI)
{
// Uniform rejection
while (true)
{
double x = Rand.Double() * delta + lowerBound;
double u = -System.Math.Log(Rand.Double());
if (2 * u > x * x)
{
return(x);
}
}
}
else
{
// Gaussian rejection
while (true)
{
double x = Rand.Normal();
if (x >= lowerBound && x < upperBound)
{
return(x);
}
}
}
}
#pragma warning disable 162
#endif
/// <summary>
/// Find a Beta distribution with given integral and mean times a Beta weight function.
/// </summary>
/// <param name="mean">The desired value of the mean</param>
/// <param name="logZ">The desired value of the integral</param>
/// <param name="a">trueCount-1 of the weight function</param>
/// <param name="b">falseCount-1 of the weight function</param>
/// <returns></returns>
private static Beta BetaFromMeanAndIntegral(double mean, double logZ, double a, double b)
{
// The constraints are:
// 1. int_p to_p(p) p^a (1-p)^b dp = exp(logZ)
// 2. int_p to_p(p) p p^a (1-p)^b dp = mean*exp(logZ)
// Let to_p(p) = Beta(p; af, bf)
// The LHS of (1) is gamma(af+bf)/gamma(af+bf+a+b) gamma(af+a)/gamma(af) gamma(bf+b)/gamma(bf)
// The LHS of (2) is gamma(af+bf)/gamma(af+bf+a+b+1) gamma(af+a+1)/gamma(af) gamma(bf+b)/gamma(bf)
// The ratio of (2)/(1) is gamma(af+a+1)/gamma(af+a) gamma(af+bf+a+b)/gamma(af+bf+a+b+1) = (af+a)/(af+bf+a+b) = mean
// Solving for bf gives bf = (af+a)/mean - (af+a+b).
// To solve for af, we apply a generalized Newton algorithm to solve equation (1) with bf substituted.
// af0 is the smallest value of af that ensures (af >= 0, bf >= 0).
if (mean <= 0)
{
throw new ArgumentException("mean <= 0");
}
if (mean >= 1)
{
throw new ArgumentException("mean >= 1");
}
if (double.IsNaN(mean))
{
throw new ArgumentException("mean is NaN");
}
// If exp(logZ) exceeds the largest possible value of (1), then we return a point mass.
// gammaln(x) =approx (x-0.5)*log(x) - x + 0.5*log(2pi)
// (af+x)*log(af+x) =approx (af+x)*log(af) + x + 0.5*x*x/af
// For large af, logZ = (af+bf-0.5)*log(af+bf) - (af+bf+a+b-0.5)*log(af+bf+a+b) +
// (af+a-0.5)*log(af+a) - (af-0.5)*log(af) +
// (bf+b-0.5)*log(bf+b) - (bf-0.5)*log(bf)
// =approx (af+bf-0.5)*log(af+bf) - ((af+bf+a+b-0.5)*log(af+bf) + (a+b) + 0.5*(a+b)*(a+b)/(af+bf) -0.5*(a+b)/(af+bf)) +
// ((af+a-0.5)*log(af) + a + 0.5*a*a/af - 0.5*a/af) - (af-0.5)*log(af) +
// ((bf+b-0.5)*log(bf) + b + 0.5*b*b/bf - 0.5*b/bf) - (bf-0.5)*log(bf)
// = -(a+b)*log(af+bf) - 0.5*(a+b)*(a+b-1)/(af+bf) + a*log(af) + 0.5*a*(a-1)/af + b*log(bf) + 0.5*b*(b-1)/bf
// =approx (a+b)*log(m) + b*log((1-m)/m) + 0.5*(a+b)*(a+b-1)*m/af - 0.5*a*(a+1)/af - 0.5*b*(b+1)*m/(1-m)/af
// =approx (a+b)*log(mean) + b*log((1-mean)/mean)
double maxLogZ = (a + b) * Math.Log(mean) + b * Math.Log((1 - mean) / mean);
// slope determines whether maxLogZ is the maximum or minimum possible value of logZ
double slope = (a + b) * (a + b - 1) * mean - a * (a + 1) - b * (b + 1) * mean / (1 - mean);
if ((slope <= 0 && logZ >= maxLogZ) || (slope > 0 && logZ <= maxLogZ))
{
// optimal af is infinite
return(Beta.PointMass(mean));
}
// bf = (af+bx)*(1-m)/m
double bx = -(mean * (a + b) - a) / (1 - mean);
// af0 is the lower bound for af
// we need both af>0 and bf>0
double af0 = Math.Max(0, -bx);
double x = Math.Max(0, bx);
double af = af0 + 1; // initial guess for af
double invMean = 1 / mean;
double bf = (af + a) * invMean - (af + a + b);
int numIters = 20;
for (int iter = 0; iter < numIters; iter++)
{
double old_af = af;
double f = (MMath.GammaLn(af + bf) - MMath.GammaLn(af + bf + a + b)) + (MMath.GammaLn(af + a) - MMath.GammaLn(af)) +
(MMath.GammaLn(bf + b) - MMath.GammaLn(bf));
double g = (MMath.Digamma(af + bf) - MMath.Digamma(af + bf + a + b)) * invMean + (MMath.Digamma(af + a) - MMath.Digamma(af)) +
(MMath.Digamma(bf + b) - MMath.Digamma(bf)) * (invMean - 1);
// fit a fcn of the form: s*log((af-af0)/(af+x)) + c
// whose deriv is s/(af-af0) - s/(af+x)
double s = g / (1 / (af - af0) - 1 / (af + x));
double c = f - s * Math.Log((af - af0) / (af + x));
bool isIncreasing = (x > -af0);
if ((!isIncreasing && c >= logZ) || (isIncreasing && c <= logZ))
{
// the approximation doesn't fit; use Gauss-Newton instead
af += (logZ - f) / g;
}
else
{
// now solve s*log((af-af0)/(af+x))+c = logz
// af-af0 = exp((logz-c)/s) (af+x)
af = af0 + (x + af0) / MMath.ExpMinus1((c - logZ) / s);
//if (af == af0)
// throw new ArgumentException("logZ is out of range");
}
if (double.IsNaN(af))
{
throw new InferRuntimeException("af is nan");
}
bf = (af + a) / mean - (af + a + b);
if (Math.Abs(af - old_af) < 1e-8 || af == af0)
{
break;
}
//if (iter == numIters-1)
// throw new Exception("not converging");
}
if (false)
{
// check that integrals are correct
double f = (MMath.GammaLn(af + bf) - MMath.GammaLn(af + bf + a + b)) + (MMath.GammaLn(af + a) - MMath.GammaLn(af)) +
(MMath.GammaLn(bf + b) - MMath.GammaLn(bf));
if (Math.Abs(f - logZ) > 1e-6)
{
throw new InferRuntimeException("wrong f");
}
double f2 = (MMath.GammaLn(af + bf) - MMath.GammaLn(af + bf + a + b + 1)) + (MMath.GammaLn(af + a + 1) - MMath.GammaLn(af)) +
(MMath.GammaLn(bf + b) - MMath.GammaLn(bf));
if (Math.Abs(f2 - (Math.Log(mean) + logZ)) > 1e-6)
{
throw new InferRuntimeException("wrong f2");
}
}
return(new Beta(af, bf));
}