/// <summary>
/// Adds a Gamma distribution item to the estimator
/// </summary>
/// <param name="distribution">The distribution instance to add</param>
public void Add(Gamma distribution)
{
double x, noiseVariance;
distribution.GetMeanAndVariance(out x, out noiseVariance);
mva.Add(x, noiseVariance, 1.0);
}
/// <summary>
/// Adds a Gaussian distribution with given weight to the estimator
/// </summary>
/// <param name="distribution">The distribution instance to add</param>
/// <param name="weight">The weight of the distribution</param>
public void Add(Gaussian distribution, double weight)
{
double x, noiseVariance;
distribution.GetMeanAndVariance(out x, out noiseVariance);
mva.Add(x, noiseVariance, weight);
}
public void MeanVarianceAccumulator_Add_Infinity()
{
MeanVarianceAccumulator mva = new MeanVarianceAccumulator();
mva.Add(double.PositiveInfinity);
mva.Add(0.0);
Assert.True(double.IsPositiveInfinity(mva.Mean));
Assert.True(double.IsPositiveInfinity(mva.Variance));
}
public void MeanVarianceAccumulator_Add_ZeroWeight()
{
MeanVarianceAccumulator mva = new MeanVarianceAccumulator();
mva.Add(4.5, 0.0);
Assert.Equal(4.5, mva.Mean);
mva.Add(4.5);
mva.Add(double.PositiveInfinity, 0.0);
Assert.Equal(4.5, mva.Mean);
}
private void RandNormalBetween(double lowerBound, double upperBound)
{
double meanExpected, varianceExpected;
new Microsoft.ML.Probabilistic.Distributions.TruncatedGaussian(0, 1, lowerBound, upperBound).GetMeanAndVariance(out meanExpected, out varianceExpected);
MeanVarianceAccumulator mva = new MeanVarianceAccumulator();
for (int i = 0; i < nsamples; i++)
{
double x = Rand.NormalBetween(lowerBound, upperBound);
mva.Add(x);
}
double m = mva.Mean;
double v = mva.Variance;
Console.WriteLine("mean = {0} should be {1}", m, meanExpected);
Console.WriteLine("variance = {0} should be {1}", v, varianceExpected);
// the sample mean has stddev = 1/sqrt(n)
double dError = System.Math.Abs(m - meanExpected);
if (dError > 4 / System.Math.Sqrt(nsamples))
{
Assert.True(false, string.Format("m: error = {0}", dError));
}
// the sample variance is Gamma(n/2,n/2) whose stddev = sqrt(2/n)
dError = System.Math.Abs(v - varianceExpected);
if (dError > 4 * System.Math.Sqrt(2.0 / nsamples))
{
Assert.True(false, string.Format("v: error = {0}", dError));
}
}
public void SampleGeometric()
{
Rand.Restart(96);
const double StoppingProbability = 0.7;
// The length of sequences sampled from this distribution must follow a geometric distribution
StringAutomaton automaton = StringAutomaton.Zero();
automaton.Start = automaton.AddState();
automaton.Start.SetEndWeight(Weight.FromValue(StoppingProbability));
automaton.Start.AddTransition('a', Weight.FromValue(1 - StoppingProbability), automaton.Start);
StringDistribution dist = StringDistribution.FromWeightFunction(automaton);
var acc = new MeanVarianceAccumulator();
const int SampleCount = 30000;
for (int i = 0; i < SampleCount; ++i)
{
string sample = dist.Sample();
acc.Add(sample.Length);
}
const double ExpectedMean = (1.0 - StoppingProbability) / StoppingProbability;
const double ExpectedVariance = (1.0 - StoppingProbability) / (StoppingProbability * StoppingProbability);
Assert.Equal(ExpectedMean, acc.Mean, 1e-2);
Assert.Equal(ExpectedVariance, acc.Variance, 1e-2);
}
/// <summary>
/// Adds a TruncatedGaussian distribution item to the estimator
/// </summary>
/// <param name="distribution">The distribution instance to add</param>
public void Add(TruncatedGaussian distribution)
{
minLowerBound = Math.Min(minLowerBound, distribution.LowerBound);
maxUpperBound = Math.Max(maxUpperBound, distribution.UpperBound);
double x, noiseVariance;
distribution.GetMeanAndVariance(out x, out noiseVariance);
mva.Add(x, noiseVariance, 1.0);
}
/// <summary>
/// Writes a single performance metric to the specified writer.
/// </summary>
/// <param name="writer">The writer to write the metrics to.</param>
/// <param name="description">The metric description.</param>
/// <param name="metric">The metric.</param>
private static void SaveSinglePerformanceMetric(TextWriter writer, string description, IEnumerable <double> metric)
{
// Write description
writer.WriteLine("# " + description);
// Write metric
var mva = new MeanVarianceAccumulator();
foreach (double value in metric)
{
writer.Write("{0}, ", value);
mva.Add(value);
}
writer.WriteLine("{0}, {1}", mva.Mean, Math.Sqrt(mva.Variance));
writer.WriteLine();
}
public void SampleImproper()
{
Rand.Restart(96);
const double probChar = (double)1 / (char.MaxValue + 1);
const double StoppingProbability = probChar * 0.99;
var dist = StringDistribution.Any();
// The length of sequences sampled from this distribution will follow a geometric distribution
var acc = new MeanVarianceAccumulator();
const int SampleCount = 30000;
for (int i = 0; i < SampleCount; ++i)
{
string sample = dist.Sample();
acc.Add(sample.Length);
}
const double ExpectedMean = (1.0 - StoppingProbability) / StoppingProbability;
const double ExpectedVariance = (1.0 - StoppingProbability) / (StoppingProbability * StoppingProbability);
Assert.Equal(ExpectedMean, acc.Mean, 1e-2);
Assert.Equal(ExpectedVariance, acc.Variance, 1e-2);
}
public static Gamma PrecisionAverageConditional([SkipIfUniform] Gaussian sample, [SkipIfUniform] Gaussian mean, [SkipIfUniform] Gamma precision)
{
if (sample.IsPointMass && mean.IsPointMass) return PrecisionAverageConditional(sample.Point, mean.Point);
else if (precision.IsPointMass) {
// The correct answer here is not uniform, but rather a limit.
// However it doesn't really matter what we return since multiplication by a point mass
// always yields a point mass.
return Gamma.Uniform();
} else if (sample.IsUniform() || mean.IsUniform()) {
return Gamma.Uniform();
} else if (!precision.IsProper()) {
// improper prior
throw new ImproperMessageException(precision);
} else {
double mx, vx;
sample.GetMeanAndVariance(out mx, out vx);
double mm, vm;
mean.GetMeanAndVariance(out mm, out vm);
double m = mx-mm;
double v = vx+vm;
if (double.IsPositiveInfinity(v)) return Gamma.Uniform();
double m2 = m*m;
Gamma q = GaussianOp_Laplace.Q(sample, mean, precision, GaussianOp_Laplace.QInit());
double a = precision.Shape;
double b = precision.Rate;
double r = q.GetMean();
double rmin, rmax;
GetIntegrationBoundsForPrecision(r, m, v, a, b, out rmin, out rmax);
int n = 20000;
double inc = (Math.Log(rmax)-Math.Log(rmin))/(n-1);
MeanVarianceAccumulator mva = new MeanVarianceAccumulator();
double shift = 0;
for (int i = 0; i < n; i++) {
r = rmin*Math.Exp(i*inc);
double logp = -0.5*Math.Log(v+1/r) -0.5*m2/(v+1/r) + a*Math.Log(r) - b*r;
if (i == 0) shift = logp;
mva.Add(r, Math.Exp(logp-shift));
}
if (mva.Count == 0) throw new Exception("Quadrature found zero mass");
if (double.IsNaN(mva.Count)) throw new Exception("count is nan");
Gamma precMarginal = Gamma.FromMeanAndVariance(mva.Mean, mva.Variance);
Gamma result = new Gamma();
result.SetToRatio(precMarginal, precision, GaussianOp.ForceProper);
if (double.IsNaN(result.Rate)) throw new Exception("result is nan");
return result;
}
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="GammaPowerProductOp_Laplace"]/message_doc[@name="AAverageConditional(GammaPower, GammaPower, GammaPower, Gamma, GammaPower)"]/*'/>
public static GammaPower AAverageConditional([SkipIfUniform] GammaPower product, GammaPower A, [SkipIfUniform] GammaPower B, Gamma q, GammaPower result)
{
if (B.Shape < A.Shape)
{
return(BAverageConditional(product, B, A, q, result));
}
if (B.IsPointMass)
{
return(GammaProductOp.AAverageConditional(product, B.Point, result));
}
if (A.IsPointMass)
{
return(GammaPower.Uniform(A.Power)); // TODO
}
if (product.IsUniform())
{
return(product);
}
double qPoint = q.GetMean();
GammaPower aMarginal;
if (product.IsPointMass)
{
// Z = int Ga(y/q; s, r)/q Ga(q; q_s, q_r) dq
// E[a] = E[product/q]
// E[a^2] = E[product^2/q^2]
// aVariance = E[a^2] - aMean^2
double productPoint = product.Point;
if (productPoint == 0)
{
aMarginal = GammaPower.PointMass(0, result.Power);
}
else
{
double iqMean, iqVariance;
GetIQMoments(product, A, q, qPoint, out iqMean, out iqVariance);
double aMean = productPoint * iqMean;
double aVariance = productPoint * productPoint * iqVariance;
aMarginal = GammaPower.FromGamma(Gamma.FromMeanAndVariance(aMean, aVariance), result.Power);
}
}
else
{
if (double.IsPositiveInfinity(product.Rate))
{
return(GammaPower.PointMass(0, result.Power));
}
if (A.Power != product.Power)
{
throw new NotSupportedException($"A.Power ({A.Power}) != product.Power ({product.Power})");
}
if (B.Power != product.Power)
{
throw new NotSupportedException($"B.Power ({B.Power}) != product.Power ({product.Power})");
}
double r = product.Rate;
double g = 1 / (qPoint * r + A.Rate);
double g2 = g * g;
double shape2 = GammaFromShapeAndRateOp_Slow.AddShapesMinus1(product.Shape, A.Shape) + (1 - A.Power);
// From above:
// a^(y_s-pa + a_s-1) exp(-(y_r b + a_r)*a)
if (shape2 > 2)
{
// Compute the moments of a^(-1/a.Power)
// Here q = b^(1/b.Power)
// E[a^(-1/a.Power)] = E[(q r + a_r)/(shape2-1)]
// var(a^(-1/a.Power)) = E[(q r + a_r)^2/(shape2-1)/(shape2-2)] - E[a^(-1/a.Power)]^2
// = (var(q r + a_r) + E[(q r + a_r)]^2)/(shape2-1)/(shape2-2) - E[(q r + a_r)]^2/(shape2-1)^2
// = var(q r + a_r)/(shape2-1)/(shape2-2) + E[(q r + a_r)/(shape2-1)]^2/(shape2-2)
// TODO: share this computation with BAverageConditional
double qMean, qVariance;
GetQMoments(product, A, q, qPoint, out qMean, out qVariance);
double iaMean = (qMean * r + A.Rate) / (shape2 - 1);
//double iaVariance = (qVariance * r2 / (shape2 - 1) + iaMean * iaMean) / (shape2 - 2);
// shape = mean^2/variance + 2
//double iaVarianceOverMeanSquared = (qVariance / (shape2 - 1) * r / iaMean * r / iaMean + 1) / (shape2 - 2);
double iaVarianceOverMeanSquared = (qVariance * (shape2 - 1) / (qMean + A.Rate / r) / (qMean + A.Rate / r) + 1) / (shape2 - 2);
//GammaPower iaMarginal = GammaPower.FromMeanAndVariance(iaMean, iaVariance, -1);
GammaPower iaMarginal = InverseGammaFromMeanAndVarianceOverMeanSquared(iaMean, iaVarianceOverMeanSquared);
if (iaMarginal.IsUniform())
{
if (result.Power > 0)
{
return(GammaPower.PointMass(0, result.Power));
}
else
{
return(GammaPower.Uniform(result.Power));
}
}
else
{
aMarginal = GammaPower.FromShapeAndRate(iaMarginal.Shape, iaMarginal.Rate, result.Power);
}
bool check = false;
if (check)
{
// Importance sampling
MeanVarianceAccumulator mvaB = new MeanVarianceAccumulator();
MeanVarianceAccumulator mvaInvA = new MeanVarianceAccumulator();
Gamma bPrior = Gamma.FromShapeAndRate(B.Shape, B.Rate);
q = bPrior;
double shift = (product.Shape - product.Power) * Math.Log(qPoint) - shape2 * Math.Log(A.Rate + qPoint * r) + bPrior.GetLogProb(qPoint) - q.GetLogProb(qPoint);
for (int i = 0; i < 1000000; i++)
{
double bSample = q.Sample();
// logf = (y_s-y_p)*log(b) - (s+y_s-pa)*log(r + b*y_r)
double logf = (product.Shape - product.Power) * Math.Log(bSample) - shape2 * Math.Log(A.Rate + bSample * r) + bPrior.GetLogProb(bSample) - q.GetLogProb(bSample);
double weight = Math.Exp(logf - shift);
mvaB.Add(bSample, weight);
double invA = (bSample * r + A.Rate) / (shape2 - 1);
mvaInvA.Add(invA, weight);
}
Trace.WriteLine($"b = {mvaB}, {qMean}, {qVariance}");
Trace.WriteLine($"invA = {mvaInvA} {mvaInvA.Variance * (shape2 - 1) / (shape2 - 2) + mvaInvA.Mean * mvaInvA.Mean / (shape2 - 2)}, {iaMean}, {iaVarianceOverMeanSquared * iaMean * iaMean}");
Trace.WriteLine($"aMarginal = {aMarginal}");
}
}
else
{
// Compute the moments of a^(1/a.Power)
// aMean = shape2/(b y_r + a_r)
// aVariance = E[shape2*(shape2+1)/(b y_r + a_r)^2] - aMean^2 = var(shape2/(b y_r + a_r)) + E[shape2/(b y_r + a_r)^2]
// = shape2^2*var(1/(b y_r + a_r)) + shape2*(var(1/(b y_r + a_r)) + (aMean/shape2)^2)
double r2 = r * r;
double[] gDerivatives = new double[] { g, -r * g2, 2 * g2 * g * r2, -6 * g2 * g2 * r2 * r };
double gMean, gVariance;
GaussianOp_Laplace.LaplaceMoments(q, gDerivatives, dlogfs(qPoint, product, A), out gMean, out gVariance);
double aMean = shape2 * gMean;
double aVariance = shape2 * shape2 * gVariance + shape2 * (gVariance + gMean * gMean);
aMarginal = GammaPower.FromGamma(Gamma.FromMeanAndVariance(aMean, aVariance), result.Power);
}
}
result.SetToRatio(aMarginal, A, GammaProductOp_Laplace.ForceProper);
if (double.IsNaN(result.Shape) || double.IsNaN(result.Rate))
{
throw new InferRuntimeException("result is nan");
}
return(result);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="GammaPowerProductOp_Laplace"]/message_doc[@name="ProductAverageConditional(GammaPower, GammaPower, GammaPower, Gamma, GammaPower)"]/*'/>
public static GammaPower ProductAverageConditional(GammaPower product, [Proper] GammaPower A, [SkipIfUniform] GammaPower B, Gamma q, GammaPower result)
{
if (B.Shape < A.Shape)
{
return(ProductAverageConditional(product, B, A, q, result));
}
if (B.IsPointMass)
{
return(GammaProductOp.ProductAverageConditional(A, B.Point));
}
if (B.IsUniform())
{
return(GammaPower.Uniform(result.Power));
}
if (A.IsPointMass)
{
return(GammaProductOp.ProductAverageConditional(A.Point, B));
}
if (product.IsPointMass)
{
return(GammaPower.Uniform(result.Power)); // TODO
}
if (A.Power != product.Power)
{
throw new NotSupportedException($"A.Power ({A.Power}) != product.Power ({product.Power})");
}
if (B.Power != product.Power)
{
throw new NotSupportedException($"B.Power ({B.Power}) != product.Power ({product.Power})");
}
if (A.Rate == 0)
{
if (B.Rate == 0)
{
return(GammaPower.FromShapeAndRate(Math.Min(A.Shape, B.Shape), 0, result.Power));
}
else
{
return(A);
}
}
if (B.Rate == 0)
{
return(B);
}
double qPoint = q.GetMean();
double r = product.Rate;
double shape2 = GammaFromShapeAndRateOp_Slow.AddShapesMinus1(product.Shape, A.Shape) + (1 - A.Power);
GammaPower productMarginal;
// threshold ensures 6/qPoint^4 does not overflow
double threshold = Math.Sqrt(Math.Sqrt(6 / double.MaxValue));
if (shape2 > 2 && result.Power < 0 && qPoint > threshold)
{
// Compute the moments of product^(-1/product.Power)
// Here q = b^(1/b.Power)
// E[a^(-1/a.Power) b^(-1/b.Power)] = E[(q r + a_r)/(shape2-1)/q]
// var(a^(-1/a.Power) b^(-1/b.Power)) = E[(q r + a_r)^2/(shape2-1)/(shape2-2)/q^2] - E[a^(-1/a.Power) b^(-1/b.Power)]^2
// = (var((q r + a_r)/q) + E[(q r + a_r)/q]^2)/(shape2-1)/(shape2-2) - E[(q r + a_r)/q]^2/(shape2-1)^2
// = var((q r + a_r)/q)/(shape2-1)/(shape2-2) + E[(q r + a_r)/(shape2-1)/q]^2/(shape2-2)
double iqMean, iqVariance;
GetIQMoments(product, A, q, qPoint, out iqMean, out iqVariance);
double ipMean = (r + A.Rate * iqMean) / (shape2 - 1);
double ipVariance = (iqVariance * A.Rate * A.Rate / (shape2 - 1) + ipMean * ipMean) / (shape2 - 2);
// TODO: use ipVarianceOverMeanSquared
GammaPower ipMarginal = GammaPower.FromMeanAndVariance(ipMean, ipVariance, -1);
if (ipMarginal.IsUniform())
{
return(GammaPower.Uniform(result.Power));
}
else
{
productMarginal = GammaPower.FromShapeAndRate(ipMarginal.Shape, ipMarginal.Rate, result.Power);
}
bool check = false;
if (check)
{
// Importance sampling
MeanVarianceAccumulator mvaInvQ = new MeanVarianceAccumulator();
MeanVarianceAccumulator mvaInvProduct = new MeanVarianceAccumulator();
Gamma qPrior = Gamma.FromShapeAndRate(B.Shape, B.Rate);
double shift = (product.Shape - product.Power) * Math.Log(qPoint) - shape2 * Math.Log(A.Rate + qPoint * r) + qPrior.GetLogProb(qPoint) - q.GetLogProb(qPoint);
for (int i = 0; i < 1000000; i++)
{
double qSample = q.Sample();
// logf = (y_s-y_p)*log(b) - (s+y_s-pa)*log(r + b*y_r)
double logf = (product.Shape - product.Power) * Math.Log(qSample) - shape2 * Math.Log(A.Rate + qSample * r) + qPrior.GetLogProb(qSample) - q.GetLogProb(qSample);
double weight = Math.Exp(logf - shift);
mvaInvQ.Add(1 / qSample, weight);
double invProduct = (r + A.Rate / qSample) / (shape2 - 1);
mvaInvProduct.Add(invProduct, weight);
}
Trace.WriteLine($"invQ = {mvaInvQ}, {iqMean}, {iqVariance}");
Trace.WriteLine($"invProduct = {mvaInvProduct}");
Trace.WriteLine($"invA = {mvaInvProduct.Variance * (shape2 - 1) / (shape2 - 2) + mvaInvProduct.Mean * mvaInvProduct.Mean / (shape2 - 2)}, {ipMean}, {ipVariance}");
Trace.WriteLine($"productMarginal = {productMarginal}");
}
}
else
{
// Compute the moments of y = product^(1/product.Power)
// yMean = E[shape2*b/(b y_r + a_r)]
// yVariance = E[shape2*(shape2+1)*b^2/(b y_r + a_r)^2] - yMean^2
// = var(shape2*b/(b y_r + a_r)) + E[shape2*b^2/(b y_r + a_r)^2]
// = shape2^2*var(b/(b y_r + a_r)) + shape2*(var(b/(b y_r + a_r)) + (yMean/shape2)^2)
// Let g = b/(b y_r + a_r)
double denom = qPoint * r + A.Rate;
double denom2 = denom * denom;
double rOverDenom = r / denom;
double[] gDerivatives = (denom == 0)
? new double[] { 0, 0, 0, 0 }
: new double[] { qPoint / denom, A.Rate / denom2, -2 * A.Rate / denom2 * rOverDenom, 6 * A.Rate / denom2 * rOverDenom * rOverDenom };
double gMean, gVariance;
GaussianOp_Laplace.LaplaceMoments(q, gDerivatives, dlogfs(qPoint, product, A), out gMean, out gVariance);
double yMean = shape2 * gMean;
double yVariance = shape2 * shape2 * gVariance + shape2 * (gVariance + gMean * gMean);
productMarginal = GammaPower.FromGamma(Gamma.FromMeanAndVariance(yMean, yVariance), result.Power);
}
result.SetToRatio(productMarginal, product, GammaProductOp_Laplace.ForceProper);
if (double.IsNaN(result.Shape) || double.IsNaN(result.Rate))
{
throw new InferRuntimeException("result is nan");
}
return(result);
}
public static Gaussian DAverageConditional([SkipIfUniform] Gamma exp, [Proper] Gaussian d)
{
// as a function of d, the factor is Ga(exp(d); shape, rate) = exp(d*(shape-1) -rate*exp(d))
if (exp.IsUniform())
{
return(Gaussian.Uniform());
}
if (exp.IsPointMass)
{
return(ExpOp.DAverageConditional(exp.Point));
}
if (exp.Rate < 0)
{
throw new ImproperMessageException(exp);
}
if (exp.Rate == 0)
{
return(Gaussian.FromNatural(exp.Shape - 1, 0));
}
if (d.IsUniform())
{
if (exp.Shape <= 1)
{
throw new ArgumentException("The posterior has infinite variance due to input of Exp distributed as " + d + " and output of Exp distributed as " + exp +
" (shape <= 1)");
}
// posterior for d is a shifted log-Gamma distribution:
// exp((a-1)*d - b*exp(d)) =propto exp(a*(d+log(b)) - exp(d+log(b)))
// we find the Gaussian with same moments.
// u = d+log(b)
// E[u] = digamma(a-1)
// E[d] = E[u]-log(b) = digamma(a-1)-log(b)
// var(d) = var(u) = trigamma(a-1)
double lnRate = Math.Log(exp.Rate);
return(new Gaussian(MMath.Digamma(exp.Shape - 1) - lnRate, MMath.Trigamma(exp.Shape - 1)));
}
double aMinus1 = exp.Shape - 1;
double b = exp.Rate;
if (d.IsPointMass)
{
double x = d.Point;
double expx = Math.Exp(x);
double dlogf = aMinus1 - b * expx;
double ddlogf = -b * expx;
return(Gaussian.FromDerivatives(x, dlogf, ddlogf, true));
}
double dmode, dmin, dmax;
GetIntegrationBounds(exp, d, out dmode, out dmin, out dmax);
double expmode = Math.Exp(dmode);
int n = QuadratureNodeCount;
double inc = (dmax - dmin) / (n - 1);
MeanVarianceAccumulator mva = new MeanVarianceAccumulator();
for (int i = 0; i < n; i++)
{
double x = dmin + i * inc;
double xMinusMode = x - dmode;
double diff = aMinus1 * xMinusMode - b * (Math.Exp(x) - expmode)
- 0.5 * ((x * x - dmode * dmode) * d.Precision - 2 * xMinusMode * d.MeanTimesPrecision);
double p = Math.Exp(diff);
mva.Add(x, p);
if (double.IsNaN(mva.Variance))
{
throw new Exception();
}
}
double dMean = mva.Mean;
double dVariance = mva.Variance;
Gaussian result = Gaussian.FromMeanAndVariance(dMean, dVariance);
result.SetToRatio(result, d, true);
return(result);
}
public static Gamma RateAverageConditional([SkipIfUniform] Gamma sample, double shape, Gamma rate)
{
if (rate.IsPointMass) return Gamma.Uniform();
if (sample.IsPointMass) return RateAverageConditional(sample.Point, shape);
if (sample.Rate == 0) return Gamma.Uniform();
double shape1 = AddShapesMinus1(shape, rate.Shape);
double shape2 = AddShapesMinus1(shape, sample.Shape);
double rateMean, rateVariance;
double r, rmin, rmax;
GetIntegrationBoundsForRate(sample, shape, rate, out r, out rmin, out rmax);
int n = QuadratureNodeCount;
double inc = (rmax-rmin)/(n-1);
double shift = shape1*Math.Log(r) - shape2*Math.Log(r+sample.Rate) - r*rate.Rate;
MeanVarianceAccumulator mva = new MeanVarianceAccumulator();
for (int i = 0; i < n; i++) {
double x = rmin + i*inc;
double logp = shape1*Math.Log(x) - shape2*Math.Log(x+sample.Rate) - x*rate.Rate;
if ((i == 0 || i == n-1) && (logp-shift > -50)) throw new Exception("invalid integration bounds");
double p = Math.Exp(logp - shift);
mva.Add(x, p);
}
rateMean = mva.Mean;
rateVariance = mva.Variance;
Gamma rateMarginal = Gamma.FromMeanAndVariance(rateMean, rateVariance);
Gamma result = new Gamma();
result.SetToRatio(rateMarginal, rate, true);
if (double.IsNaN(result.Shape) || double.IsNaN(result.Rate)) throw new Exception("result is nan");
return result;
}
public static Gamma SampleAverageConditional(Gamma sample, double shape, [SkipIfUniform] Gamma rate)
{
if (sample.IsPointMass || rate.Rate == 0) return Gamma.Uniform();
if (rate.IsPointMass) return SampleAverageConditional(shape, rate.Point);
double shape1 = AddShapesMinus1(shape, rate.Shape);
double shape2 = AddShapesMinus1(shape, sample.Shape);
double sampleMean, sampleVariance;
if (sample.Rate == 0) sample = Gamma.FromShapeAndRate(sample.Shape, 1e-20);
if (sample.Rate == 0) {
sampleMean = shape2*rate.GetMeanInverse();
sampleVariance = shape2*(1+shape2)*rate.GetMeanPower(-2) - sampleMean*sampleMean;
} else if(true) {
// quadrature over sample
double y, ymin, ymax;
GetIntegrationBoundsForSample(sample, shape, rate, out y, out ymin, out ymax);
int n = QuadratureNodeCount;
double inc = (ymax-ymin)/(n-1);
shape1 = sample.Shape+shape-2;
shape2 = shape+rate.Shape;
double shift = shape1*Math.Log(y) - shape2*Math.Log(y+rate.Rate) - y*sample.Rate;
MeanVarianceAccumulator mva = new MeanVarianceAccumulator();
for (int i = 0; i < n; i++) {
double x = ymin + i*inc;
double logp = shape1*Math.Log(x) - shape2*Math.Log(x+rate.Rate) - x*sample.Rate;
//if (i == 0 || i == n-1) Console.WriteLine(logp-shift);
if ((i == 0 || i == n-1) && (logp-shift > -50)) throw new Exception("invalid integration bounds");
double p = Math.Exp(logp - shift);
mva.Add(x, p);
}
sampleMean = mva.Mean;
sampleVariance = mva.Variance;
} else {
// quadrature over rate
// sampleMean = E[ shape2/(sample.Rate + r) ]
// sampleVariance = var(shape2/(sample.Rate + r)) + E[ shape2/(sample.Rate+r)^2 ]
// = shape2^2*var(1/(sample.Rate + r)) + shape2*(var(1/(sample.Rate+r)) + (sampleMean/shape2)^2)
double r, rmin, rmax;
GetIntegrationBoundsForRate(sample, shape, rate, out r, out rmin, out rmax);
int n = QuadratureNodeCount;
double inc = (rmax-rmin)/(n-1);
double shift = shape1*Math.Log(r) - shape2*Math.Log(r+sample.Rate) - r*rate.Rate;
MeanVarianceAccumulator mva = new MeanVarianceAccumulator();
for (int i = 0; i < n; i++) {
double x = rmin + i*inc;
double logp = shape1*Math.Log(x) - shape2*Math.Log(x+sample.Rate) - x*rate.Rate;
//if (i == 0 || i == n-1) Console.WriteLine(logp-shift);
if ((i == 0 || i == n-1) && (logp-shift > -50)) throw new Exception("invalid integration bounds");
double p = Math.Exp(logp - shift);
double f = 1/(x + sample.Rate);
mva.Add(f, p);
}
sampleMean = shape2*mva.Mean;
sampleVariance = shape2*(1+shape2)*mva.Variance + shape2*mva.Mean*mva.Mean;
}
Gamma sampleMarginal = Gamma.FromMeanAndVariance(sampleMean, sampleVariance);
Gamma result = new Gamma();
result.SetToRatio(sampleMarginal, sample, true);
if (double.IsNaN(result.Shape) || double.IsNaN(result.Rate)) throw new Exception("result is nan");
return result;
}
/// <summary>
/// Finds the input that maximizes a function.
/// </summary>
/// <param name="bounds">Bounds for the search.</param>
/// <param name="Evaluate">The function to maximize.</param>
/// <param name="GetUpperBound">Returns an upper bound to the function in a region. Need not be tight, but must become tight as the region shrinks.</param>
/// <param name="xTolerance">Allowable relative error in the solution on any dimension. Must be greater than zero.</param>
/// <returns>A Vector close to the global maximum of the function.</returns>
public static Vector Search(Region bounds, Func <Vector, double> Evaluate, Func <Region, double> GetUpperBound, double xTolerance = 1e-4)
{
if (xTolerance <= 0)
{
throw new ArgumentOutOfRangeException($"xTolerance <= 0");
}
int dim = bounds.Lower.Count;
if (dim == 0)
{
return(Vector.Zero(dim));
}
PriorityQueue <QueueNode> queue = new PriorityQueue <QueueNode>();
double lowerBound = double.NegativeInfinity;
Vector argmax = bounds.GetMidpoint();
long upperBoundCount = 0;
if (Debug)
{
Func <Region, double> GetUpperBound1 = GetUpperBound;
GetUpperBound = region =>
{
Stopwatch watch = Stopwatch.StartNew();
double upperBound = GetUpperBound1(region);
watch.Stop();
if (timeAccumulator.Count > 10 && watch.ElapsedMilliseconds > timeAccumulator.Mean + 4 * Math.Sqrt(timeAccumulator.Variance))
{
Trace.WriteLine($"GetUpperBound took {watch.ElapsedMilliseconds}ms");
}
timeAccumulator.Add(watch.ElapsedMilliseconds);
//if (upperBoundCount % 100 == 0) Trace.WriteLine($"lowerBound = {lowerBound}");
return(upperBound);
};
}
Action <Region, int, double> addRegion = delegate(Region region, int splitDim, double upperBound)
{
if (upperBound > lowerBound)
{
if (Debug)
{
Trace.WriteLine($"added region {region} with upperBound = {upperBound}");
}
QueueNode node = new QueueNode(region, upperBound);
queue.Add(node);
}
else if (Debug)
{
Trace.WriteLine($"rejected region {region} with upperBound {upperBound} <= lowerBound {lowerBound}");
}
};
upperBoundCount++;
addRegion(bounds, 0, GetUpperBound(bounds));
while (queue.Count > 0)
{
var node = queue.ExtractMinimum(); // gets the node with highest upper bound
if (node.UpperBound <= lowerBound)
{
continue;
}
Region region = node.Region;
// compute the lower bound
Vector midpoint = region.GetMidpoint();
Stopwatch watch = Stopwatch.StartNew();
double nodeLowerBound = Evaluate(midpoint);
watch.Stop();
if (Debug)
{
if (timeAccumulator.Count > 10 && watch.ElapsedMilliseconds > timeAccumulator.Mean + 4 * Math.Sqrt(timeAccumulator.Variance))
{
Trace.WriteLine($"Evaluate took {watch.ElapsedMilliseconds}ms");
}
timeAccumulator.Add(watch.ElapsedMilliseconds);
Trace.WriteLine($"expanding {node} lower bound = {nodeLowerBound}");
}
if (nodeLowerBound > node.UpperBound)
{
throw new Exception("nodeLowerBound > node.UpperBound");
}
if (nodeLowerBound > lowerBound)
{
argmax = midpoint;
lowerBound = nodeLowerBound;
}
Func <int, bool> DimensionCanSplit = i => MMath.AbsDiff(region.Upper[i], region.Lower[i], 1e-10) >= xTolerance;
int splitDim;
bool lowestChild = false;
Region leftRegion = null;
double upperBoundLeft = default(double);
Region rightRegion = null;
double upperBoundRight = default(double);
if (lowestChild)
{
splitDim = -1;
double lowestUpperBound = double.PositiveInfinity;
for (int i = 0; i < dim; i++)
{
if (DimensionCanSplit(i))
{
double splitValue2 = midpoint[i];
Region leftRegion2 = new Region(region);
leftRegion2.Upper[i] = splitValue2;
upperBoundCount++;
double upperBoundLeft2 = GetUpperBound(leftRegion2);
Region rightRegion2 = new Region(region);
rightRegion2.Lower[i] = splitValue2;
upperBoundCount++;
double upperBoundRight2 = GetUpperBound(rightRegion2);
double lowerUpperBound = Math.Min(upperBoundLeft2, upperBoundRight2);
if (lowerUpperBound < lowestUpperBound)
{
lowestUpperBound = lowerUpperBound;
upperBoundLeft = upperBoundLeft2;
upperBoundRight = upperBoundRight2;
leftRegion = leftRegion2;
rightRegion = rightRegion2;
splitDim = i;
}
}
}
if (splitDim < 0)
{
break;
}
}
else
{
// Find a dimension to split on.
splitDim = Rand.Int(dim);
bool foundSplit = false;
for (int i = 0; i < dim; i++)
{
if (!DimensionCanSplit(splitDim))
{
splitDim++;
if (splitDim == dim)
{
splitDim = 0;
}
}
else
{
foundSplit = true;
break;
}
}
if (!foundSplit)
{
break;
}
}
// split the node
double splitValue = midpoint[splitDim];
if (Debug)
{
Trace.WriteLine($"splitting dimension {splitDim}");
}
if (region.Upper[splitDim] != splitValue)
{
if (leftRegion == null)
{
leftRegion = new Region(region);
leftRegion.Upper[splitDim] = splitValue;
upperBoundCount++;
upperBoundLeft = GetUpperBound(leftRegion);
}
addRegion(leftRegion, splitDim, upperBoundLeft);
}
if (region.Lower[splitDim] != splitValue)
{
if (rightRegion == null)
{
rightRegion = new Region(region);
rightRegion.Lower[splitDim] = splitValue;
upperBoundCount++;
upperBoundRight = GetUpperBound(rightRegion);
}
addRegion(rightRegion, splitDim, upperBoundRight);
}
}
if (Debug)
{
Trace.WriteLine($"BranchAndBound.Search upperBoundCount = {upperBoundCount}");
}
return(argmax);
}
/// <summary>
/// EP message to 'precision'
/// </summary>
/// <param name="sample">Incoming message from 'sample'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="mean">Incoming message from 'mean'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="precision">Incoming message from 'precision'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <returns>The outgoing EP message to the 'precision' argument</returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'precision' as the random arguments are varied.
/// The formula is <c>proj[p(precision) sum_(sample,mean) p(sample,mean) factor(sample,mean,precision)]/p(precision)</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="sample"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="mean"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="precision"/> is not a proper distribution</exception>
public static Gamma PrecisionAverageConditional([SkipIfUniform] Gaussian sample, [SkipIfUniform] Gaussian mean, [SkipIfUniform] Gamma precision, Gamma to_precision)
{
if (sample.IsPointMass && mean.IsPointMass)
return PrecisionAverageConditional(sample.Point, mean.Point);
Gamma result = new Gamma();
if (precision.IsPointMass) {
// The correct answer here is not uniform, but rather a limit.
// However it doesn't really matter what we return since multiplication by a point mass
// always yields a point mass.
result.SetToUniform();
} else if (sample.IsUniform() || mean.IsUniform()) {
result.SetToUniform();
} else if (!precision.IsProper()) {
// improper prior
throw new ImproperMessageException(precision);
} else {
//to_precision.SetToUniform(); // TEMPORARY
// use quadrature to integrate over the precision
// see LogAverageFactor
double xm, xv, mm, mv;
sample.GetMeanAndVarianceImproper(out xm, out xv);
mean.GetMeanAndVarianceImproper(out mm, out mv);
double upperBound = Double.PositiveInfinity;
if (xv + mv < 0) upperBound = -1.0 / (xv + mv);
double[] nodes = new double[QuadratureNodeCount];
double[] logWeights = new double[nodes.Length];
Gamma precMarginal = precision*to_precision;
QuadratureNodesAndWeights(precMarginal, nodes, logWeights);
if (!to_precision.IsUniform()) {
// modify the weights
for (int i = 0; i < logWeights.Length; i++) {
logWeights[i] += precision.GetLogProb(nodes[i]) - precMarginal.GetLogProb(nodes[i]);
}
}
double shift = 0;
MeanVarianceAccumulator mva = new MeanVarianceAccumulator();
for (int i = 0; i < nodes.Length; i++) {
double v = 1.0 / nodes[i] + xv + mv;
if (v < 0) continue;
double lp = Gaussian.GetLogProb(xm, mm, v);
if (shift == 0) shift = lp;
double f = Math.Exp(logWeights[i] + lp - shift);
mva.Add(nodes[i], f);
}
// Adaptive Clenshaw-Curtis quadrature: gives same results on easy integrals but
// still fails ExpFactorTest2
//Converter<double,double> func = delegate(double y) {
// double x = Math.Exp(y);
// double v = 1.0 / x + xv + mv;
// if (v < 0) return 0.0;
// return Math.Exp(Gaussian.GetLogProb(xm, mm, v) + Gamma.GetLogProb(x, precision.Shape+1, precision.Rate));
//};
//double Z2 = BernoulliFromLogOddsOp.AdaptiveClenshawCurtisQuadrature(func, 1, 16, 1e-10);
//Converter<double, double> func2 = delegate(double y)
//{
// return Math.Exp(y) * func(y);
//};
//double rmean2 = BernoulliFromLogOddsOp.AdaptiveClenshawCurtisQuadrature(func2, 1, 16, 1e-10);
//Converter<double, double> func3 = delegate(double y)
//{
// return Math.Exp(2*y) * func(y);
//};
//double rvariance2 = BernoulliFromLogOddsOp.AdaptiveClenshawCurtisQuadrature(func3, 1, 16, 1e-10);
//rmean2 = rmean2/ Z2;
//rvariance2 = rvariance2 / Z2 - rmean2 * rmean2;
double Z = mva.Count;
if (double.IsNaN(Z)) throw new Exception("Z is nan");
if (Z == 0.0) {
throw new Exception("Quadrature found zero mass");
//Console.WriteLine("Warning: Quadrature found zero mass. Results may be inaccurate.");
result.SetToUniform();
return result;
}
double rmean = mva.Mean;
double rvariance = mva.Variance;
if (rvariance == 0) throw new Exception("Quadrature found zero variance");
if (Double.IsInfinity(rmean)) {
result.SetToUniform();
} else {
result.SetMeanAndVariance(rmean, rvariance);
result.SetToRatio(result, precision, ForceProper);
}
}
#if KeepLastMessage
if (LastPrecisionMessage != null) {
if (Stepsize != 1 && Stepsize != 0) {
LastPrecisionMessage.SetToPower(LastPrecisionMessage, 1 - Stepsize);
result.SetToPower(result, Stepsize);
result.SetToProduct(result, LastPrecisionMessage);
}
}
// FIXME: this is not entirely safe since the caller could overwrite result.
LastPrecisionMessage = result;
#endif
return result;
}