//-- Constant bounds --------------------------------------------------------------------------------
/// <summary>
/// The logarithm of the probability that L <= X < U.
/// </summary>
/// <param name="X"></param>
/// <param name="L">Can be negative infinity.</param>
/// <param name="U">Can be positive infinity.</param>
/// <returns></returns>
public static double LogProbBetween(Gaussian X, double L, double U)
{
if (L > U) throw new AllZeroException("low > high (" + L + " > " + U + ")");
if (X.IsPointMass)
{
return Factor.IsBetween(X.Point, L, U) ? 0.0 : Double.NegativeInfinity;
}
else if (X.IsUniform())
{
if (Double.IsNegativeInfinity(L))
{
if (Double.IsPositiveInfinity(U)) return 0.0; // always between
else return -MMath.Ln2; // between half the time
}
else if (Double.IsPositiveInfinity(U)) return -MMath.Ln2; // between half the time
else return Double.NegativeInfinity; // never between two finite numbers
}
else
{
double sqrtPrec = Math.Sqrt(X.Precision);
double mx = X.GetMean();
double pl = MMath.NormalCdfLn(sqrtPrec * (L - mx)); // log(p(x <= L))
double pu = MMath.NormalCdfLn(sqrtPrec * (U - mx)); // log(p(x <= U))
if (pl == pu) return Double.NegativeInfinity;
if (Double.IsNegativeInfinity(pl)) return pu;
// log(NormalCdf(yu) - NormalCdf(yl)) = NormalCdfLn(yu) + log(1 - NormalCdf(yl)/NormalCdf(yu))
return pu + MMath.Log1MinusExp(pl - pu);
}
}
// logw1 = N(mx;m1,vx+v1) phi((mx1 - m2)/sqrt(vx1+v2))
// a1 = N(mx1;m2,vx1+v2)/phi
internal static void ComputeStats(Gaussian max, Gaussian a, Gaussian b, out double logz,
out double logw1, out double a1, out double vx1, out double mx1,
out double logw2, out double a2, out double vx2, out double mx2)
{
double m1, v1, m2, v2;
a.GetMeanAndVariance(out m1, out v1);
b.GetMeanAndVariance(out m2, out v2);
if (max.IsPointMass) {
vx1 = 0.0;
mx1 = max.Point;
vx2 = 0.0;
mx2 = max.Point;
if (b.IsPointMass) {
if (b.Point > max.Point) throw new AllZeroException();
else if (b.Point == max.Point) {
// the factor reduces to the constraint (max.Point > a)
logw1 = Double.NegativeInfinity;
logw2 = MMath.NormalCdfLn((max.Point - m1)/Math.Sqrt(v1));
logz = logw2;
a1 = 0;
a2 = Math.Exp(Gaussian.GetLogProb(max.Point, m1, v1) - logw2);
return;
} else {
// b.Point < max.Point
// the factor reduces to the constraint (a == max.Point)
throw new NotImplementedException();
}
} else if (a.IsPointMass) throw new NotImplementedException();
} else {
if (a.IsPointMass) {
vx1 = 0.0;
mx1 = a.Point;
} else {
vx1 = 1.0 / (max.Precision + a.Precision);
mx1 = vx1 * (max.MeanTimesPrecision + a.MeanTimesPrecision);
}
if (b.IsPointMass) {
vx2 = 0.0;
mx2 = b.Point;
} else {
vx2 = 1.0 / (max.Precision + b.Precision);
mx2 = vx2 * (max.MeanTimesPrecision + b.MeanTimesPrecision);
}
}
logw1 = max.GetLogAverageOf(a);
double logPhi1 = MMath.NormalCdfLn((mx1 - m2) / Math.Sqrt(vx1 + v2));
logw1 += logPhi1;
logw2 = max.GetLogAverageOf(b);
double logPhi2 = MMath.NormalCdfLn((mx2 - m1) / Math.Sqrt(vx2 + v1));
logw2 += logPhi2;
logz = MMath.LogSumExp(logw1, logw2);
double logN1 = Gaussian.GetLogProb(mx1, m2, vx1 + v2);
a1 = Math.Exp(logN1 - logPhi1);
double logN2 = Gaussian.GetLogProb(mx2, m1, vx2 + v1);
a2 = Math.Exp(logN2 - logPhi2);
}
public Gaussian[] InferTomorrowsTime()
{
Gaussian[] tomorrowsTime = new Gaussian[2];
tomorrowsTime[0] = cyclist1.InferTomorrowsTime();
tomorrowsTime[1] = cyclist2.InferTomorrowsTime();
return tomorrowsTime;
}
/// <summary>
/// VMP message to 'product'
/// </summary>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="B">Incoming message from 'b'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'product' as the random arguments are varied.
/// The formula is <c>proj[sum_(b) p(b) factor(product,a,b)]</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="B"/> is not a proper distribution</exception>
public static Gaussian ProductAverageLogarithm(double A, [SkipIfUniform] Beta B)
{
double mb, vb;
B.GetMeanAndVariance(out mb, out vb);
Gaussian result = new Gaussian();
result.SetMeanAndVariance(A * mb, A * A * vb);
return result;
}
public void Base_ExpectedValue()
{
var Mu = 5.0;
var X = new Gaussian(Mu, 2.0);
var E = X.ExpectedValueWithConfidence().Mean;
var Err = Math.Abs(E - Mu) / Mu;
Assert.IsTrue(Err < eps);
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="logistic">Constant value for 'logistic'.</param>
/// <param name="x">Incoming message from 'x'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(x) p(x) factor(logistic,x))</c>.
/// </para></remarks>
public static double LogAverageFactor(double logistic, Gaussian x)
{
if (logistic >= 1.0 || logistic <= 0.0) return x.GetLogProb(MMath.Logit(logistic));
// p(y,x) = delta(y - 1/(1+exp(-x))) N(x;mx,vx)
// x = log(y/(1-y))
// dx = 1/(y*(1-y))
return x.GetLogProb(MMath.Logit(logistic))/(logistic*(1-logistic));
}
public static Gaussian UsesAverageLogarithm(NonconjugateGaussian[] Uses, Gaussian Def, Gaussian result)
{
NonconjugateGaussian prod = Uses[0];
for (int i = 1; i < Uses.Length; i++)
prod.SetToProduct(prod, Uses[i]);
result = prod.GetGaussian(true);
result.SetToProduct(result, Def);
return result;
}
/// <summary>
/// VMP message to 'innerProduct'
/// </summary>
/// <param name="AMean">Buffer 'AMean'.</param>
/// <param name="AVariance">Buffer 'AVariance'.</param>
/// <param name="BMean">Buffer 'BMean'.</param>
/// <param name="BVariance">Buffer 'BVariance'.</param>
/// <returns>The outgoing VMP message to the 'innerProduct' argument</returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'innerProduct' conditioned on the given values.
/// </para></remarks>
public static Gaussian InnerProductAverageLogarithm(Vector AMean, PositiveDefiniteMatrix AVariance, Vector BMean, PositiveDefiniteMatrix BVariance)
{
Gaussian result = new Gaussian();
// p(x|a,b) = N(E[a]'*E[b], E[b]'*var(a)*E[b] + E[a]'*var(b)*E[a] + trace(var(a)*var(b)))
// Uses John Winn's rule for deterministic factors.
// Strict variational inference would set the variance to 0.
result.SetMeanAndVariance(AMean.Inner(BMean), AVariance.QuadraticForm(BMean) + BVariance.QuadraticForm(AMean) + AVariance.Inner(BVariance));
return result;
}
/// <summary>
/// VMP message to 'innerProduct'
/// </summary>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="BMean">Buffer 'BMean'.</param>
/// <param name="BVariance">Buffer 'BVariance'.</param>
/// <returns>The outgoing VMP message to the 'innerProduct' argument</returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'innerProduct' conditioned on the given values.
/// </para></remarks>
public static Gaussian InnerProductAverageLogarithm(Vector A, Vector BMean, PositiveDefiniteMatrix BVariance)
{
Gaussian result = new Gaussian();
// Uses John Winn's rule for deterministic factors.
// Strict variational inference would set the variance to 0.
// p(x) = N(a' E[b], a' var(b) a)
result.SetMeanAndVariance(A.Inner(BMean), BVariance.QuadraticForm(A));
return result;
}
/// <summary>
/// VMP message to 'a'
/// </summary>
/// <param name="Product">Incoming message from 'product'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="B">Incoming message from 'b'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the exponential of the average log-factor value, where the average is over all arguments except 'a'.
/// Because the factor is deterministic, 'product' is integrated out before taking the logarithm.
/// The formula is <c>exp(sum_(b) p(b) log(sum_product p(product) factor(product,a,b)))</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="Product"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="B"/> is not a proper distribution</exception>
public static Gaussian AAverageLogarithm([SkipIfUniform] Gaussian Product, [Proper] Beta B)
{
if (B.IsPointMass) return GaussianProductVmpOp.AAverageLogarithm(Product, B.Point);
if (Product.IsPointMass) return AAverageLogarithm(Product.Point, B);
double mb, vb;
B.GetMeanAndVariance(out mb, out vb);
Gaussian result = new Gaussian();
result.Precision = Product.Precision * (vb + mb*mb);
result.MeanTimesPrecision = Product.MeanTimesPrecision * mb;
return result;
}
/// <summary>
/// Gradient matching VMP message from factor to logOdds variable
/// </summary>
/// <param name="sample">Incoming message from 'sample'.</param>
/// <param name="logOdds">Incoming message. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="result">Previous message sent, used for damping</param>
/// <returns>The outgoing VMP message.</returns>
/// <remarks><para>
/// The outgoing message is the Gaussian approximation to the factor which results in the
/// same derivatives of the KL(q||p) divergence with respect to the parameters of the posterior
/// as if the true factor had been used.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="logOdds"/> is not a proper distribution</exception>
public static Gaussian LogRateAverageLogarithm(int sample, [Proper, SkipIfUniform] Gaussian logRate, Gaussian result)
{
double m,v;
logRate.GetMeanAndVariance(out m,out v);
Gaussian message = new Gaussian();
message.Precision = Math.Exp(m + v / 2);
message.MeanTimesPrecision = (m - 1) * Math.Exp(m + v / 2) + sample;
if (damping == 0)
return message;
else
return (message ^ (1 - damping)) * (result ^ damping);
}
private static void TestEnvelope()
{
Gaussian gaussian = new Gaussian(1, 1);
ExponentialEnvelope env = new ExponentialEnvelope(double.NegativeInfinity, double.PositiveInfinity, gaussian, new double[] { -2, -1, 0, 1, 2 });
for (int i = 0; i < 10000; i++)
{
double test = env.SampleContinuous();
Console.WriteLine(test);
}
Console.WriteLine("All Worked!");
}
/// <summary>
/// VMP message to 'a'
/// </summary>
/// <param name="Product">Incoming message from 'product'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="B">Incoming message from 'b'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <returns>The outgoing VMP message to the 'a' argument</returns>
/// <remarks><para>
/// The outgoing message is the exponential of the average log-factor value, where the average is over all arguments except 'a'.
/// Because the factor is deterministic, 'product' is integrated out before taking the logarithm.
/// The formula is <c>exp(sum_(b) p(b) log(sum_product p(product) factor(product,a,b)))</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="Product"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="B"/> is not a proper distribution</exception>
public static Gaussian AAverageLogarithm([SkipIfUniform] Gaussian Product, [Proper] Gamma B)
{
if (B.IsPointMass) return GaussianProductVmpOp.AAverageLogarithm(Product, B.Point);
if (Product.IsPointMass) return AAverageLogarithm(Product.Point, B);
if (!B.IsProper()) throw new ImproperMessageException(B);
double mb, vb;
B.GetMeanAndVariance(out mb, out vb);
// catch uniform case to avoid 0*Inf
if (Product.IsUniform()) return Product;
Gaussian result = new Gaussian();
result.Precision = Product.Precision * (vb + mb*mb);
result.MeanTimesPrecision = Product.MeanTimesPrecision * mb;
return result;
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="isPositive">Incoming message from 'isPositive'.</param>
/// <param name="x">Incoming message from 'x'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(isPositive,x) p(isPositive,x) factor(isPositive,x))</c>.
/// </para></remarks>
public static double LogAverageFactor(Bernoulli isPositive, Gaussian x)
{
if (isPositive.IsPointMass && x.Precision == 0) {
double tau = x.MeanTimesPrecision;
if (isPositive.Point && tau < 0) {
// int I(x>0) exp(tau*x) dx = -1/tau
return -Math.Log(-tau);
}
if (!isPositive.Point && tau > 0) {
// int I(x<0) exp(tau*x) dx = 1/tau
return -Math.Log(tau);
}
}
Bernoulli to_isPositive = IsPositiveAverageConditional(x);
return isPositive.GetLogAverageOf(to_isPositive);
}
public void Gaussian_Bernoulli_Conditional()
{
// arrange
Uncertain<double> X = new Gaussian(1.0, 1.0);
Uncertain<double> Y = new Gaussian(4.0, 2.0);
// act
if ((X > Y).Pr()) {
Assert.Fail("X > Y evaluates true, incorrectly");
}
if ((Y < X).Pr()) {
Assert.Fail("Y < X evaluates true, incorrectly");
}
if (!(Y > X).Pr()) {
Assert.Fail("Y > X evaluates false, incorrectly");
}
if (!(X < Y).Pr()) {
Assert.Fail("X < Y evaluates false, incorrectly");
}
}
public void Gaussian_Bernoulli_Mean()
{
// arrange
Uncertain<double> X = new Gaussian(1.0, 1.0);
Uncertain<double> Y = new Gaussian(3.0, 2.0);
var Z = X > Y;
var sampler = Sampler.Create(Z);
int k = 0;
// act
foreach (var s in sampler.Take(100))
if (s.Value) k += 1;
// assert
// Y - X is Gaussian(1, sqrt(5)), and has a 32.74% chance of being < 0
// (i.e. 32.74% chance that X > Y)
// The normal approximation to the binomial distribution says that the
// 99.9997% confidence interval with n=100, p=0.3274 is +/- 0.1883.
// So the confidence interval is roughly [0.13, 0.52].
Assert.IsTrue(k >= 13 && k < 52); // flaky at N = 100!
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="isPositive">Incoming message from 'isPositive'.</param>
/// <param name="x">Incoming message from 'x'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(isPositive,x) p(isPositive,x) factor(isPositive,x))</c>.
/// </para></remarks>
public static double LogAverageFactor(Bernoulli isPositive, Gaussian x)
{
Bernoulli to_isPositive = IsPositiveAverageConditional(x);
return isPositive.GetLogAverageOf(to_isPositive);
#if false
// Z = p(b=T) p(x > 0) + p(b=F) p(x <= 0)
// = p(b=F) + (p(b=T) - p(b=F)) p(x > 0)
if (x.IsPointMass) {
return Factor.IsPositive(x.Point) ? isPositive.GetLogProbTrue() : isPositive.GetLogProbFalse();
} else if(x.IsUniform()) {
return Bernoulli.LogProbEqual(isPositive.LogOdds,0.0);
} else {
// m/sqrt(v) = (m/v)/sqrt(1/v)
double z = x.MeanTimesPrecision / Math.Sqrt(x.Precision);
if (isPositive.IsPointMass) {
return isPositive.Point ? MMath.NormalCdfLn(z) : MMath.NormalCdfLn(-z);
} else {
return MMath.LogSumExp(isPositive.GetLogProbTrue() + MMath.NormalCdfLn(z), isPositive.GetLogProbFalse() + MMath.NormalCdfLn(-z));
}
}
#endif
}
/// <summary>
/// EP message to 'sample'
/// </summary>
/// <param name="mean">Incoming message from 'mean'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="precision">Constant value for 'precision'.</param>
/// <returns>The outgoing EP message to the 'sample' argument</returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'sample' as the random arguments are varied.
/// The formula is <c>proj[p(sample) sum_(mean) p(mean) factor(sample,mean,precision)]/p(sample)</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="mean"/> is not a proper distribution</exception>
public static Gaussian SampleAverageConditional([SkipIfUniform] Gaussian mean, double precision)
{
if (mean.IsPointMass) return SampleAverageConditional(mean.Point, precision);
// if (precision < 0) throw new ArgumentException("The constant precision given to the Gaussian factor is negative", "precision");
if (precision == 0) {
return Gaussian.Uniform();
} else if (double.IsPositiveInfinity(precision)) {
return mean;
} else {
if (mean.Precision <= -precision) throw new ImproperMessageException(mean);
// The formula is int_mean N(x;mean,1/prec) p(mean) = N(x; mm, mv + 1/prec)
// sample.Precision = inv(mv + inv(prec)) = mprec*prec/(prec + mprec)
// sample.MeanTimesPrecision = sample.Precision*mm = R*(mprec*mm)
// R = Prec/(Prec + mean.Prec)
// This code works for mean.IsUniform() since then mean.Precision = 0, mean.MeanTimesPrecision = 0
Gaussian result = new Gaussian();
double R = precision / (precision + mean.Precision);
result.Precision = R * mean.Precision;
result.MeanTimesPrecision = R * mean.MeanTimesPrecision;
return result;
}
}
// Generates data from the model
void GenerateData(
int numUsers,
int numItems,
int numTraits,
int numObservations,
int numLevels,
VariableArray <int> userData,
VariableArray <int> itemData,
VariableArray <VariableArray <bool>, bool[][]> ratingData,
Gaussian[][] userTraitsPrior,
Gaussian[][] itemTraitsPrior,
Gaussian[] userBiasPrior,
Gaussian[] itemBiasPrior,
Gaussian[][] userThresholdsPrior,
double affinityNoiseVariance,
double thresholdsNoiseVariance)
{
int[] generatedUserData = new int[numObservations];
int[] generatedItemData = new int[numObservations];
bool[][] generatedRatingData = new bool[numObservations][];
// Sample model parameters from the priors
Rand.Restart(12347);
double[][] userTraits = Util.ArrayInit(numUsers, u => Util.ArrayInit(numTraits, t => userTraitsPrior[u][t].Sample()));
double[][] itemTraits = Util.ArrayInit(numItems, i => Util.ArrayInit(numTraits, t => itemTraitsPrior[i][t].Sample()));
double[] userBias = Util.ArrayInit(numUsers, u => userBiasPrior[u].Sample());
double[] itemBias = Util.ArrayInit(numItems, i => itemBiasPrior[i].Sample());
double[][] userThresholds = Util.ArrayInit(numUsers, u => Util.ArrayInit(numLevels, l => userThresholdsPrior[u][l].Sample()));
// Repeat the model with fixed parameters
HashSet <int> visited = new HashSet <int>();
for (int observation = 0; observation < numObservations; observation++)
{
int user = Rand.Int(numUsers);
int item = Rand.Int(numItems);
int userItemPairID = user * numItems + item; // pair encoding
if (visited.Contains(userItemPairID)) // duplicate generated
{
observation--; // reject pair
continue;
}
visited.Add(userItemPairID);
double[] products = Util.ArrayInit(numTraits, t => userTraits[user][t] * itemTraits[item][t]);
double bias = userBias[user] + itemBias[item];
double affinity = bias + products.Sum();
double noisyAffinity = new Gaussian(affinity, affinityNoiseVariance).Sample();
double[] noisyThresholds = Util.ArrayInit(numLevels, l => new Gaussian(userThresholds[user][l], thresholdsNoiseVariance).Sample());
generatedUserData[observation] = user;
generatedItemData[observation] = item;
generatedRatingData[observation] = Util.ArrayInit(numLevels, l => noisyAffinity > noisyThresholds[l]);
}
userData.ObservedValue = generatedUserData;
itemData.ObservedValue = generatedItemData;
ratingData.ObservedValue = generatedRatingData;
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="LogisticOp"]/message_doc[@name="LogEvidenceRatio(double, Gaussian)"]/*'/>
public static double LogEvidenceRatio(double logistic, Gaussian x)
{
return(LogAverageFactor(logistic, x));
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="LogisticOp"]/message_doc[@name="XAverageLogarithm(Beta, Gaussian, Gaussian)"]/*'/>
public static Gaussian XAverageLogarithm([SkipIfUniform] Beta logistic, [Proper, SkipIfUniform] Gaussian x, Gaussian to_X)
{
if (logistic.IsPointMass)
{
return(XAverageLogarithm(logistic.Point));
}
// f(x) = sigma(x)^(a-1) sigma(-x)^(b-1)
// = sigma(x)^(a+b-2) exp(-x(b-1))
// since sigma(-x) = sigma(x) exp(-x)
double a = logistic.TrueCount;
double b = logistic.FalseCount;
double scale = a + b - 2;
if (scale == 0.0)
{
return(Gaussian.Uniform());
}
double shift = -(b - 1);
Gaussian toLogOddsPrev = Gaussian.FromNatural((to_X.MeanTimesPrecision - shift) / scale, to_X.Precision / scale);
Gaussian toLogOdds = BernoulliFromLogOddsOp.LogOddsAverageLogarithm(true, x, toLogOddsPrev);
return(Gaussian.FromNatural(scale * toLogOdds.MeanTimesPrecision + shift, scale * toLogOdds.Precision));
}
public void Run()
{
InferenceEngine engine = new InferenceEngine();
if (!(engine.Algorithm is Algorithms.ExpectationPropagation))
{
Console.WriteLine("This example only runs with Expectation Propagation");
return;
}
// The data
Vector[] inputs = new Vector[]
{
Vector.FromArray(new double[2] {
0, 0
}),
Vector.FromArray(new double[2] {
0, 1
}),
Vector.FromArray(new double[2] {
1, 0
}),
Vector.FromArray(new double[2] {
0, 0.5
}),
Vector.FromArray(new double[2] {
1.5, 0
}),
Vector.FromArray(new double[2] {
0.5, 1.0
})
};
bool[] outputs = { true, true, false, true, false, false };
// Open an evidence block to allow model scoring
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
// Set up the GP prior, a distribution over functions, which will be filled in later
Variable <SparseGP> prior = Variable.New <SparseGP>().Named("prior");
// The sparse GP variable - a random function
Variable <IFunction> f = Variable <IFunction> .Random(prior).Named("f");
// The locations to evaluate the function
VariableArray <Vector> x = Variable.Observed(inputs).Named("x");
Range j = x.Range.Named("j");
// The observation model
VariableArray <bool> y = Variable.Observed(outputs, j).Named("y");
Variable <double> score = Variable.FunctionEvaluate(f, x[j]).Named("score");
y[j] = (Variable.GaussianFromMeanAndVariance(score, 0.1) > 0);
// Close the evidence block
block.CloseBlock();
// The basis
Vector[] basis = new Vector[]
{
Vector.FromArray(new double[2] {
0.2, 0.2
}),
Vector.FromArray(new double[2] {
0.2, 0.8
}),
Vector.FromArray(new double[2] {
0.8, 0.2
}),
Vector.FromArray(new double[2] {
0.8, 0.8
})
};
for (int trial = 0; trial < 3; trial++)
{
// The kernel
IKernelFunction kf;
if (trial == 0)
{
kf = new SquaredExponential(-0.0);
}
else if (trial == 1)
{
kf = new SquaredExponential(-0.5);
}
else
{
kf = new NNKernel(new double[] { 0.0, 0.0 }, -1.0);
}
// Fill in the sparse GP prior
GaussianProcess gp = new GaussianProcess(new ConstantFunction(0), kf);
prior.ObservedValue = new SparseGP(new SparseGPFixed(gp, basis));
// Model score
double NNscore = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("{0} evidence = {1}", kf, NNscore.ToString("g4"));
}
// Infer the posterior Sparse GP
SparseGP sgp = engine.Infer <SparseGP>(f);
// Check that training set is classified correctly
Console.WriteLine("");
Console.WriteLine("Predictions on training set:");
for (int i = 0; i < outputs.Length; i++)
{
Gaussian post = sgp.Marginal(inputs[i]);
double postMean = post.GetMean();
string comment = (outputs[i] == (postMean > 0.0)) ? "correct" : "incorrect";
Console.WriteLine("f({0}) = {1} ({2})", inputs[i], post, comment);
}
}
public static Gaussian XInit()
{
return(Gaussian.Uniform());
}
public KalmanFilter1D(double mu, double sigma, double measurement_sigma, double motion_sigma)
{
mMotionDistribution = new Gaussian(0, motion_sigma);
mMeasurementDistribution = new Gaussian(0, measurement_sigma);
mBeliefDistribution = new Gaussian(mu, sigma);
}
// construct from Infer.NET Gaussian object
public static DNormal FromGaussian(Gaussian g)
{
return(new DNormal(g.GetMean(), g.GetVariance()));
}
/// <summary>
/// VMP message to 'Sum'
/// </summary>
/// <param name="A">Constant value for 'A'.</param>
/// <param name="B">Incoming message from 'B'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="MeanOfB">Buffer 'MeanOfB'.</param>
/// <param name="CovarianceOfB">Buffer 'CovarianceOfB'.</param>
/// <returns>The outgoing VMP message to the 'Sum' argument</returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'Sum' as the random arguments are varied.
/// The formula is <c>proj[sum_(B) p(B) factor(Sum,A,B)]</c>.
/// </para><para>
/// Uses John Winn's rule for deterministic factors.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="B"/> is not a proper distribution</exception>
public static Gaussian SumAverageLogarithm(bool[] A, [SkipIfUniform] VectorGaussian B, Vector MeanOfB, PositiveDefiniteMatrix CovarianceOfB)
{
Gaussian result = new Gaussian();
// p(x|a,b) = N(E[a]'*E[b], E[b]'*var(a)*E[b] + E[a]'*var(b)*E[a] + trace(var(a)*var(b)))
Vector ma = Vector.FromArray(A.Select(x => x?1.0:0.0).ToArray());
// Uses John Winn's rule for deterministic factors.
// Strict variational inference would set the variance to 0.
result.SetMeanAndVariance(ma.Inner(MeanOfB), CovarianceOfB.QuadraticForm(ma));
return result;
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="log">Incoming message from 'log'.</param>
/// <param name="d">Incoming message from 'd'.</param>
/// <param name="to_log">Outgoing message to 'log'.</param>
/// <returns>Logarithm of the factor's contribution the EP model evidence</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(d,log) p(d,log) factor(log,d) / sum_log p(log) messageTo(log))</c>.
/// Adding up these values across all factors and variables gives the log-evidence estimate for EP.
/// </para></remarks>
public static double LogEvidenceRatio(Gaussian log, Gamma d, [Fresh] Gaussian to_log)
{
return LogAverageFactor(log, d, to_log) - to_log.GetAverageLog(log);
}
public static Gaussian UsesAverageLogarithm(NonconjugateGaussian[] Uses, Gaussian Def, Gaussian result)
{
NonconjugateGaussian prod = Uses[0];
for (int i = 1; i < Uses.Length; i++)
{
prod.SetToProduct(prod, Uses[i]);
}
result = prod.GetGaussian(true);
result.SetToProduct(result, Def);
return(result);
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="sum">Incoming message from 'Sum'.</param>
/// <param name="to_sum">Outgoing message to 'sum'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(Sum) p(Sum) factor(Sum,A,B))</c>.
/// </para></remarks>
public static double LogAverageFactor(Gaussian sum, [Fresh] Gaussian to_sum)
{
return to_sum.GetLogAverageOf(sum);
}
public void Predict(double motion)
{
mMotionDistribution.Mean = motion;
mBeliefDistribution = KalmanFilter1DHelper.Predict(mBeliefDistribution, mMotionDistribution);
}
public Gaussian InferTomorrowsTime() // Prediction mode
{
TomorrowsTimeDist = InferenceEngine.Infer <Gaussian>(TomorrowsTime);
return(TomorrowsTimeDist);
}
public void Run()
{
// This example requires EP
InferenceEngine engine = new InferenceEngine();
if (!(engine.Algorithm is Algorithms.ExpectationPropagation))
{
Console.WriteLine("This example only runs with Expectation Propagation");
return;
}
// Define counts
int numUsers = 50;
int numItems = 10;
int numTraits = 2;
Variable <int> numObservations = Variable.Observed(100).Named("numObservations");
int numLevels = 2;
// Define ranges
Range user = new Range(numUsers).Named("user");
Range item = new Range(numItems).Named("item");
Range trait = new Range(numTraits).Named("trait");
Range observation = new Range(numObservations).Named("observation");
Range level = new Range(numLevels).Named("level");
// Define latent variables
var userTraits = Variable.Array(Variable.Array <double>(trait), user).Named("userTraits");
var itemTraits = Variable.Array(Variable.Array <double>(trait), item).Named("itemTraits");
var userBias = Variable.Array <double>(user).Named("userBias");
var itemBias = Variable.Array <double>(item).Named("itemBias");
var userThresholds = Variable.Array(Variable.Array <double>(level), user).Named("userThresholds");
// Define priors
var userTraitsPrior = Variable.Array(Variable.Array <Gaussian>(trait), user).Named("userTraitsPrior");
var itemTraitsPrior = Variable.Array(Variable.Array <Gaussian>(trait), item).Named("itemTraitsPrior");
var userBiasPrior = Variable.Array <Gaussian>(user).Named("userBiasPrior");
var itemBiasPrior = Variable.Array <Gaussian>(item).Named("itemBiasPrior");
var userThresholdsPrior = Variable.Array(Variable.Array <Gaussian>(level), user).Named("userThresholdsPrior");
// Define latent variables statistically
userTraits[user][trait] = Variable <double> .Random(userTraitsPrior[user][trait]);
itemTraits[item][trait] = Variable <double> .Random(itemTraitsPrior[item][trait]);
userBias[user] = Variable <double> .Random(userBiasPrior[user]);
itemBias[item] = Variable <double> .Random(itemBiasPrior[item]);
userThresholds[user][level] = Variable <double> .Random(userThresholdsPrior[user][level]);
// Initialise priors
Gaussian traitPrior = Gaussian.FromMeanAndVariance(0.0, 1.0);
Gaussian biasPrior = Gaussian.FromMeanAndVariance(0.0, 1.0);
userTraitsPrior.ObservedValue = Util.ArrayInit(numUsers, u => Util.ArrayInit(numTraits, t => traitPrior));
itemTraitsPrior.ObservedValue = Util.ArrayInit(numItems, i => Util.ArrayInit(numTraits, t => traitPrior));
userBiasPrior.ObservedValue = Util.ArrayInit(numUsers, u => biasPrior);
itemBiasPrior.ObservedValue = Util.ArrayInit(numItems, i => biasPrior);
userThresholdsPrior.ObservedValue = Util.ArrayInit(numUsers, u => Util.ArrayInit(numLevels, l => Gaussian.FromMeanAndVariance(l - numLevels / 2.0 + 0.5, 1.0)));
// Break symmetry and remove ambiguity in the traits
for (int i = 0; i < numTraits; i++)
{
// Assume that numTraits < numItems
for (int j = 0; j < numTraits; j++)
{
itemTraitsPrior.ObservedValue[i][j] = Gaussian.PointMass(0);
}
itemTraitsPrior.ObservedValue[i][i] = Gaussian.PointMass(1);
}
// Declare training data variables
var userData = Variable.Array <int>(observation).Named("userData");
var itemData = Variable.Array <int>(observation).Named("itemData");
var ratingData = Variable.Array(Variable.Array <bool>(level), observation).Named("ratingData");
// Set model noises explicitly
Variable <double> affinityNoiseVariance = Variable.Observed(0.1).Named("affinityNoiseVariance");
Variable <double> thresholdsNoiseVariance = Variable.Observed(0.1).Named("thresholdsNoiseVariance");
// Model
using (Variable.ForEach(observation))
{
VariableArray <double> products = Variable.Array <double>(trait).Named("products");
products[trait] = userTraits[userData[observation]][trait] * itemTraits[itemData[observation]][trait];
Variable <double> bias = (userBias[userData[observation]] + itemBias[itemData[observation]]).Named("bias");
Variable <double> affinity = (bias + Variable.Sum(products).Named("productSum")).Named("affinity");
Variable <double> noisyAffinity = Variable.GaussianFromMeanAndVariance(affinity, affinityNoiseVariance).Named("noisyAffinity");
VariableArray <double> noisyThresholds = Variable.Array <double>(level).Named("noisyThresholds");
noisyThresholds[level] = Variable.GaussianFromMeanAndVariance(userThresholds[userData[observation]][level], thresholdsNoiseVariance);
ratingData[observation][level] = noisyAffinity > noisyThresholds[level];
}
// Observe training data
GenerateData(
numUsers,
numItems,
numTraits,
numObservations.ObservedValue,
numLevels,
userData,
itemData,
ratingData,
userTraitsPrior.ObservedValue,
itemTraitsPrior.ObservedValue,
userBiasPrior.ObservedValue,
itemBiasPrior.ObservedValue,
userThresholdsPrior.ObservedValue,
affinityNoiseVariance.ObservedValue,
thresholdsNoiseVariance.ObservedValue);
// Allow EP to process the product factor as if running VMP
// as in Stern, Herbrich, Graepel paper.
engine.Compiler.GivePriorityTo(typeof(GaussianProductOp_SHG09));
engine.Compiler.ShowWarnings = true;
// Run inference
var userTraitsPosterior = engine.Infer <Gaussian[][]>(userTraits);
var itemTraitsPosterior = engine.Infer <Gaussian[][]>(itemTraits);
var userBiasPosterior = engine.Infer <Gaussian[]>(userBias);
var itemBiasPosterior = engine.Infer <Gaussian[]>(itemBias);
var userThresholdsPosterior = engine.Infer <Gaussian[][]>(userThresholds);
// Feed in the inferred posteriors as the new priors
userTraitsPrior.ObservedValue = userTraitsPosterior;
itemTraitsPrior.ObservedValue = itemTraitsPosterior;
userBiasPrior.ObservedValue = userBiasPosterior;
itemBiasPrior.ObservedValue = itemBiasPosterior;
userThresholdsPrior.ObservedValue = userThresholdsPosterior;
// Make a prediction
numObservations.ObservedValue = 1;
userData.ObservedValue = new int[] { 5 };
itemData.ObservedValue = new int[] { 6 };
ratingData.ClearObservedValue();
Bernoulli[] predictedRating = engine.Infer <Bernoulli[][]>(ratingData)[0];
Console.WriteLine("Predicted rating:");
foreach (var rating in predictedRating)
{
Console.WriteLine(rating);
}
}
/// <summary>Computations that depend on the observed value of vVector__27</summary>
private void Changed_vVector__27()
{
if (this.Changed_vVector__27_iterationsDone == 1)
{
return;
}
this.vVector__27_marginal = new PointMass <Vector[]>(this.VVector__27);
// The constant 'vVectorGaussian27'
VectorGaussian vVectorGaussian27 = VectorGaussian.FromNatural(DenseVector.FromArray(new double[3] {
1547829870.0, 525077980.0, 200270.0
}), new PositiveDefiniteMatrix(new double[3, 3] {
{ 4254590363351.0, 1127383488860.0, 433199230.0 }, { 1127383488860.0, 482723521821.0, 146764360.0 }, { 433199230.0, 146764360.0, 56221.0 }
}));
this.vVector81_marginal_F = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian27);
// Buffer for ReplicateOp_Divide.Marginal<VectorGaussian>
VectorGaussian vVector81_rep_B_toDef = default(VectorGaussian);
// Message to 'vVector81_rep' from Replicate factor
vVector81_rep_B_toDef = ReplicateOp_Divide.ToDefInit <VectorGaussian>(vVectorGaussian27);
// Message to 'vVector81_marginal' from Variable factor
this.vVector81_marginal_F = VariableOp.MarginalAverageConditional <VectorGaussian>(vVector81_rep_B_toDef, vVectorGaussian27, this.vVector81_marginal_F);
DistributionStructArray <Gaussian, double> vdouble__81_F = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__81' Forwards messages.
vdouble__81_F = new DistributionStructArray <Gaussian, double>(1);
for (int index27 = 0; index27 < 1; index27++)
{
vdouble__81_F[index27] = Gaussian.Uniform();
}
DistributionStructArray <Gaussian, double> vdouble__82_F = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__82' Forwards messages.
vdouble__82_F = new DistributionStructArray <Gaussian, double>(1);
for (int index27 = 0; index27 < 1; index27++)
{
vdouble__82_F[index27] = Gaussian.Uniform();
}
DistributionRefArray <VectorGaussian, Vector> vVector81_rep_F = default(DistributionRefArray <VectorGaussian, Vector>);
DistributionRefArray <VectorGaussian, Vector> vVector81_rep_B = default(DistributionRefArray <VectorGaussian, Vector>);
// Create array for 'vVector81_rep' Forwards messages.
vVector81_rep_F = new DistributionRefArray <VectorGaussian, Vector>(1);
// Create array for 'vVector81_rep' Backwards messages.
vVector81_rep_B = new DistributionRefArray <VectorGaussian, Vector>(1);
for (int index27 = 0; index27 < 1; index27++)
{
vVector81_rep_B[index27] = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian27);
vVector81_rep_F[index27] = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian27);
}
// Buffer for ReplicateOp_Divide.UsesAverageConditional<VectorGaussian>
VectorGaussian vVector81_rep_F_marginal = default(VectorGaussian);
// Message to 'vVector81_rep' from Replicate factor
vVector81_rep_F_marginal = ReplicateOp_Divide.MarginalInit <VectorGaussian>(vVectorGaussian27);
// Message to 'vVector81_rep' from Replicate factor
vVector81_rep_F_marginal = ReplicateOp_Divide.Marginal <VectorGaussian>(vVector81_rep_B_toDef, vVectorGaussian27, vVector81_rep_F_marginal);
// Buffer for InnerProductOp.InnerProductAverageConditional
// Create array for replicates of 'vVector81_rep_F_index27__AMean'
Vector[] vVector81_rep_F_index27__AMean = new Vector[1];
for (int index27 = 0; index27 < 1; index27++)
{
// Message to 'vdouble__82' from InnerProduct factor
vVector81_rep_F_index27__AMean[index27] = InnerProductOp.AMeanInit(vVector81_rep_F[index27]);
}
// Buffer for InnerProductOp.AMean
// Create array for replicates of 'vVector81_rep_F_index27__AVariance'
PositiveDefiniteMatrix[] vVector81_rep_F_index27__AVariance = new PositiveDefiniteMatrix[1];
for (int index27 = 0; index27 < 1; index27++)
{
// Message to 'vdouble__82' from InnerProduct factor
vVector81_rep_F_index27__AVariance[index27] = InnerProductOp.AVarianceInit(vVector81_rep_F[index27]);
// Message to 'vVector81_rep' from Replicate factor
vVector81_rep_F[index27] = ReplicateOp_Divide.UsesAverageConditional <VectorGaussian>(vVector81_rep_B[index27], vVector81_rep_F_marginal, index27, vVector81_rep_F[index27]);
}
// Create array for 'vdouble__82_marginal' Forwards messages.
this.vdouble__82_marginal_F = new DistributionStructArray <Gaussian, double>(1);
for (int index27 = 0; index27 < 1; index27++)
{
this.vdouble__82_marginal_F[index27] = Gaussian.Uniform();
}
// Message from use of 'vdouble__82'
DistributionStructArray <Gaussian, double> vdouble__82_use_B = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__82_use' Backwards messages.
vdouble__82_use_B = new DistributionStructArray <Gaussian, double>(1);
for (int index27 = 0; index27 < 1; index27++)
{
vdouble__82_use_B[index27] = Gaussian.Uniform();
// Message to 'vdouble__82' from InnerProduct factor
vVector81_rep_F_index27__AVariance[index27] = InnerProductOp.AVariance(vVector81_rep_F[index27], vVector81_rep_F_index27__AVariance[index27]);
// Message to 'vdouble__82' from InnerProduct factor
vVector81_rep_F_index27__AMean[index27] = InnerProductOp.AMean(vVector81_rep_F[index27], vVector81_rep_F_index27__AVariance[index27], vVector81_rep_F_index27__AMean[index27]);
// Message to 'vdouble__82' from InnerProduct factor
vdouble__82_F[index27] = InnerProductOp.InnerProductAverageConditional(vVector81_rep_F_index27__AMean[index27], vVector81_rep_F_index27__AVariance[index27], this.VVector__27[index27]);
// Message to 'vdouble__82_marginal' from DerivedVariable factor
this.vdouble__82_marginal_F[index27] = DerivedVariableOp.MarginalAverageConditional <Gaussian>(vdouble__82_use_B[index27], vdouble__82_F[index27], this.vdouble__82_marginal_F[index27]);
}
// Create array for 'vdouble__81_marginal' Forwards messages.
this.vdouble__81_marginal_F = new DistributionStructArray <Gaussian, double>(1);
for (int index27 = 0; index27 < 1; index27++)
{
this.vdouble__81_marginal_F[index27] = Gaussian.Uniform();
}
// Message from use of 'vdouble__81'
DistributionStructArray <Gaussian, double> vdouble__81_use_B = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__81_use' Backwards messages.
vdouble__81_use_B = new DistributionStructArray <Gaussian, double>(1);
for (int index27 = 0; index27 < 1; index27++)
{
vdouble__81_use_B[index27] = Gaussian.Uniform();
// Message to 'vdouble__81' from GaussianFromMeanAndVariance factor
vdouble__81_F[index27] = GaussianFromMeanAndVarianceOp.SampleAverageConditional(vdouble__82_F[index27], 0.1);
// Message to 'vdouble__81_marginal' from Variable factor
this.vdouble__81_marginal_F[index27] = VariableOp.MarginalAverageConditional <Gaussian>(vdouble__81_use_B[index27], vdouble__81_F[index27], this.vdouble__81_marginal_F[index27]);
}
this.Changed_vVector__27_iterationsDone = 1;
}
/// <summary>
/// EP message to 'log'
/// </summary>
/// <param name="log">Incoming message from 'log'.</param>
/// <param name="d">Incoming message from 'd'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'log' as the random arguments are varied.
/// The formula is <c>proj[p(log) sum_(d) p(d) factor(log,d)]/p(log)</c>.
/// </para></remarks>
public static Gaussian LogAverageConditional(Gaussian log, Gamma d, Gaussian result)
{
var g = Gamma.FromShapeAndRate(d.Shape + 1, d.Rate);
return ExpOp.DAverageConditional(g, log, result);
}
public static double LogEvidenceRatio(Beta logistic, Gaussian x, Gaussian falseMsg, [Fresh] Beta to_logistic)
{
// always zero when using the stabilized message from LogisticAverageConditional
return(0.0);
//return LogAverageFactor(logistic, x, falseMsg) - to_logistic.GetLogAverageOf(logistic);
}
/// <summary>
/// VMP message to 'log'
/// </summary>
/// <param name="log">Incoming message from 'log'.</param>
/// <param name="d">Incoming message from 'd'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'log' as the random arguments are varied.
/// The formula is <c>proj[sum_(d) p(d) factor(log,d)]</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="d"/> is not a proper distribution</exception>
public static Gaussian LogAverageLogarithm(Gaussian log, [SkipIfUniform] Gamma d, Gaussian result)
{
if (d.IsPointMass)
return LogAverageLogarithm(log, d.Point, result);
result.SetMeanAndVariance(d.GetMeanLog(), MMath.Trigamma(d.Shape));
return result;
}
public static DNormal PointMass(double mean)
{
// Construct a Gaussian with very small width.
return(DNormal.FromGaussian(Gaussian.PointMass(mean)));
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="log">Incoming message from 'log'.</param>
/// <param name="d">Constant value for 'd'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(log) p(log) factor(log,d))</c>.
/// </para></remarks>
public static double LogAverageFactor(Gaussian log, double d)
{
return log.GetLogProb(Math.Log(d));
}
public static BasisFunctionBase BasisFunctionSelector(BasisFunctionsEnum functionChoise)
{
//default
BasisFunctionBase solver = null;
switch (functionChoise)
{
case BasisFunctionsEnum.AsymmetricGaussian:
solver = new AsymmetricGaussian();
solver.Coefficients = new double[4];
solver.Coefficients[0] = .5;
solver.Coefficients[1] = .5;
solver.Coefficients[2] = .3;
solver.Coefficients[3] = 1;
break;
case BasisFunctionsEnum.Linear:
solver = new Linear();
solver.Coefficients = new double[2];
solver.Coefficients[0] = 1; // m
solver.Coefficients[1] = 0; // b
break;
case BasisFunctionsEnum.PolynomialQuadratic:
{
solver = new Quadratic();
solver.Coefficients = new double[3];
solver.Coefficients[0] = 1; //ax^2
solver.Coefficients[1] = 1; //bx
solver.Coefficients[2] = 1; //c
}
break;
case BasisFunctionsEnum.PolynomialCubic:
{
solver = new Cubic();
solver.Coefficients = new double[7];
solver.Coefficients[0] = 1; //ax^3
solver.Coefficients[1] = 1; //bx^2
solver.Coefficients[2] = 1; //cx
solver.Coefficients[3] = 1; //d
solver.Coefficients[4] = 1; //d
solver.Coefficients[5] = 1; //d
solver.Coefficients[6] = 1; //d
}
break;
case BasisFunctionsEnum.Lorentzian:
{
solver = new Lorentian();
solver.Coefficients = new double[3];
solver.Coefficients[0] = 6; //width
solver.Coefficients[1] = 50; //height
solver.Coefficients[2] = -1; //xoffset
}
break;
case BasisFunctionsEnum.Gaussian:
{
solver = new Gaussian();
solver.Coefficients = new double[3];
solver.Coefficients[0] = 6; //sigma
solver.Coefficients[1] = 50; //height
solver.Coefficients[2] = -1; //xoffset
}
break;
case BasisFunctionsEnum.Chebyshev:
{
solver = new Chebyshev();
solver.Coefficients = new double[6];
solver.Coefficients[0] = 0; //?
solver.Coefficients[1] = 1; //?
solver.Coefficients[2] = 1; //?
solver.Coefficients[3] = 0; //?
solver.Coefficients[4] = 0; //?
solver.Coefficients[5] = 0; //?
}
break;
case BasisFunctionsEnum.Orbitrap:
{
solver = new OrbitrapFunction();
solver.Coefficients = new double[3];
solver.Coefficients[0] = 1; //?
solver.Coefficients[1] = 1; //?
solver.Coefficients[2] = 1; //?
//quadSolver.Coefficients[3] = 1;//?
//quadSolver.Coefficients[4] = 1;//?
//quadSolver.Coefficients[4] = 1;//?
}
break;
case BasisFunctionsEnum.Hanning:
{
solver = new Hanning();
solver.Coefficients = new double[3];
solver.Coefficients[0] = 1; //?
solver.Coefficients[1] = 1; //?
solver.Coefficients[2] = 1; //?
}
break;
}
return(solver);
}
public void Update(double measurement)
{
mMeasurementDistribution.Mean = measurement;
mBeliefDistribution = KalmanFilter1DHelper.Update(mBeliefDistribution, mMeasurementDistribution);
}
#pragma warning disable 429
#endif
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="LogisticOp_SJ99"]/message_doc[@name="XAverageLogarithm(Beta, Gaussian, Gaussian, double)"]/*'/>
public static Gaussian XAverageLogarithm([SkipIfUniform] Beta logistic, Gaussian x, Gaussian to_x, double a, IPolyrand random = null)
{
if (logistic.IsPointMass)
{
return(LogisticOp.XAverageLogarithm(logistic.Point));
}
// f(x) = sigma(x)^(a-1) sigma(-x)^(b-1)
// = sigma(x)^(a+b-2) exp(-x(b-1))
// since sigma(-x) = sigma(x) exp(-x)
double scale = logistic.TrueCount + logistic.FalseCount - 2;
if (scale == 0.0)
{
return(Gaussian.Uniform());
}
double shift = -(logistic.FalseCount - 1);
double m, v;
x.GetMeanAndVariance(out m, out v);
double sa;
if (double.IsPositiveInfinity(v))
{
a = 0.5;
sa = MMath.Logistic(m);
}
else
{
sa = MMath.Logistic(m + (1 - 2 * a) * v * 0.5);
}
double precision = a * a + (1 - 2 * a) * sa;
// meanTimesPrecision = m*a*a + 1-2*a*sa;
double meanTimesPrecision = m * precision + 1 - sa;
//double vf = 1/(a*a + (1-2*a)*sa);
//double mf = m + vf*(true ? 1-sa : sa);
//double precision = 1/vf;
//double meanTimesPrecision = mf*precision;
Gaussian result = Gaussian.FromNatural(scale * meanTimesPrecision + shift, scale * precision);
double step
= (LogisticOp_SJ99.global_step == 0.0)
? 1.0
: (Rand.Double(random) * LogisticOp_SJ99.global_step);
// random damping helps convergence, especially with parallel updates
if (false && !x.IsPointMass)
{
// if the update would change the sign of 1-2*sa, send a message to make sa=0.5
double newPrec = x.Precision - to_x.Precision + result.Precision;
double newv = 1 / newPrec;
double newm = newv * (x.MeanTimesPrecision - to_x.MeanTimesPrecision + result.MeanTimesPrecision);
double newarg = newm + (1 - 2 * a) * newv * 0.5;
if ((sa < 0.5 && newarg > 0) || (sa > 0.5 && newarg < 0))
{
// send a message to make newarg=0
// it is sufficient to make (x.MeanTimesPrecision + step*(result.MeanTimesPrecision - to_x.MeanTimesPrecision) + 0.5-a) = 0
double mpOffset = x.MeanTimesPrecision + 0.5 - a;
double precOffset = x.Precision;
double mpScale = result.MeanTimesPrecision - to_x.MeanTimesPrecision;
double precScale = result.Precision - to_x.Precision;
double arg = m + (1 - 2 * a) * v * 0.5;
//arg = 0;
step = (arg * precOffset - mpOffset) / (mpScale - arg * precScale);
//step = (a-0.5-x.MeanTimesPrecision)/(result.MeanTimesPrecision - to_x.MeanTimesPrecision);
//Console.WriteLine(step);
}
}
if (step != 1.0)
{
result.Precision = step * result.Precision + (1 - step) * to_x.Precision;
result.MeanTimesPrecision = step * result.MeanTimesPrecision + (1 - step) * to_x.MeanTimesPrecision;
}
return(result);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="RatioGaussianOp"]/message_doc[@name="QuantileAverageConditional(CanGetQuantile{double}, Gaussian)"]/*'/>
public static Gaussian QuantileAverageConditional(CanGetQuantile <double> canGetQuantile, Gaussian probability)
{
if (probability.IsPointMass)
{
return(Gaussian.PointMass(canGetQuantile.GetQuantile(probability.Point)));
}
else
{
throw new NotSupportedException();
}
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="LogisticOp"]/message_doc[@name="LogAverageFactor(Beta, Gaussian, Gaussian)"]/*'/>
public static double LogAverageFactor(Beta logistic, Gaussian x, Gaussian falseMsg)
{
// return log(int_y int_x delta(y - Logistic(x)) Beta(y) Gaussian(x) dx dy)
double m, v;
x.GetMeanAndVariance(out m, out v);
if (logistic.TrueCount == 2 && logistic.FalseCount == 1)
{
// shortcut for common case
return(Math.Log(2 * MMath.LogisticGaussian(m, v)));
}
else if (logistic.TrueCount == 1 && logistic.FalseCount == 2)
{
return(Math.Log(2 * MMath.LogisticGaussian(-m, v)));
}
else
{
// logistic(sigma(x)) N(x;m,v)
// = sigma(x)^(a-1) sigma(-x)^(b-1) N(x;m,v) gamma(a+b)/gamma(a)/gamma(b)
// = e^((a-1)x) sigma(-x)^(a+b-2) N(x;m,v)
// = sigma(-x)^(a+b-2) N(x;m+(a-1)v,v) exp((a-1)m + (a-1)^2 v/2)
// int_x logistic(sigma(x)) N(x;m,v) dx
// =approx (int_x sigma(-x)/falseMsg(x) falseMsg(x)^(a+b-2) N(x;m+(a-1)v,v))^(a+b-2)
// * (int_x falseMsg(x)^(a+b-2) N(x;m+(a-1)v,v))^(1 - (a+b-2))
// * exp((a-1)m + (a-1)^2 v/2) gamma(a+b)/gamma(a)/gamma(b)
// This formula comes from (66) in Minka (2005)
// Alternatively,
// =approx (int_x falseMsg(x)/sigma(-x) falseMsg(x)^(a+b-2) N(x;m+(a-1)v,v))^(-(a+b-2))
// * (int_x falseMsg(x)^(a+b-2) N(x;m+(a-1)v,v))^(1 + (a+b-2))
// * exp((a-1)m + (a-1)^2 v/2) gamma(a+b)/gamma(a)/gamma(b)
double tc1 = logistic.TrueCount - 1;
double fc1 = logistic.FalseCount - 1;
Gaussian prior = new Gaussian(m + tc1 * v, v);
if (tc1 + fc1 < 0)
{
// numerator2 = int_x falseMsg(x)^(a+b-1) N(x;m+(a-1)v,v) dx
double numerator2 = prior.GetLogAverageOfPower(falseMsg, tc1 + fc1 + 1);
Gaussian prior2 = prior * (falseMsg ^ (tc1 + fc1 + 1));
double mp, vp;
prior2.GetMeanAndVariance(out mp, out vp);
// numerator = int_x (1+exp(x)) falseMsg(x)^(a+b-1) N(x;m+(a-1)v,v) dx / int_x falseMsg(x)^(a+b-1) N(x;m+(a-1)v,v) dx
double numerator = Math.Log(1 + Math.Exp(mp + 0.5 * vp));
// denominator = int_x falseMsg(x)^(a+b-2) N(x;m+(a-1)v,v) dx
double denominator = prior.GetLogAverageOfPower(falseMsg, tc1 + fc1);
return(-(tc1 + fc1) * (numerator + numerator2 - denominator) + denominator + (tc1 * m + tc1 * tc1 * v * 0.5) - logistic.GetLogNormalizer());
}
else
{
// numerator2 = int_x falseMsg(x)^(a+b-3) N(x;m+(a-1)v,v) dx
double numerator2 = prior.GetLogAverageOfPower(falseMsg, tc1 + fc1 - 1);
Gaussian prior2 = prior * (falseMsg ^ (tc1 + fc1 - 1));
double mp, vp;
prior2.GetMeanAndVariance(out mp, out vp);
// numerator = int_x sigma(-x) falseMsg(x)^(a+b-3) N(x;m+(a-1)v,v) dx / int_x falseMsg(x)^(a+b-3) N(x;m+(a-1)v,v) dx
double numerator = Math.Log(MMath.LogisticGaussian(-mp, vp));
// denominator = int_x falseMsg(x)^(a+b-2) N(x;m+(a-1)v,v) dx
double denominator = prior.GetLogAverageOfPower(falseMsg, tc1 + fc1);
return((tc1 + fc1) * (numerator + numerator2 - denominator) + denominator + (tc1 * m + tc1 * tc1 * v * 0.5) - logistic.GetLogNormalizer());
}
}
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="RatioGaussianOp"]/message_doc[@name="ProbabilityAverageConditional(Gaussian, CanGetQuantile{double}, Gaussian)"]/*'/>
public static Gaussian ProbabilityAverageConditional(double quantile, CanGetQuantile <double> canGetQuantile)
{
CanGetProbLessThan <double> canGetProbLessThan = (CanGetProbLessThan <double>)canGetQuantile;
return(Gaussian.PointMass(canGetProbLessThan.GetProbLessThan(quantile)));
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="LogisticOp_JJ96"]/message_doc[@name="XAverageLogarithm(Beta, Gaussian, Gaussian)"]/*'/>
public static Gaussian XAverageLogarithm([SkipIfUniform] Beta logistic, [Proper, SkipIfUniform] Gaussian x, Gaussian result)
{
if (logistic.IsPointMass)
{
return(LogisticOp.XAverageLogarithm(logistic.Point));
}
// f(x) = sigma(x)^(a-1) sigma(-x)^(b-1)
// = sigma(x)^(a+b-2) exp(-x(b-1))
// since sigma(-x) = sigma(x) exp(-x)
double a = logistic.TrueCount;
double b = logistic.FalseCount;
double scale = a + b - 2;
if (scale == 0.0)
{
return(Gaussian.Uniform());
}
double shift = -(b - 1);
// sigma(x) >= sigma(t) exp((x-t)/2 - a/2*(x^2 - t^2))
double m, v;
x.GetMeanAndVariance(out m, out v);
double t = Math.Sqrt(m * m + v);
double lambda = (t == 0) ? 0.25 : Math.Tanh(t / 2) / (2 * t);
return(Gaussian.FromNatural(scale * 0.5 + shift, scale * lambda));
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="RatioGaussianOp"]/message_doc[@name="ProbabilityAverageConditional(Gaussian, CanGetQuantile{double}, Gaussian)"]/*'/>
public static Gaussian ProbabilityAverageConditional(Gaussian quantile, CanGetQuantile <double> canGetQuantile, [RequiredArgument] Gaussian probability)
{
if (quantile.IsPointMass)
{
return(ProbabilityAverageConditional(quantile.Point, canGetQuantile));
}
else if (probability.IsPointMass)
{
return(ProbabilityAverageConditional(quantile, canGetQuantile, probability.Point));
}
else
{
throw new NotSupportedException();
}
}
public static Gaussian MarginalAverageLogarithm(NonconjugateGaussian[] Uses, Gaussian Def, Gaussian result)
{
return(UsesAverageLogarithm(Uses, Def, result));
}
public void TestEvaluate()
{
double v = new Gaussian(10, 2).Evaluate(8);
Assert.AreEqual(0.121, v, 0.001);
}
/// <summary>
/// VMP message to 'Sum'
/// </summary>
/// <param name="A">Incoming message from 'A'.</param>
/// <param name="B">Constant value for 'B'.</param>
/// <returns>The outgoing VMP message to the 'Sum' argument</returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'Sum' as the random arguments are varied.
/// The formula is <c>proj[sum_(A) p(A) factor(Sum,A,B)]</c>.
/// </para></remarks>
public static Gaussian SumAverageLogarithm(DistributionStructArray<Bernoulli, bool> A, [SkipIfUniform] Vector B)
{
Gaussian result = new Gaussian();
// p(x|a,b) = N(E[a]'*E[b], E[b]'*var(a)*E[b] + E[a]'*var(b)*E[a] + trace(var(a)*var(b)))
Vector ma = Vector.Zero(A.Count);
Vector va = Vector.Zero(A.Count);
for (int i = 0; i < A.Count; i++) {
ma[i] = A[i].GetMean();
va[i] = A[i].GetVariance();
}
// Uses John Winn's rule for deterministic factors.
// Strict variational inference would set the variance to 0.
var MeanOfBSquared = Vector.Zero(B.Count);
MeanOfBSquared.SetToFunction(B, x => x * x);
result.SetMeanAndVariance(ma.Inner(B), va.Inner(MeanOfBSquared));
return result;
}
/// <summary>
/// Initializes a new instance of the <see cref="ItemParameterDistribution"/> class.
/// </summary>
/// <param name="traitDistribution">The distribution over the user traits.</param>
/// <param name="biasDistribution">The distribution over the user bias.</param>
public ItemParameterDistribution(GaussianArray traitDistribution, Gaussian biasDistribution)
: base(traitDistribution, biasDistribution)
{
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="sum">Incoming message from 'Sum'.</param>
/// <param name="A">Constant value for 'A'.</param>
/// <param name="B">Constant value for 'B'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(Sum) p(Sum) factor(Sum,A,B))</c>.
/// </para></remarks>
public static double LogAverageFactor(Gaussian sum, bool[] A, Vector B)
{
return sum.GetLogProb(Factor.SumWhere(A, B));
}
/// <summary>Computations that depend on the observed value of vVector__68 and vdouble__204</summary>
private void Changed_vVector__68_vdouble__204()
{
if (this.Changed_vVector__68_vdouble__204_iterationsDone == 1)
{
return;
}
this.vVector__68_marginal = new PointMass <Vector[]>(this.VVector__68);
this.vdouble__204_marginal = new DistributionStructArray <Gaussian, double>(5622, delegate(int index68) {
return(Gaussian.Uniform());
});
this.vdouble__204_marginal = Distribution.SetPoint <DistributionStructArray <Gaussian, double>, double[]>(this.vdouble__204_marginal, this.Vdouble__204);
// The constant 'vVectorGaussian68'
VectorGaussian vVectorGaussian68 = VectorGaussian.FromNatural(DenseVector.FromArray(new double[3] {
0.0, 0.0, 0.0
}), new PositiveDefiniteMatrix(new double[3, 3] {
{ 1.0, 0.0, 0.0 }, { 0.0, 1.0, 0.0 }, { 0.0, 0.0, 1.0 }
}));
this.vVector205_marginal_F = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian68);
// Message from use of 'vdouble__205'
DistributionStructArray <Gaussian, double> vdouble__205_use_B = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__205_use' Backwards messages.
vdouble__205_use_B = new DistributionStructArray <Gaussian, double>(5622);
for (int index68 = 0; index68 < 5622; index68++)
{
vdouble__205_use_B[index68] = Gaussian.Uniform();
// Message to 'vdouble__205_use' from GaussianFromMeanAndVariance factor
vdouble__205_use_B[index68] = GaussianFromMeanAndVarianceOp.MeanAverageConditional(this.Vdouble__204[index68], 0.1);
}
DistributionRefArray <VectorGaussian, Vector> vVector205_rep_B = default(DistributionRefArray <VectorGaussian, Vector>);
// Create array for 'vVector205_rep' Backwards messages.
vVector205_rep_B = new DistributionRefArray <VectorGaussian, Vector>(5622);
for (int index68 = 0; index68 < 5622; index68++)
{
vVector205_rep_B[index68] = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian68);
// Message to 'vVector205_rep' from InnerProduct factor
vVector205_rep_B[index68] = InnerProductOp.AAverageConditional(vdouble__205_use_B[index68], this.VVector__68[index68], vVector205_rep_B[index68]);
}
// Buffer for ReplicateOp_Divide.Marginal<VectorGaussian>
VectorGaussian vVector205_rep_B_toDef = default(VectorGaussian);
// Message to 'vVector205_rep' from Replicate factor
vVector205_rep_B_toDef = ReplicateOp_Divide.ToDefInit <VectorGaussian>(vVectorGaussian68);
// Message to 'vVector205_rep' from Replicate factor
vVector205_rep_B_toDef = ReplicateOp_Divide.ToDef <VectorGaussian>(vVector205_rep_B, vVector205_rep_B_toDef);
// Message to 'vVector205_marginal' from Variable factor
this.vVector205_marginal_F = VariableOp.MarginalAverageConditional <VectorGaussian>(vVector205_rep_B_toDef, vVectorGaussian68, this.vVector205_marginal_F);
DistributionStructArray <Gaussian, double> vdouble__205_F = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__205' Forwards messages.
vdouble__205_F = new DistributionStructArray <Gaussian, double>(5622);
for (int index68 = 0; index68 < 5622; index68++)
{
vdouble__205_F[index68] = Gaussian.Uniform();
}
DistributionRefArray <VectorGaussian, Vector> vVector205_rep_F = default(DistributionRefArray <VectorGaussian, Vector>);
// Create array for 'vVector205_rep' Forwards messages.
vVector205_rep_F = new DistributionRefArray <VectorGaussian, Vector>(5622);
for (int index68 = 0; index68 < 5622; index68++)
{
vVector205_rep_F[index68] = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian68);
}
// Buffer for ReplicateOp_Divide.UsesAverageConditional<VectorGaussian>
VectorGaussian vVector205_rep_F_marginal = default(VectorGaussian);
// Message to 'vVector205_rep' from Replicate factor
vVector205_rep_F_marginal = ReplicateOp_Divide.MarginalInit <VectorGaussian>(vVectorGaussian68);
// Message to 'vVector205_rep' from Replicate factor
vVector205_rep_F_marginal = ReplicateOp_Divide.Marginal <VectorGaussian>(vVector205_rep_B_toDef, vVectorGaussian68, vVector205_rep_F_marginal);
// Buffer for InnerProductOp.InnerProductAverageConditional
// Create array for replicates of 'vVector205_rep_F_index68__AMean'
Vector[] vVector205_rep_F_index68__AMean = new Vector[5622];
for (int index68 = 0; index68 < 5622; index68++)
{
// Message to 'vdouble__205' from InnerProduct factor
vVector205_rep_F_index68__AMean[index68] = InnerProductOp.AMeanInit(vVector205_rep_F[index68]);
}
// Buffer for InnerProductOp.AMean
// Create array for replicates of 'vVector205_rep_F_index68__AVariance'
PositiveDefiniteMatrix[] vVector205_rep_F_index68__AVariance = new PositiveDefiniteMatrix[5622];
for (int index68 = 0; index68 < 5622; index68++)
{
// Message to 'vdouble__205' from InnerProduct factor
vVector205_rep_F_index68__AVariance[index68] = InnerProductOp.AVarianceInit(vVector205_rep_F[index68]);
// Message to 'vVector205_rep' from Replicate factor
vVector205_rep_F[index68] = ReplicateOp_Divide.UsesAverageConditional <VectorGaussian>(vVector205_rep_B[index68], vVector205_rep_F_marginal, index68, vVector205_rep_F[index68]);
}
// Create array for 'vdouble__205_marginal' Forwards messages.
this.vdouble__205_marginal_F = new DistributionStructArray <Gaussian, double>(5622);
for (int index68 = 0; index68 < 5622; index68++)
{
this.vdouble__205_marginal_F[index68] = Gaussian.Uniform();
// Message to 'vdouble__205' from InnerProduct factor
vVector205_rep_F_index68__AVariance[index68] = InnerProductOp.AVariance(vVector205_rep_F[index68], vVector205_rep_F_index68__AVariance[index68]);
// Message to 'vdouble__205' from InnerProduct factor
vVector205_rep_F_index68__AMean[index68] = InnerProductOp.AMean(vVector205_rep_F[index68], vVector205_rep_F_index68__AVariance[index68], vVector205_rep_F_index68__AMean[index68]);
// Message to 'vdouble__205' from InnerProduct factor
vdouble__205_F[index68] = InnerProductOp.InnerProductAverageConditional(vVector205_rep_F_index68__AMean[index68], vVector205_rep_F_index68__AVariance[index68], this.VVector__68[index68]);
// Message to 'vdouble__205_marginal' from DerivedVariable factor
this.vdouble__205_marginal_F[index68] = DerivedVariableOp.MarginalAverageConditional <Gaussian>(vdouble__205_use_B[index68], vdouble__205_F[index68], this.vdouble__205_marginal_F[index68]);
}
this.Changed_vVector__68_vdouble__204_iterationsDone = 1;
}
/// <summary>
/// EP message to 'd'
/// </summary>
/// <param name="log">Incoming message from 'log'.</param>
/// <param name="d">Incoming message from 'd'.</param>
/// <param name="to_log">Previous outgoing message to 'log'.</param>
/// <returns>The outgoing EP message to the 'd' argument</returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'd' as the random arguments are varied.
/// The formula is <c>proj[p(d) sum_(log) p(log) factor(log,d)]/p(d)</c>.
/// </para></remarks>
public static Gamma DAverageConditional(Gaussian log, Gamma d, Gaussian to_log)
{
var g = Gamma.FromShapeAndRate(d.Shape + 1, d.Rate);
return ExpOp.ExpAverageConditional(g, log, to_log);
}
private void btnRunAnalysis_Click(object sender, EventArgs e)
{
if (dgvAnalysisSource.Rows.Count == 0)
{
MessageBox.Show("Please load the training data before clicking this button");
return;
}
lbStatus.Text = "Gathering data. This may take a while...";
Application.DoEvents();
// Extract inputs and outputs
int rows = dgvAnalysisSource.Rows.Count;
double[][] input = Jagged.Zeros(rows, 32 * 32);
int[] output = new int[rows];
for (int i = 0; i < rows; i++)
{
input.SetRow(i, (double[])dgvAnalysisSource.Rows[i].Cells["colTrainingFeatures"].Value);
output[i] = (int)dgvAnalysisSource.Rows[i].Cells["colTrainingLabel"].Value;
}
// Create the chosen Kernel with given parameters
IKernel kernel;
if (rbGaussian.Checked)
{
kernel = new Gaussian((double)numSigma.Value);
}
else
{
kernel = new Polynomial((int)numDegree.Value, (double)numConstant.Value);
}
// Create the Kernel Discriminant Analysis using the selected Kernel
kda = new KernelDiscriminantAnalysis(kernel)
{
Threshold = (double)numThreshold.Value,
Regularization = (double)numRegularization.Value
};
lbStatus.Text = "Computing the analysis. This may take a significant amount of time...";
Application.DoEvents();
// Compute the analysis.
kda.Learn(input, output);
// Show information about the analysis in the form
dgvPrincipalComponents.DataSource = kda.Discriminants;
dgvFeatureVectors.DataSource = new ArrayDataView(kda.DiscriminantVectors);
dgvClasses.DataSource = kda.Classes;
// Create the component graphs
distributionView.DataSource = kda.Discriminants;
cumulativeView.DataSource = kda.Discriminants;
lbStatus.Text = "Analysis complete. Click Classify to test the analysis.";
btnClassify.Enabled = true;
}
/// <summary>
/// VMP message to 'log'
/// </summary>
/// <param name="log">Incoming message from 'log'.</param>
/// <param name="d">Constant value for 'd'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'log' conditioned on the given values.
/// </para></remarks>
public static Gaussian LogAverageLogarithm(Gaussian log, double d, Gaussian result)
{
result.Point = Math.Log(d);
return result;
}
public void Run()
{
InferenceEngine engine = new InferenceEngine();
if (!(engine.Algorithm is ExpectationPropagation))
{
return;
}
int[][] fighter1Data, fighter2Data, outcomeData, winTypeData;
int[] firstYearData;
int[] lastYearData;
int nFighters, nYears, startYear;
LoadData(out fighter1Data, out fighter2Data, out outcomeData, out winTypeData, out firstYearData, out lastYearData,
out nFighters, out nYears, out startYear);
//Skill prior
var skillPrior = new Gaussian(1000, 500 * 500);
var performancePrecisionPrior = Gamma.FromShapeAndRate(2, 26 * 26);
var skillChangePrecisionPrior = Gamma.FromShapeAndRate(2, 26 * 26);
var performancePrecision = Variable.Random(performancePrecisionPrior).Named("performancePrecision");
var skillChangePrecision = Variable.Random(skillChangePrecisionPrior).Named("skillChangePrecision");
var matchupThresholdPrior = new Gaussian(200, 50 * 50);
var matchupThreshold = Variable.Random(matchupThresholdPrior).Named("matchupThreshold");
var finishThresholdPrior = new Gaussian(20, 10 * 10);
var finishThreshold = Variable.Random(finishThresholdPrior).Named("finishThreshold");
var decicionThresholdPrior = new Gaussian(10, 10 * 10);
var decisionThreshold = Variable.Random(decicionThresholdPrior).Named("decisionThreshold");
Range fighter = new Range(nFighters).Named("fighter");
Range year = new Range(nYears).Named("year");
VariableArray <int> firstYear = Variable.Array <int>(fighter).Named("firstYear");
var skill = Variable.Array(Variable.Array <double>(fighter), year).Named("skill");
using (var yearBlock = Variable.ForEach(year))
{
var y = yearBlock.Index;
using (Variable.If(y == 0))
{
skill[year][fighter] = Variable.Random(skillPrior).ForEach(fighter);
}
using (Variable.If(y > 0))
{
using (Variable.ForEach(fighter))
{
Variable <bool> isFirstYear = (firstYear[fighter] >= y).Named("isFirstYear");
using (Variable.If(isFirstYear))
{
skill[year][fighter] = Variable.Random(skillPrior);
}
using (Variable.IfNot(isFirstYear))
{
skill[year][fighter] = Variable.GaussianFromMeanAndPrecision(skill[y - 1][fighter], skillChangePrecision);
}
}
}
}
firstYear.ObservedValue = firstYearData;
int[] nMatchesData = Util.ArrayInit(nYears, y => outcomeData[y].Length);
var nMatches = Variable.Observed(nMatchesData, year).Named("nMatches");
Range match = new Range(nMatches[year]).Named("match");
var fighter1 = Variable.Observed(fighter1Data, year, match).Named("fighter1");
var fighter2 = Variable.Observed(fighter2Data, year, match).Named("fighter2");
var outcome = Variable.Observed(outcomeData, year, match).Named("outcome");
var winType = Variable.Observed(winTypeData, year, match).Named("winType");
Variable.ConstrainTrue(finishThreshold > decisionThreshold);
Variable.ConstrainTrue(decisionThreshold > 0);
using (Variable.ForEach(year))
{
using (Variable.ForEach(match))
{
var w = fighter1[year][match];
var b = fighter2[year][match];
Variable <double> fighter1_performance = Variable.GaussianFromMeanAndPrecision(skill[year][w], performancePrecision).Named("fighter1_performance");
Variable <double> fighter2_performance = Variable.GaussianFromMeanAndPrecision(skill[year][b], performancePrecision).Named("fighter2_performance");
Variable.ConstrainFalse(skill[year][w] - skill[year][b] > matchupThreshold);
Variable.ConstrainFalse(skill[year][b] - skill[year][w] > matchupThreshold);
using (Variable.Case(outcome[year][match], 0))
{ // fighter2 wins
Variable.ConstrainTrue((fighter2_performance - fighter1_performance) > decisionThreshold);
//Finish
using (Variable.Case(winType[year][match], 1))
{
Variable.ConstrainTrue((fighter2_performance - fighter1_performance) > finishThreshold);
}
}
using (Variable.Case(outcome[year][match], 1))
{ // draw
Variable.ConstrainFalse((fighter2_performance - fighter1_performance) < decisionThreshold);
Variable.ConstrainFalse((fighter1_performance - fighter2_performance) < decisionThreshold);
}
using (Variable.Case(outcome[year][match], 2))
{ // fighter1 wins
Variable.ConstrainTrue((fighter1_performance - fighter2_performance) > decisionThreshold);
//Finish
using (Variable.Case(winType[year][match], 1))
{
Variable.ConstrainTrue((fighter1_performance - fighter2_performance) > finishThreshold);
}
}
}
}
year.AddAttribute(new Sequential()); // helps inference converge faster
engine.Compiler.UseSerialSchedules = false;
engine.NumberOfIterations = 200;
var skillPost = engine.Infer <Gaussian[][]>(skill);
var matchupThresholdPost = engine.Infer <Gaussian>(matchupThreshold);
var decisionThresholdPost = engine.Infer <Gaussian>(decisionThreshold);
var finishThresholdPost = engine.Infer <Gaussian>(finishThreshold);
var skillChangePrecisionPost = engine.Infer <Gamma>(skillChangePrecision);
Console.WriteLine("Matchup Threshold prec {0}", matchupThresholdPost);
Console.WriteLine("Decision Threshold {0}", decisionThresholdPost);
Console.WriteLine("Finish threshold {0}", finishThresholdPost);
Console.WriteLine("Skilll change prec {0}", skillChangePrecisionPost);
using (System.IO.StreamWriter file =
new System.IO.StreamWriter(@"outfile.txt"))
{
file.WriteLine("key year name skillmean variance");
for (int i = 0; i < nFighters; i++)
{
for (int y = 0; y < nYears; y++)
{
if (y >= firstYearData[i] && y <= lastYearData[i])
{
file.WriteLine("{0} {1} {2} {3} {4}", i, startYear + y, keyToFighter[i], skillPost[y][i].GetMean(), skillPost[y][i].GetVariance());
}
}
}
}
}
/// <summary>
/// Evidence message for VMP
/// </summary>
/// <param name="log">Incoming message from 'log'.</param>
/// <param name="d">Incoming message from 'd'.</param>
/// <returns>Zero</returns>
/// <remarks><para>
/// In Variational Message Passing, the evidence contribution of a deterministic factor is zero.
/// Adding up these values across all factors and variables gives the log-evidence estimate for VMP.
/// </para></remarks>
public static double AverageLogFactor(Gaussian log, Gamma d)
{
double m, v;
log.GetMeanAndVariance(out m, out v);
double Elogd=d.GetMeanLog();
double Elogd2;
if (!d.IsPointMass)
Elogd2 = MMath.Trigamma(d.Shape) + Elogd * Elogd;
else
Elogd2 = Math.Log(d.Point) * Math.Log(d.Point);
return -Elogd2/(2*v)+m*Elogd/v-m*m/(2*v)-MMath.LnSqrt2PI-.5*Math.Log(v);
}
public ModelData(Gaussian mean, Gamma precision)
{
AverageTimeDist = mean;
TrafficNoiseDist = precision;
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="log">Incoming message from 'log'.</param>
/// <param name="d">Incoming message from 'd'.</param>
/// <param name="to_log">Previous outgoing message to 'log'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(d,log) p(d,log) factor(log,d))</c>.
/// </para></remarks>
public static double LogAverageFactor(Gaussian log, Gamma d, [Fresh] Gaussian to_log)
{
Gamma g = Gamma.FromShapeAndRate(d.Shape + 1, d.Rate);
return d.Shape/d.Rate*ExpOp.LogAverageFactor(g, log, to_log);
}
public void WeightRatioTest()
{
var dataset = KernelSupportVectorMachineTest.training;
var inputs = dataset.Submatrix(null, 0, 3);
var labels = Tools.Scale(0, 1, -1, 1, dataset.GetColumn(4)).ToInt32();
Gaussian kernel = Gaussian.Estimate(inputs);
{
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo = new SequentialMinimalOptimization(machine, inputs, labels);
smo.Complexity = 1.0;
smo.WeightRatio = 10;
double error = smo.Run();
Assert.AreEqual(1.0, smo.PositiveWeight);
Assert.AreEqual(0.1, smo.NegativeWeight);
Assert.AreEqual(0.7142857142857143, error);
Assert.AreEqual(265.78327637381551, (machine.Kernel as Gaussian).Sigma);
Assert.AreEqual(39, machine.SupportVectors.Length);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = Math.Sign(machine.Compute(inputs[i]));
}
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(12, matrix.TruePositives); // has more importance
Assert.AreEqual(0, matrix.FalseNegatives); // has more importance
Assert.AreEqual(30, matrix.FalsePositives);
Assert.AreEqual(0, matrix.TrueNegatives);
Assert.AreEqual(1.0, matrix.Sensitivity);
Assert.AreEqual(0.0, matrix.Specificity);
Assert.AreEqual(0.44444444444444448, matrix.FScore);
Assert.AreEqual(0.0, matrix.MatthewsCorrelationCoefficient);
}
{
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo = new SequentialMinimalOptimization(machine, inputs, labels);
smo.Complexity = 1.0;
smo.WeightRatio = 0.1;
double error = smo.Run();
Assert.AreEqual(0.1, smo.PositiveWeight);
Assert.AreEqual(1.0, smo.NegativeWeight);
Assert.AreEqual(0.21428571428571427, error);
Assert.AreEqual(265.78327637381551, (machine.Kernel as Gaussian).Sigma);
Assert.AreEqual(18, machine.SupportVectors.Length);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = Math.Sign(machine.Compute(inputs[i]));
}
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(8, matrix.FalseNegatives);
Assert.AreEqual(1, matrix.FalsePositives); // has more importance
Assert.AreEqual(4, matrix.TruePositives);
Assert.AreEqual(29, matrix.TrueNegatives); // has more importance
Assert.AreEqual(0.33333333333333331, matrix.Sensitivity);
Assert.AreEqual(0.96666666666666667, matrix.Specificity);
Assert.AreEqual(0.47058823529411764, matrix.FScore);
Assert.AreEqual(0.41849149947774944, matrix.MatthewsCorrelationCoefficient);
}
}
