//[Fact]
internal void GaussianTimesBetaTest2()
{
Variable<bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
//Variable<double> x = Variable.GaussianFromMeanAndVariance(1.2, 3.4).Named("x");
//var x = Variable.Constant<double>(1.2).Named("x");
var s = Variable.Beta(5.6, 4.8).Named("s");
//var s = Variable.GaussianFromMeanAndPrecision(0, 1).Named("s");
Variable<double> y = 1.2*s;
y.Name = "y";
Variable.ConstrainEqualRandom(y, new Gaussian(2.7, 0.001));
block.CloseBlock();
InferenceEngine engine = new InferenceEngine(new VariationalMessagePassing());
var ca = engine.GetCompiledInferenceAlgorithm(evidence, s);
ca.Reset();
double oldLogEvidence = double.NegativeInfinity;
for (int i = 0; i < 1000; i++)
{
ca.Update(1);
double logEvidence1 = ca.Marginal<Bernoulli>(evidence.NameInGeneratedCode).LogOdds;
Console.WriteLine(logEvidence1);
if (i > 20 && System.Math.Abs(logEvidence1 - oldLogEvidence) < 0.01)
break;
oldLogEvidence = logEvidence1;
}
//Gaussian xActual = ca.Marginal<Gaussian>(x);
Beta sActual = ca.Marginal<Beta>(s.NameInGeneratedCode);
//Console.WriteLine("x = {0}", xActual);
Console.WriteLine("s = {0}", sActual);
}
public void GPClassificationTest4()
{
bool[] yData = new bool[] { false, false, true, false };
double[] xData = new double[]
{
-1.555555555555556, -1.111111111111111, -0.2222222222222223, 1.555555555555555
};
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { Vector.Zero(1) };
IKernelFunction kf = new SquaredExponential(0.0);
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <bool> y = Variable.Array <bool>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = (h[item] > 0);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { 0.409693797629808 });
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
}
/// <summary>
/// Test modified EP updates
/// </summary>
internal void StudentIsPositiveTest()
{
double shape = 1;
Gamma precPrior = Gamma.FromShapeAndRate(shape, shape);
// mean=-1 causes improper messages
double mean = -1;
double evExpected;
Gaussian xExpected = StudentIsPositiveExact(mean, precPrior, out evExpected);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <double> prec = Variable.Random(precPrior).Named("prec");
Variable <double> x = Variable.GaussianFromMeanAndPrecision(mean, prec).Named("x");
Variable.ConstrainPositive(x);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
x.AddAttribute(new TraceMessages());
//x.InitialiseTo(Gaussian.FromMeanAndVariance(-3.719, 4.836));
//GaussianOp.ForceProper = false;
//GaussianOp.modified = true;
//engine.Compiler.GivePriorityTo(typeof(GaussianOp_Laplace));
//engine.Compiler.GivePriorityTo(typeof(GaussianOp_Slow));
GaussianOp_Laplace.modified = true;
GaussianOp_Laplace.modified2 = true;
Console.WriteLine("x = {0} should be {1}", engine.Infer(x), xExpected);
double evActual = System.Math.Exp(engine.Infer <Bernoulli>(evidence).LogOdds);
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
}
/// <summary>
/// Constructs an LDA model
/// </summary>
/// <param name="sizeVocab">Size of vocabulary</param>
/// <param name="numTopics">Number of topics</param>
public LDAModel(int sizeVocab, int numTopics)
{
SizeVocab = sizeVocab;
NumTopics = numTopics;
ThetaSparsity = Sparsity.Dense;
PhiSparsity = Sparsity.ApproximateWithTolerance(0.00000000001); // Allow for round-off error
NumDocuments = Variable.New <int>().Named("NumDocuments");
//---------------------------------------------
// The model
//---------------------------------------------
Range D = new Range(NumDocuments).Named("D");
Range W = new Range(SizeVocab).Named("W");
Range T = new Range(NumTopics).Named("T");
NumWordsInDoc = Variable.Array <int>(D).Named("NumWordsInDoc");
Range WInD = new Range(NumWordsInDoc[D]).Named("WInD");
// Surround model by a stochastic If block so that we can compute model evidence
Evidence = Variable.Bernoulli(0.5).Named("Evidence");
IfBlock evidenceBlock = Variable.If(Evidence);
Theta = Variable.Array <Vector>(D);
Theta.SetSparsity(ThetaSparsity);
Theta.SetValueRange(T);
ThetaPrior = Variable.Array <Dirichlet>(D).Named("ThetaPrior");
Theta[D] = Variable <Vector> .Random(ThetaPrior[D]);
Phi = Variable.Array <Vector>(T);
Phi.SetSparsity(PhiSparsity);
Phi.SetValueRange(W);
PhiPrior = Variable.Array <Dirichlet>(T).Named("PhiPrior");
Phi[T] = Variable <Vector> .Random(PhiPrior[T]);
Words = Variable.Array(Variable.Array <int>(WInD), D).Named("Words");
WordCounts = Variable.Array(Variable.Array <double>(WInD), D).Named("WordCounts");
using (Variable.ForEach(D))
{
using (Variable.ForEach(WInD))
{
using (Variable.Repeat(WordCounts[D][WInD]))
{
Variable <int> topic = Variable.Discrete(Theta[D]).Named("topic");
using (Variable.Switch(topic))
{
Words[D][WInD] = Variable.Discrete(Phi[topic]);
}
}
}
}
evidenceBlock.CloseBlock();
ThetaInit = Variable.New <IDistribution <Vector[]> >().Named("ThetaInit");
Theta.InitialiseTo(ThetaInit);
Engine = new InferenceEngine(new VariationalMessagePassing());
Engine.Compiler.ShowWarnings = false;
Engine.ModelName = "LDAModel";
}
public void GPClassificationTest2()
{
bool[] yData = new bool[] { false, true, false };
double[] xData = new double[]
{
-1.555555555555556, -0.2222222222222223, 1.555555555555555
};
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { Vector.Zero(1) };
IKernelFunction kf = new SquaredExponential(0.0);
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <bool> y = Variable.Array <bool>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = (h[item] > 0);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { 0.573337393823702 });
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
double[] xTest = new double[]
{
-2, -1, 0.0
};
Vector[] xTestVec = Array.ConvertAll(xTest, v => Vector.Constant(1, v));
double[] yMeanTest = new double[]
{
0.077592778583272, 0.347746707713812, 0.573337393823702
};
double[] yVarTest = new double[]
{
0.986784459962251, 0.734558782611933, 0.278455962249970
};
for (int i = 0; i < xTestVec.Length; i++)
{
Gaussian pred = sgp.Marginal(xTestVec[i]);
Gaussian predExpected = new Gaussian(yMeanTest[i], yVarTest[i]);
Console.WriteLine("f({0}) = {1} should be {2}", xTest[i], pred, predExpected);
Assert.True(predExpected.MaxDiff(pred) < 1e-4);
}
double evExpected = -2.463679892165236;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-6) < 1e-4);
}
public void GPClassificationTest1()
{
bool[] yData = new bool[] { true };
double[] xData = new double[] { -0.2222222222222223 };
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { Vector.Zero(1) };
IKernelFunction kf = new SquaredExponential(0.0);
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <bool> y = Variable.Array <bool>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = (h[item] > 0);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { 0.778424938343491 });
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
double[] xTest = new double[]
{
-2, -1, 0.0
};
Vector[] xTestVec = Array.ConvertAll(xTest, v => Vector.Constant(1, v));
double[] yMeanTest = new double[]
{
0.105348359509159, 0.472138591390244, 0.778424938343491
};
double[] yVarTest = new double[]
{
0.988901723148729, 0.777085150520037, 0.394054615364932
};
for (int i = 0; i < xTestVec.Length; i++)
{
Gaussian pred = sgp.Marginal(xTestVec[i]);
Gaussian predExpected = new Gaussian(yMeanTest[i], yVarTest[i]);
Console.WriteLine("f({0}) = {1} should be {2}", xTest[i], pred, predExpected);
Assert.True(predExpected.MaxDiff(pred) < 1e-4);
}
double evExpected = -0.693147180559945;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-6) < 1e-4);
}
/// <inheritdoc />
public override void CreateModel(int numTweets, int numClasses, int numVocab = 0, bool withEvidence = true, bool withGoldLabels = false)
{
IfBlock block = null;
if (withEvidence)
{
this.Evidence = Variable.Bernoulli(0.5).Named("evidence");
block = Variable.If(this.Evidence);
}
// Biased community model
base.CreateModel(numTweets, numClasses, numVocab, false, withGoldLabels);
// Add in the word generation conditioned on true label variable.
this.WordsInVocabulary = new Range(numVocab).Named("wordsInVocabulary");
this.ProbWords = Variable.Array <Vector>(this.Labels).Named("probWords");
this.ProbWords.SetValueRange(this.WordsInVocabulary);
this.ProbWords.SetSparsity(Sparsity.Sparse);
this.WordCount = Variable.Array <int>(this.Tweets).Named("wordCount");
this.Words = new Range(this.WordCount[this.Tweets]).Named("words");
this.Word = Variable.Array(Variable.Array <int>(this.Words), this.Tweets).Named("word");
this.ProbWordsPrior = Variable.Array <Dirichlet>(this.Labels).Named("probWordsPrior");
this.ProbWords[this.Labels] = Variable <Vector> .Random(this.ProbWordsPrior[this.Labels]);
this.LikelihoodExponent = Variable.Array <double>(this.Tweets).Named(nameof(this.LikelihoodExponent));
using (Variable.ForEach(this.Tweets))
{
RepeatBlock repeatBlock = null;
if (UseRepeatFactor)
{
repeatBlock = Variable.Repeat(this.LikelihoodExponent[this.Tweets]);
}
using (Variable.Switch(this.TrueLabel[this.Tweets]))
{
this.Word[this.Tweets][this.Words] = Variable.Discrete(this.ProbWords[this.TrueLabel[this.Tweets]]).ForEach(this.Words);
}
if (UseRepeatFactor)
{
repeatBlock?.CloseBlock();
}
}
if (withEvidence)
{
block.CloseBlock();
}
this.HasEvidence = withEvidence;
}
public void GPRegressionTest1()
{
double[] yData = new double[] { 1.036659040456137 };
double[] xData = new double[] { -0.2222222222222223 };
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { xVec[0] };
IKernelFunction kf = new SquaredExponential(System.Math.Log(2.0));
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <double> y = Variable.Array <double>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = Variable.GaussianFromMeanAndVariance(h[item], 0.1);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { 0.942417309505579 });
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
double[] xTest = new double[]
{
-2, -1, 0.0
};
Vector[] xTestVec = Array.ConvertAll(xTest, v => Vector.Constant(1, v));
// computed by matlab/MNT/GP/test_gpr.m
double[] yTest = new double[]
{
0.634848540665472, 0.873781982196160, 0.936617836728720
};
for (int i = 0; i < xTestVec.Length; i++)
{
double pred = sgp.Mean(xTestVec[i]);
Console.WriteLine("Ef({0}) = {1} should be {2}", xTest[i], pred, yTest[i]);
Assert.True(MMath.AbsDiff(pred, yTest[i], 1e-6) < 1e-4);
}
double evExpected = -1.455076334997490;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-6) < 1e-4);
}
internal void FitNegativeBinomial()
{
// generate data from the model
double r = 2;
double p = 0.3;
int[] data = new int[] { 1, 4, 5, 14, 0, 3, 2, 18, 0, 1, 8, 1, 4, 3, 6, 4, 9, 5, 1, 10, 5, 9, 2, 3, 3, 9, 14, 3, 5, 12 };
int N = data.Length;
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
var rate = Variable.GammaFromShapeAndRate(1, 1).Named("rate");
//var shape = Variable.GammaFromShapeAndRate(1,1).Named("shape");
var shape = Variable.GammaFromShapeAndRate(1, 1).Named("shape");
var lambda = Variable.GammaFromShapeAndRate(shape, rate).Named("lambda");
Range Nrange = new Range(N);
var y = Variable.Array <int>(Nrange).Named("y");
y[Nrange] = Variable.Poisson(lambda).ForEach(Nrange);
y.ObservedValue = data;
block.CloseBlock();
InferenceEngine ie = new InferenceEngine(new VariationalMessagePassing());
ie.ShowFactorGraph = true;
var ca = ie.GetCompiledInferenceAlgorithm(evidence, rate, shape);
ca.Reset();
double oldLogEvidence = double.NegativeInfinity;
for (int i = 0; i < 1000; i++)
{
ca.Update(1);
double logEvidence1 = ca.Marginal <Bernoulli>(evidence.NameInGeneratedCode).LogOdds;
Console.WriteLine(logEvidence1);
if (i > 20 && System.Math.Abs(logEvidence1 - oldLogEvidence) < 0.01)
{
break;
}
oldLogEvidence = logEvidence1;
}
Gamma shapePost = ca.Marginal <Gamma>(shape.NameInGeneratedCode);
Gamma ratePost = ca.Marginal <Gamma>(rate.NameInGeneratedCode);
double mean, variance;
shapePost.GetMeanAndVariance(out mean, out variance);
Console.WriteLine("shape = " + mean + " +/- " + System.Math.Sqrt(variance) + " true= " + r);
ratePost.GetMeanAndVariance(out mean, out variance);
Console.WriteLine("rate = " + mean + " +/- " + System.Math.Sqrt(variance) + " true= " + p / (1 - p));
}
public void PoissonExpTest2()
{
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <double> x = Variable.GaussianFromMeanAndVariance(1.2, 3.4).Named("x");
Rand.Restart(12347);
int n = 10;
int N = 10;
int[] data = new int[N];
for (int i = 0; i < N; i++)
{
data[i] = Rand.Binomial(n, 1.0 / (double)n);
}
data = new int[] { 5, 6, 7 };
Range item = new Range(data.Length).Named("item");
VariableArray <double> ex = Variable.Array <double>(item).Named("ex");
ex[item] = Variable.Exp(x).ForEach(item);
VariableArray <int> y = Variable.Array <int>(item).Named("y");
y[item] = Variable.Poisson(ex[item]);
block.CloseBlock();
y.ObservedValue = data;
InferenceEngine ie = new InferenceEngine(new VariationalMessagePassing());
var ca = ie.GetCompiledInferenceAlgorithm(evidence, x);
double oldLogEvidence = double.NegativeInfinity;
for (int i = 0; i < 1000; i++)
{
ca.Update(1);
double logEvidence1 = ca.Marginal <Bernoulli>(evidence.NameInGeneratedCode).LogOdds;
Console.WriteLine(logEvidence1);
if (i > 20 && System.Math.Abs(logEvidence1 - oldLogEvidence) < 1e-10)
{
break;
}
oldLogEvidence = logEvidence1;
}
Gaussian xExpected = new Gaussian(1.755071011884509, 0.055154577283323);
Gaussian xActual = ca.Marginal <Gaussian>(x.NameInGeneratedCode);
Console.WriteLine("x = {0} should be {1}", xActual, xExpected);
Assert.True(xExpected.MaxDiff(xActual) < 1e-3);
}
//Addition by Guy Templeton, get log evidence from learned mixing coeff.
public double GetLogEvidence()
{
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
Range classes = new Range(numOfClasses);
IfBlock block = Variable.If(evidence);
VectorGaussian[] wObserved = trainModel.wPrior.ObservedValue;
VectorGaussian[] empty = Util.ArrayInit(numOfClasses, c => (c == 0) ?
VectorGaussian.PointMass(Vector.Zero(numOfFeatures)) :
VectorGaussian.FromMeanAndPrecision(Vector.Zero(numOfFeatures), PositiveDefiniteMatrix.Identity(numOfFeatures)));
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
return(engine.Infer <Bernoulli>(evidence).LogOdds);
}
/// <inheritdoc />
public override void CreateModel(int numTweets, int numClasses, int numVocab = 0, bool withEvidence = true, bool withGoldLabels = false)
{
IfBlock block = null;
if (withEvidence)
{
this.Evidence = Variable.Bernoulli(0.5).Named("evidence");
block = Variable.If(this.Evidence);
}
this.CreateModelStub(numTweets, numClasses, withGoldLabels);
// Honest worker
this.AbilityPrior = Variable.Array <Beta>(this.Workers).Named("abilityPrior");
this.Ability = Variable.Array <double>(this.Workers).Named("ability");
this.Ability[this.Workers] = Variable <double> .Random(this.AbilityPrior[this.Workers]);
this.RandomGuessPrior = Variable.New <Dirichlet>().Named("randomGuessPrior");
this.RandomGuessProbability = Variable <Vector> .Random(this.RandomGuessPrior).Named("randomGuessProb");
using (Variable.ForEach(this.Workers))
{
var trueLabels = Variable.Subarray(this.TrueLabel, this.WorkerJudgedTweetIndex[this.Workers]).Named("trueLabelSubarray");
trueLabels.SetValueRange(this.Labels);
using (Variable.ForEach(this.WorkerJudgment))
{
var workerIsCorrect = Variable.Bernoulli(this.Ability[this.Workers]).Named("isCorrect");
using (Variable.If(workerIsCorrect))
{
var labelsEqual = (this.WorkerLabel[this.Workers][this.WorkerJudgment] == trueLabels[this.WorkerJudgment]).Named("labelsEqual");
Variable.ConstrainEqualRandom(labelsEqual, new Bernoulli(0.9999)); // Add a slight amount of noise due to Infer.NET compiler bug.
}
using (Variable.IfNot(workerIsCorrect))
{
this.WorkerLabel[this.Workers][this.WorkerJudgment] = Variable.Discrete(this.RandomGuessProbability);
}
}
}
if (withEvidence)
{
block.CloseBlock();
}
this.HasEvidence = withEvidence;
}
/// <summary>
/// Helper class to instantiate Gaussian Process regressor
/// </summary>
public GaussianProcessRegressor(Vector[] trainingInputs, bool closeBlock = true)
{
// Modelling code
Evidence = Variable.Bernoulli(0.5).Named("evidence");
Block = Variable.If(Evidence);
Prior = Variable.New <SparseGP>().Named("prior");
F = Variable <IFunction> .Random(Prior).Named("f");
X = Variable.Observed(trainingInputs).Named("x");
J = X.Range.Named("j");
// If generating data, we can close block here
if (closeBlock)
{
Block.CloseBlock();
}
}
public GaussianProcessRegressor(Vector[] trainingInputs, bool useStudentTLikelihood, double[] trainingOutputs) : this(trainingInputs, closeBlock : false)
{
VariableArray <double> y = Variable.Observed(trainingOutputs, J).Named("y");
if (!useStudentTLikelihood)
{
// Standard Gaussian Process
Console.WriteLine("Training a Gaussian Process regressor");
var score = GetScore(X, F, J);
y[J] = Variable.GaussianFromMeanAndVariance(score, 0.0);
}
else
{
// Gaussian Process with Student-t likelihood
Console.WriteLine("Training a Gaussian Process regressor with Student-t likelihood");
var noisyScore = GetNoisyScore(X, F, J, trainingOutputs);
y[J] = Variable.GaussianFromMeanAndVariance(noisyScore[J], 0.0);
}
Block.CloseBlock();
}
/// <inheritdoc />
public override void CreateModel(int numTweets, int numClasses, int numVocab = 0, bool withEvidence = true, bool withGoldLabels = false)
{
IfBlock block = null;
if (withEvidence)
{
this.Evidence = Variable.Bernoulli(0.5).Named("evidence");
block = Variable.If(this.Evidence);
}
this.CreateModelStub(numTweets, numClasses, withGoldLabels);
// Worker biases
this.ProbWorkerLabelPrior = Variable.Array(Variable.Array <Dirichlet>(this.Labels), this.Workers).Named("probWorkerLabelPrior");
this.ProbWorkerLabel = Variable.Array(Variable.Array <Vector>(this.Labels), this.Workers).Named("probWorkerLabel");
this.ProbWorkerLabel[this.Workers][this.Labels] = Variable <Vector> .Random(this.ProbWorkerLabelPrior[this.Workers][this.Labels]);
this.ProbWorkerLabel.SetValueRange(this.Labels);
// Condition on latent truth
using (Variable.ForEach(this.Workers))
{
var trueLabels = Variable.Subarray(this.TrueLabel, this.WorkerJudgedTweetIndex[this.Workers]).Named("trueLabelSubarray");
trueLabels.SetValueRange(this.Labels);
using (Variable.ForEach(this.WorkerJudgment))
{
var trueLabel = trueLabels[this.WorkerJudgment];
using (Variable.Switch(trueLabel))
{
this.WorkerLabel[this.Workers][this.WorkerJudgment] = Variable.Discrete(this.ProbWorkerLabel[this.Workers][trueLabel]);
}
}
}
if (withEvidence)
{
block.CloseBlock();
}
this.HasEvidence = withEvidence;
}
/// <summary>
/// Test a difficult case
/// </summary>
internal void StudentIsPositiveTest5()
{
// depending on the exact setting of priors, the messages will alternate between proper and improper
Gamma precPrior = new Gamma(5, 0.2);
// mean=-1 causes improper messages
var mean = Variable.Random(new Gaussian(-0.9, 0.25)).Named("mean");
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <double> prec = Variable.Random(precPrior).Named("prec");
Variable <double> x = Variable.GaussianFromMeanAndPrecision(mean, prec).Named("x");
Variable <bool> y = Variable.IsPositive(x);
Variable.ConstrainEqualRandom(y, new Bernoulli(0.8889));
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
x.AddAttribute(new TraceMessages());
Console.WriteLine(engine.Infer(x));
//Console.WriteLine("x = {0} should be {1}", engine.Infer(x), xExpected);
double evActual = System.Math.Exp(engine.Infer <Bernoulli>(evidence).LogOdds);
//Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
}
internal void NevenaRunArrayIntervention()
{
int S1 = 0, S2 = 1, Y1 = 2, Y2 = 3;
int numVar = 4;
int[] dimVar = Enumerable.Range(0, numVar).Select(o => 2).ToArray();
// Data
int numData = 100;
int[][] dataX = new int[numVar][];
bool[][] dataI = new bool[numVar][];
int[] observed = new int[] { Y1, Y2 };
int[][] parents = new int[numVar][];
parents[S1] = new int[0] {
};
parents[S2] = new int[0] {
};
parents[Y1] = new int[] { S2 };
parents[Y2] = new int[] { S1, S2 };
// here data iid (not sampled from dag)
var coin = new Discrete(new double[2] {
0, 1
});
for (int j = 0; j < numVar; j++)
{
dataX[j] = new int[numData];
dataI[j] = new bool[numData];
for (int i = 0; i < numData; i++)
{
dataX[j][i] = coin.Sample();
dataI[j][i] = false;
}
}
// Model
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock H1 = Variable.If(evidence);
Range N = new Range(numData).Named("N");
Range[] rVariables = new Range[numVar];
for (int d = 0; d < numVar; d++)
{
rVariables[d] = new Range(dimVar[d]);
}
// Six different priors, depending on #parents and intervention
var priorX0 = new Variable <Vector> [numVar];
var priorX1 = new VariableArray <Vector> [numVar];
var priorX2 = new VariableArray <VariableArray <Vector>, Vector[][]> [numVar];
var priorXI0 = new Variable <Vector> [numVar];
var priorXI1 = new VariableArray <Vector> [numVar];
var priorXI2 = new VariableArray <VariableArray <Vector>, Vector[][]> [numVar];
for (int d = 0; d < numVar; d++)
{
int numPar = parents[d].Length;
if (numPar == 0)
{
priorX0[d] = Variable.DirichletSymmetric(rVariables[d], 1 / (double)dimVar[d]).Named("priorX0" + d);
priorXI0[d] = Variable.DirichletSymmetric(rVariables[d], 1 / (double)dimVar[d]).Named("priorXI0" + d);
}
if (numPar == 1)
{
priorX1[d] = CPT1(rVariables[d], rVariables[parents[d][0]]).Named("priorX1" + d);
priorXI1[d] = CPT1(rVariables[d], rVariables[parents[d][0]]).Named("priorXI1" + d);
}
if (numPar == 2)
{
priorX2[d] = CPT2(rVariables[d], rVariables[parents[d][0]], rVariables[parents[d][1]]).Named("cptX2" + d);
priorXI2[d] = CPT2(rVariables[d], rVariables[parents[d][0]], rVariables[parents[d][1]]).Named("cptXI2" + d);
}
}
// Variable arrays
VariableArray <int>[] X = new VariableArray <int> [numVar];
VariableArray <bool>[] I = new VariableArray <bool> [numVar];
for (int d = 0; d < numVar; d++)
{
I[d] = Variable.Array <bool>(N).Named("I" + d);
I[d].ObservedValue = dataI[d];
X[d] = Variable.Array <int>(N).Named("X" + d);
X[d].SetValueRange(rVariables[d]);
int ind = Array.IndexOf(observed, d);
if (ind > -1)
{
X[d].ObservedValue = dataX[d];
}
}
for (int d = 0; d < numVar; d++)
{
int numPar = parents[d].Length;
if (numPar == 0)
{
using (Variable.ForEach(N))
{
using (Variable.IfNot(I[d][N]))
X[d][N] = Variable.Discrete(priorX0[d]);
using (Variable.If(I[d][N]))
X[d][N] = Variable.Discrete(priorXI0[d]);
}
}
if (numPar == 1)
{
int par = parents[d][0];
using (Variable.ForEach(N))
{
using (Variable.Switch(X[par][N]))
{
using (Variable.IfNot(I[d][N]))
X[d][N] = Variable.Discrete(priorX1[d][X[par][N]]);
using (Variable.If(I[d][N]))
X[d][N] = Variable.Discrete(priorXI1[d][X[par][N]]);
}
}
}
if (numPar == 2)
{
using (Variable.ForEach(N))
{
using (Variable.Switch(X[parents[d][0]][N]))
using (Variable.Switch(X[parents[d][1]][N]))
{
using (Variable.IfNot(I[d][N]))
X[d][N] = Variable.Discrete(priorX2[d][X[parents[d][0]][N]][X[parents[d][1]][N]]);
using (Variable.If(I[d][N]))
X[d][N] = Variable.Discrete(priorXI2[d][X[parents[d][0]][N]][X[parents[d][1]][N]]);
}
}
}
}
H1.CloseBlock();
// Inference engine
var ieVMP = new InferenceEngine(new VariationalMessagePassing());
ieVMP.NumberOfIterations = 100;
double eviVMP = ieVMP.Infer <Bernoulli>(evidence).LogOdds;
var ieEP = new InferenceEngine(new ExpectationPropagation());
ieEP.NumberOfIterations = 100;
double eviEP = ieEP.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine(eviVMP + " " + eviEP);
}
private void GaussianFromMeanAndVarianceTest(double mm, double vm, double mx, double vx, double a, double b)
{
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <double> mean = Variable.GaussianFromMeanAndVariance(mm, vm).Named("mean");
Variable <double> variance = Variable.GammaFromShapeAndRate(a, b).Named("variance");
Variable <double> x = Variable.GaussianFromMeanAndVariance(mean, variance).Named("x");
Variable.ConstrainEqualRandom(x, new Gaussian(mx, vx));
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
engine.Compiler.RecommendedQuality = QualityBand.Experimental;
double evExpected;
Gaussian xExpected;
Gamma vExpected;
if (a == 1 || a == 2)
{
double c = System.Math.Sqrt(2 * b);
double m = c * (mx - mm);
double v = c * c * (vx + vm);
double Z, mu, m2u;
VarianceGammaTimesGaussianMoments(a, m, v, out Z, out mu, out m2u);
evExpected = System.Math.Log(Z * c);
double vu = m2u - mu * mu;
double r = Double.IsPositiveInfinity(vx) ? 1.0 : vx / (vx + vm);
double mp = r * (mu / c + mm) + (1 - r) * mx;
double vp = r * r * vu / (c * c) + r * vm;
xExpected = new Gaussian(mp, vp);
double Zplus1, Zplus2;
VarianceGammaTimesGaussianMoments(a + 1, m, v, out Zplus1, out mu, out m2u);
VarianceGammaTimesGaussianMoments(a + 2, m, v, out Zplus2, out mu, out m2u);
double vmp = a / b * Zplus1 / Z;
double vm2p = a * (a + 1) / (b * b) * Zplus2 / Z;
double vvp = vm2p - vmp * vmp;
vExpected = Gamma.FromMeanAndVariance(vmp, vvp);
}
else
{
int n = 1000000;
GaussianEstimator est = new GaussianEstimator();
GammaEstimator vEst = new GammaEstimator();
Gaussian xLike = new Gaussian(mx, vx);
for (int i = 0; i < n; i++)
{
double m = Gaussian.Sample(mm, 1 / vm);
double v = Rand.Gamma(a) / b;
double xSample = Gaussian.Sample(m, 1 / v);
double weight = System.Math.Exp(xLike.GetLogProb(xSample));
est.Add(xSample, weight);
vEst.Add(v, weight);
}
evExpected = System.Math.Log(est.mva.Count / n);
xExpected = est.GetDistribution(new Gaussian());
vExpected = vEst.GetDistribution(new Gamma());
}
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Gaussian xActual = engine.Infer <Gaussian>(x);
Console.WriteLine("x = {0} should be {1}", xActual, xExpected);
Gamma vActual = engine.Infer <Gamma>(variance);
Console.WriteLine("variance = {0} should be {1}", vActual, vExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-10) < 1e-4);
Assert.True(xExpected.MaxDiff(xActual) < 1e-4);
Assert.True(vExpected.MaxDiff(vActual) < 1e-4);
}
/// <summary>
/// Constructs an LDA model
/// </summary>
/// <param name="sizeVocab">Size of vocabulary</param>
/// <param name="numTopics">Number of topics</param>
public LDAShared(int numBatches, int sizeVocab, int numTopics)
{
SizeVocab = sizeVocab;
NumTopics = numTopics;
ThetaSparsity = Sparsity.Dense;
PhiSparsity = Sparsity.ApproximateWithTolerance(0.00000000001); // Allow for round-off error
NumDocuments = Variable.New <int>().Named("NumDocuments");
NumBatches = numBatches;
IterationsPerPass = new int[] { 1, 3, 5, 7, 9 };
//---------------------------------------------
// The model
//---------------------------------------------
Range D = new Range(NumDocuments).Named("D");
Range W = new Range(SizeVocab).Named("W");
Range T = new Range(NumTopics).Named("T");
NumWordsInDoc = Variable.Array <int>(D).Named("NumWordsInDoc");
Range WInD = new Range(NumWordsInDoc[D]).Named("WInD");
Evidence = SharedVariable <bool> .Random(new Bernoulli(0.5)).Named("Evidence");
Evidence.IsEvidenceVariable = true;
Phi = SharedVariable <Vector> .Random(T, CreateUniformDirichletArray(numTopics, sizeVocab, PhiSparsity)).Named("Phi");
// Phi definition sub-model - just one copy
PhiDefModel = new Model(1).Named("PhiDefModel");
IfBlock evidencePhiDefBlock = null;
EvidencePhiDef = Evidence.GetCopyFor(PhiDefModel).Named("EvidencePhiDef");
evidencePhiDefBlock = Variable.If(EvidencePhiDef);
PhiDef = Variable.Array <Vector>(T).Named("PhiDef");
PhiDef.SetSparsity(PhiSparsity);
PhiDef.SetValueRange(W);
PhiPrior = Variable.Array <Dirichlet>(T).Named("PhiPrior");
PhiDef[T] = Variable <Vector> .Random(PhiPrior[T]);
Phi.SetDefinitionTo(PhiDefModel, PhiDef);
evidencePhiDefBlock.CloseBlock();
// Document sub-model - many copies
DocModel = new Model(numBatches).Named("DocModel");
IfBlock evidenceDocBlock = null;
EvidenceDoc = Evidence.GetCopyFor(DocModel).Named("EvidenceDoc");
evidenceDocBlock = Variable.If(EvidenceDoc);
Theta = Variable.Array <Vector>(D).Named("Theta");
Theta.SetSparsity(ThetaSparsity);
Theta.SetValueRange(T);
ThetaPrior = Variable.Array <Dirichlet>(D).Named("ThetaPrior");
Theta[D] = Variable <Vector> .Random(ThetaPrior[D]);
PhiDoc = Phi.GetCopyFor(DocModel);
PhiDoc.AddAttribute(new MarginalPrototype(Dirichlet.Uniform(sizeVocab, PhiSparsity)));
Words = Variable.Array(Variable.Array <int>(WInD), D).Named("Words");
WordCounts = Variable.Array(Variable.Array <double>(WInD), D).Named("WordCounts");
using (Variable.ForEach(D))
{
using (Variable.ForEach(WInD))
{
using (Variable.Repeat(WordCounts[D][WInD]))
{
Variable <int> topic = Variable.Discrete(Theta[D]).Named("topic");
using (Variable.Switch(topic))
{
Words[D][WInD] = Variable.Discrete(PhiDoc[topic]);
}
}
}
}
evidenceDocBlock.CloseBlock();
// Initialization to break symmetry
ThetaInit = Variable.Array <Dirichlet>(D).Named("ThetaInit");
Theta[D].InitialiseTo(ThetaInit[D]);
EnginePhiDef = new InferenceEngine(new VariationalMessagePassing());
EnginePhiDef.Compiler.ShowWarnings = false;
EnginePhiDef.ModelName = "LDASharedPhiDef";
Engine = new InferenceEngine(new VariationalMessagePassing());
Engine.OptimiseForVariables = new IVariable[] { Theta, PhiDoc, EvidenceDoc };
Engine.Compiler.ShowWarnings = false;
Engine.ModelName = "LDAShared";
Engine.Compiler.ReturnCopies = false;
Engine.Compiler.FreeMemory = true;
}
public void CreateModel(int NumTasks, int NumClasses, int VocabSize, int numBatches = 3)
{
WorkerCount = Variable.New <int>().Named("WorkerCount");
// Set up inference engine
Engine = new InferenceEngine(new VariationalMessagePassing());
// Set engine flags
Engine.Compiler.WriteSourceFiles = true;
Engine.Compiler.UseParallelForLoops = true;
evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
// Set up ranges
n = new Range(NumTasks).Named("N");
c = new Range(NumClasses).Named("C");
k = new Range(WorkerCount).Named("K");
WorkerTaskCount = Variable.Array <int>(k).Named("WorkerTaskCount");
kn = new Range(WorkerTaskCount[k]).Named("KN");
WorkerTaskIndex = Variable.Array(Variable.Array <int>(kn), k).Named("Task");
WorkerTaskIndex.SetValueRange(n);
// Initialise truth
BackgroundLabelProbPrior = Variable.New <Dirichlet>().Named("TruthProbPrior");
BackgroundLabelProb = Variable <Vector> .Random(BackgroundLabelProbPrior).Named("TruthProb");
BackgroundLabelProb.SetValueRange(c);
// Truth distributions
TrueLabel = Variable.Array <int>(n).Named("Truth");
TrueLabel[n] = Variable.Discrete(BackgroundLabelProb).ForEach(n);
//VocabSize = Variable.New<int>();
w = new Range(VocabSize).Named("W");
ProbWord = Variable.Array <Vector>(c).Named("ProbWord");
ProbWord.SetValueRange(w);
WordCount = Variable.Array <int>(n).Named("WordCount");
nw = new Range(WordCount[n]).Named("WN");
Words = Variable.Array(Variable.Array <int>(nw), n).Named("Word");
ProbWordPrior = Variable.New <Dirichlet>().Named("ProbWordPrior");
ProbWord[c] = Variable <Vector> .Random(ProbWordPrior).ForEach(c);
// Initialise user profiles
ConfusionMatrixPrior = Variable.Array(Variable.Array <Dirichlet>(c), k).Named("WorkerConfusionMatrixPrior");
WorkerConfusionMatrix = Variable.Array(Variable.Array <Vector>(c), k).Named("WorkerConfusionMatrix");
WorkerConfusionMatrix[k][c] = Variable <Vector> .Random(ConfusionMatrixPrior[k][c]);
WorkerConfusionMatrix.SetValueRange(c);
// Vote distributions
WorkerLabel = Variable.Array(Variable.Array <int>(kn), k).Named("WorkerLabel");
using (Variable.ForEach(k))
{
var trueLabel = Variable.Subarray(TrueLabel, WorkerTaskIndex[k]).Named("TrueLabelSubarray");
trueLabel.SetValueRange(c);
using (Variable.ForEach(kn))
{
using (Variable.Switch(trueLabel[kn]))
{
WorkerLabel[k][kn] = Variable.Discrete(WorkerConfusionMatrix[k][trueLabel[kn]]);
}
}
}
// Words inference
using (Variable.ForEach(n))
{
using (Variable.Switch(TrueLabel[n]))
{
Words[n][nw] = Variable.Discrete(ProbWord[TrueLabel[n]]).ForEach(nw);
}
}
block.CloseBlock();
}
public void Run()
{
InferenceEngine engine = new InferenceEngine();
if (!(engine.Algorithm is Algorithms.ExpectationPropagation))
{
Console.WriteLine("This example only runs with Expectation Propagation");
return;
}
// The data
Vector[] inputs = new Vector[]
{
Vector.FromArray(new double[2] {
0, 0
}),
Vector.FromArray(new double[2] {
0, 1
}),
Vector.FromArray(new double[2] {
1, 0
}),
Vector.FromArray(new double[2] {
0, 0.5
}),
Vector.FromArray(new double[2] {
1.5, 0
}),
Vector.FromArray(new double[2] {
0.5, 1.0
})
};
bool[] outputs = { true, true, false, true, false, false };
// Open an evidence block to allow model scoring
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
// Set up the GP prior, a distribution over functions, which will be filled in later
Variable <SparseGP> prior = Variable.New <SparseGP>().Named("prior");
// The sparse GP variable - a random function
Variable <IFunction> f = Variable <IFunction> .Random(prior).Named("f");
// The locations to evaluate the function
VariableArray <Vector> x = Variable.Observed(inputs).Named("x");
Range j = x.Range.Named("j");
// The observation model
VariableArray <bool> y = Variable.Observed(outputs, j).Named("y");
Variable <double> score = Variable.FunctionEvaluate(f, x[j]).Named("score");
y[j] = (Variable.GaussianFromMeanAndVariance(score, 0.1) > 0);
// Close the evidence block
block.CloseBlock();
// The basis
Vector[] basis = new Vector[]
{
Vector.FromArray(new double[2] {
0.2, 0.2
}),
Vector.FromArray(new double[2] {
0.2, 0.8
}),
Vector.FromArray(new double[2] {
0.8, 0.2
}),
Vector.FromArray(new double[2] {
0.8, 0.8
})
};
for (int trial = 0; trial < 3; trial++)
{
// The kernel
IKernelFunction kf;
if (trial == 0)
{
kf = new SquaredExponential(-0.0);
}
else if (trial == 1)
{
kf = new SquaredExponential(-0.5);
}
else
{
kf = new NNKernel(new double[] { 0.0, 0.0 }, -1.0);
}
// Fill in the sparse GP prior
GaussianProcess gp = new GaussianProcess(new ConstantFunction(0), kf);
prior.ObservedValue = new SparseGP(new SparseGPFixed(gp, basis));
// Model score
double NNscore = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("{0} evidence = {1}", kf, NNscore.ToString("g4"));
}
// Infer the posterior Sparse GP
SparseGP sgp = engine.Infer <SparseGP>(f);
// Check that training set is classified correctly
Console.WriteLine("");
Console.WriteLine("Predictions on training set:");
for (int i = 0; i < outputs.Length; i++)
{
Gaussian post = sgp.Marginal(inputs[i]);
double postMean = post.GetMean();
string comment = (outputs[i] == (postMean > 0.0)) ? "correct" : "incorrect";
Console.WriteLine("f({0}) = {1} ({2})", inputs[i], post, comment);
}
}
internal void BetaRegression()
{
int P = 8;
double[] b = new double[P];
for (int p = 0; p < P; p++)
{
b[p] = Rand.Beta(1, 1);
}
int N = 100;
double[][] X = new double[N][];
//Gaussian[][] softX = new Gaussian[N][];
double[] Y = new double[N];
for (int n = 0; n < N; n++)
{
X[n] = new double[P];
//softX[n] = new Gaussian[P];
Y[n] = 0;
for (int p = 0; p < P; p++)
{
X[n][p] = Rand.Normal();
//softX[n][p] = new Gaussian(X[n][p], 1e-4);
Y[n] += X[n][p] * b[p];
}
}
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Range dim = new Range(P).Named("P");
Range item = new Range(N).Named("N");
VariableArray <double> w = Variable.Array <double>(dim).Named("w");
w[dim] = Variable.Beta(1, 1).ForEach(dim);
var x = Variable.Array(Variable.Array <double>(dim), item).Named("x");
var softXvar = Variable.Array(Variable.Array <double>(dim), item).Named("softx");
softXvar.ObservedValue = X;
x[item][dim] = Variable.GaussianFromMeanAndPrecision(softXvar[item][dim], 1e4);
var wx = Variable.Array(Variable.Array <double>(dim), item).Named("wx");
wx[item][dim] = x[item][dim] * w[dim];
var sum = Variable.Array <double>(item).Named("sum");
sum[item] = Variable.Sum(wx[item]);
var prec = Variable.GammaFromShapeAndRate(.1, .1).Named("Noise");
var y = Variable.Array <double>(item).Named("y");
y[item] = Variable.GaussianFromMeanAndPrecision(sum[item], prec);
block.CloseBlock();
y.ObservedValue = Y;
InferenceEngine engine = new InferenceEngine(new VariationalMessagePassing());
var ca = engine.GetCompiledInferenceAlgorithm(evidence, w);
ca.Reset();
double oldLogEvidence = double.NegativeInfinity;
for (int i = 0; i < 1000; i++)
{
ca.Update(1);
double logEvidence1 = ca.Marginal <Bernoulli>(evidence.NameInGeneratedCode).LogOdds;
Console.WriteLine(logEvidence1);
if (i > 20 && System.Math.Abs(logEvidence1 - oldLogEvidence) < 0.01)
{
break;
}
oldLogEvidence = logEvidence1;
}
DistributionArray <Beta> wInferred = ca.Marginal <DistributionArray <Beta> >(w.NameInGeneratedCode);
for (int p = 0; p < P; p++)
{
Console.WriteLine("w[{0}] = {1} +/- {2} should be {3}",
p, wInferred[p].GetMean(), System.Math.Sqrt(wInferred[p].GetVariance()), b[p]);
}
}
public void GatedOutcomeAreEqualTest()
{
foreach (var algorithm in new Models.Attributes.IAlgorithm[] { new ExpectationPropagation(), new VariationalMessagePassing() })
{
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Vector priorA = Vector.FromArray(0.1, 0.9);
Vector priorB = Vector.FromArray(0.2, 0.8);
Variable <Outcome> a = Variable.EnumDiscrete <Outcome>(priorA).Named("a");
Variable <Outcome> b = Variable.EnumDiscrete <Outcome>(priorB).Named("b");
Variable <bool> c = (a == b).Named("c");
double priorC = 0.3;
Variable.ConstrainEqualRandom(c, new Bernoulli(priorC));
block.CloseBlock();
InferenceEngine engine = new InferenceEngine(algorithm);
double probEqual = priorA.Inner(priorB);
double evPrior = 0;
for (int atrial = 0; atrial < 2; atrial++)
{
if (atrial == 1)
{
a.ObservedValue = Outcome.Bad;
probEqual = priorB[1];
c.ClearObservedValue();
evPrior = System.Math.Log(priorA[1]);
priorA[0] = 0.0;
priorA[1] = 1.0;
}
double evExpected = System.Math.Log(probEqual * priorC + (1 - probEqual) * (1 - priorC)) + evPrior;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
if (algorithm is ExpectationPropagation || atrial == 1)
{
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-5) < 1e-5);
}
Bernoulli cExpected = new Bernoulli(probEqual * priorC / (probEqual * priorC + (1 - probEqual) * (1 - priorC)));
Bernoulli cActual = engine.Infer <Bernoulli>(c);
Console.WriteLine("c = {0} should be {1}", cActual, cExpected);
if (algorithm is ExpectationPropagation || atrial == 1)
{
Assert.True(cExpected.MaxDiff(cActual) < 1e-10);
}
Vector postB = Vector.Zero(2);
postB[0] = priorB[0] * (priorA[0] * priorC + priorA[1] * (1 - priorC));
postB[1] = priorB[1] * (priorA[1] * priorC + priorA[0] * (1 - priorC));
postB.Scale(1.0 / postB.Sum());
DiscreteEnum <Outcome> bExpected = new DiscreteEnum <Outcome>(postB);
DiscreteEnum <Outcome> bActual = engine.Infer <DiscreteEnum <Outcome> >(b);
Console.WriteLine("b = {0} should be {1}", bActual, bExpected);
if (algorithm is ExpectationPropagation || atrial == 1)
{
Assert.True(bExpected.MaxDiff(bActual) < 1e-10);
}
if (atrial == 0 && algorithm is VariationalMessagePassing)
{
continue;
}
for (int trial = 0; trial < 2; trial++)
{
if (trial == 0)
{
c.ObservedValue = true;
evExpected = System.Math.Log(probEqual * priorC) + evPrior;
}
else
{
c.ObservedValue = false;
evExpected = System.Math.Log((1 - probEqual) * (1 - priorC)) + evPrior;
}
evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-5) < 1e-5);
if (a.IsObserved)
{
Outcome flip(Outcome x) => (x == Outcome.Good ? Outcome.Bad : Outcome.Good);
bExpected = DiscreteEnum <Outcome> .PointMass(c.ObservedValue?a.ObservedValue : flip(a.ObservedValue));
}
else
{
postB[0] = priorB[0] * (c.ObservedValue ? priorA[0] : priorA[1]);
postB[1] = priorB[1] * (c.ObservedValue ? priorA[1] : priorA[0]);
postB.Scale(1.0 / postB.Sum());
bExpected = new DiscreteEnum <Outcome>(postB);
}
bActual = engine.Infer <DiscreteEnum <Outcome> >(b);
Console.WriteLine("b = {0} should be {1}", bActual, bExpected);
Assert.True(bExpected.MaxDiff(bActual) < 1e-10);
}
}
}
}
public void GPClassificationTest()
{
bool[] yData = new bool[] { false, false, false, true, true, true, true, false, false, false };
double[] xData = new double[]
{
-2, -1.555555555555556, -1.111111111111111, -0.6666666666666667, -0.2222222222222223, 0.2222222222222223, 0.6666666666666665, 1.111111111111111,
1.555555555555555, 2
};
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { xVec[1], xVec[4], xVec[8] };
//basis = xVec;
IKernelFunction kf = new SquaredExponential(0.0);
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <bool> y = Variable.Array <bool>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = (h[item] > 0);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { -1.410457563120709, 1.521306076273262, -1.008600221619413 });
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
double[] xTest = new double[]
{
-2, -1, 0.0
};
Vector[] xTestVec = Array.ConvertAll(xTest, v => Vector.Constant(1, v));
// computed by matlab/MNT/GP/test_gpc.m
double[] yMeanTest = new double[]
{
-0.966351175090184, -0.123034591744284, 0.762757400008960
};
double[] yVarTest = new double[]
{
0.323871157983366, 0.164009511251333, 0.162068482365962
};
for (int i = 0; i < xTestVec.Length; i++)
{
Gaussian pred = sgp.Marginal(xTestVec[i]);
Gaussian predExpected = new Gaussian(yMeanTest[i], yVarTest[i]);
Console.WriteLine("f({0}) = {1} should be {2}", xTest[i], pred, predExpected);
Assert.True(predExpected.MaxDiff(pred) < 1e-4);
}
double evExpected = -4.907121241357144;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-6) < 1e-4);
}
public void GPClassificationExample()
{
// The data
bool[] ydata = { true, true, false, true, false, false };
Vector[] xdata = new Vector[]
{
Vector.FromArray(new double[2] {
0, 0
}),
Vector.FromArray(new double[2] {
0, 1
}),
Vector.FromArray(new double[2] {
1, 0
}),
Vector.FromArray(new double[2] {
0, 0.5
}),
Vector.FromArray(new double[2] {
1.5, 0
}),
Vector.FromArray(new double[2] {
0.5, 1.0
})
};
// Open an evidence block to allow model scoring
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
// Set up the GP prior, which will be filled in later
Variable <SparseGP> prior = Variable.New <SparseGP>().Named("prior");
// The sparse GP variable - a distribution over functions
Variable <IFunction> f = Variable <IFunction> .Random(prior).Named("f");
// The locations to evaluate the function
VariableArray <Vector> x = Variable.Constant(xdata).Named("x");
Range j = x.Range.Named("j");
// The observation model
VariableArray <bool> y = Variable.Array <bool>(j).Named("y");
y[j] = (Variable.GaussianFromMeanAndVariance(Variable.FunctionEvaluate(f, x[j]), 0.1) > 0);
// Attach the observations
y.ObservedValue = ydata;
// Close the evidence block
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
// The basis
Vector[] basis = new Vector[]
{
Vector.FromArray(new double[2] {
0.2, 0.2
}),
Vector.FromArray(new double[2] {
0.2, 0.8
}),
Vector.FromArray(new double[2] {
0.8, 0.2
}),
Vector.FromArray(new double[2] {
0.8, 0.8
})
};
for (int trial = 0; trial < 3; trial++)
{
// The kernel
IKernelFunction kf;
if (trial == 0)
{
kf = new SquaredExponential(-0.0);
//kf = new LinearKernel(new double[] { 0.0, 0.0 });
}
else if (trial == 1)
{
kf = new SquaredExponential(-0.5);
}
else
{
kf = new NNKernel(new double[] { 0.0, 0.0 }, -1.0);
}
// Fill in the sparse GP prior
prior.ObservedValue = new SparseGP(new SparseGPFixed(kf, basis));
// Model score
double NNscore = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("{0} evidence = {1}", kf, NNscore.ToString("g4"));
}
// Infer the posterior Sparse GP
SparseGP sgp = engine.Infer <SparseGP>(f);
// Check that training set is classified correctly
Console.WriteLine();
Console.WriteLine("Predictions on training set:");
for (int i = 0; i < ydata.Length; i++)
{
Gaussian post = sgp.Marginal(xdata[i]);
double postMean = post.GetMean();
string comment = (ydata[i] == (postMean > 0.0)) ? "correct" : "incorrect";
Console.WriteLine("f({0}) = {1} ({2})", xdata[i], post, comment);
Assert.True(ydata[i] == (postMean > 0.0));
}
}
public int MulticlassRegression(Vector[] xObs, int[] yObs, int C, out VectorGaussian[] bPost, out Gaussian[] meanPost, out double lowerBound, object softmaxOperator,
bool trackLowerBound = false)
{
int N = xObs.Length;
int K = xObs[0].Count;
var c = new Range(C).Named("c");
var n = new Range(N).Named("n");
Variable <bool> ev = null;
IfBlock model = null;
if (trackLowerBound)
{
ev = Variable.Bernoulli(0.5).Named("evidence");
model = Variable.If(ev);
}
// model
var B = Variable.Array <Vector>(c).Named("coefficients");
B[c] = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(K), PositiveDefiniteMatrix.Identity(K)).ForEach(c);
var m = Variable.Array <double>(c).Named("mean");
m[c] = Variable.GaussianFromMeanAndPrecision(0, 1).ForEach(c);
Variable.ConstrainEqualRandom(B[C - 1], VectorGaussian.PointMass(Vector.Zero(K)));
Variable.ConstrainEqualRandom(m[C - 1], Gaussian.PointMass(0));
var x = Variable.Array <Vector>(n);
x.ObservedValue = xObs;
var yData = Variable.Array <int>(n);
yData.ObservedValue = yObs;
var g = Variable.Array(Variable.Array <double>(c), n);
g[n][c] = Variable.InnerProduct(B[c], x[n]) + m[c];
var p = Variable.Array <Vector>(n);
p[n] = Variable.Softmax(g[n]);
using (Variable.ForEach(n))
yData[n] = Variable.Discrete(p[n]);
if (trackLowerBound)
{
model.CloseBlock();
}
// inference
var ie = new InferenceEngine(new VariationalMessagePassing());
ie.Compiler.GivePriorityTo(softmaxOperator);
var ca = ie.GetCompiledInferenceAlgorithm(ev, B, m);
var start = DateTime.Now;
var initBound = double.NegativeInfinity;
int i = 0;
lowerBound = 0;
for (i = 1; i <= 50; i++)
{
ca.Update(1);
lowerBound = ca.Marginal <Bernoulli>(ev.NameInGeneratedCode).LogOdds;
Console.WriteLine(i + "," + lowerBound + "," + (DateTime.Now - start).TotalMilliseconds);
if (System.Math.Abs(initBound - lowerBound) < 1e-2)
{
break;
}
initBound = lowerBound;
start = DateTime.Now;
}
bPost = ca.Marginal <VectorGaussian[]>(B.NameInGeneratedCode);
meanPost = ca.Marginal <Gaussian[]>(m.NameInGeneratedCode);
return(i);
}
/// <summary>
/// Build and run a multinomial regression model.
/// </summary>
/// <param name="xObs">An array of vectors of observed inputs.
/// The length of the array is the number of samples, and the
/// length of the vectors is the number of input features. </param>
/// <param name="yObs">An array of array of counts, where the first index is the sample,
/// and the second index is the class. </param>
/// <param name="weightsPost">The returned posterior over the coefficients.</param>
/// <param name="biasPost">The returned posterior over the means.</param>
public double MultinomialRegression(Vector[] xObs, int[][] yObs, out IList <VectorGaussian> weightsPost, out IList <Gaussian> biasPost, bool trackLowerBound = true)
{
int nClasses = yObs[0].Length;
int nPoints = xObs.Length;
int dimension = xObs[0].Count;
var c = new Range(nClasses).Named("c");
var n = new Range(nPoints).Named("n");
Variable <bool> ev = null;
IfBlock model = null;
if (trackLowerBound)
{
ev = Variable.Bernoulli(0.5).Named("evidence");
model = Variable.If(ev);
}
// model
var weights = Variable.Array <Vector>(c).Named("weights");
weights[c] = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(dimension), PositiveDefiniteMatrix.Identity(dimension)).ForEach(c);
var bias = Variable.Array <double>(c).Named("bias");
bias[c] = Variable.GaussianFromMeanAndPrecision(0, 1).ForEach(c);
Variable.ConstrainEqualRandom(weights[nClasses - 1], VectorGaussian.PointMass(Vector.Zero(dimension)));
Variable.ConstrainEqualRandom(bias[nClasses - 1], Gaussian.PointMass(0));
var x = Variable.Array <Vector>(n).Named("x");
x.ObservedValue = xObs;
var yData = Variable.Array(Variable.Array <int>(c), n).Named("y");
yData.ObservedValue = yObs;
var trialCounts = Variable.Array <int>(n).Named("trialCounts");
trialCounts.ObservedValue = yObs.Select(o => o.Sum()).ToArray();
var g = Variable.Array(Variable.Array <double>(c), n).Named("g");
g[n][c] = Variable.InnerProduct(weights[c], x[n]) + bias[c];
var p = Variable.Array <Vector>(n).Named("p");
p[n] = Variable.Softmax(g[n]);
using (Variable.ForEach(n))
yData[n] = Variable.Multinomial(trialCounts[n], p[n]);
if (trackLowerBound)
{
model.CloseBlock();
}
// inference
var ie = new InferenceEngine(new VariationalMessagePassing());
//ie.Compiler.GivePriorityTo(typeof(SaulJordanSoftmaxOp_LBFGS));
//ie.Compiler.GivePriorityTo(typeof(ProductOfLogisticsSoftmaxOp));
//ie.Compiler.GivePriorityTo(typeof(SaulJordanSoftmaxOp_NCVMP));
//ie.Compiler.GivePriorityTo(typeof(SoftmaxOp_Bouchard));
//ie.Compiler.GivePriorityTo(typeof(Blei06SoftmaxOp_NCVMP));
ie.OptimiseForVariables = (trackLowerBound ? new IVariable[] { weights, bias, ev } : new IVariable[] { weights, bias });
if (trackLowerBound)
{
ie.ShowProgress = false;
}
for (int iter = 1; iter <= 50; iter++)
{
ie.NumberOfIterations = iter;
if (trackLowerBound)
{
Console.WriteLine(ie.Infer <Bernoulli>(ev).LogOdds);
}
}
weightsPost = ie.Infer <IList <VectorGaussian> >(weights);
biasPost = ie.Infer <IList <Gaussian> >(bias);
return(trackLowerBound ? ie.Infer <Bernoulli>(ev).LogOdds : 0.0);
}
public void GPRegressionTest()
{
double[] yData = new double[]
{
-0.06416828853982412, -0.6799959810206935, -0.4541652863622044, 0.155770359928991, 1.036659040456137, 0.7353821980830825, 0.8996680933259047,
-0.05368704705684217, -0.7905775695015919, -0.1436284683992815
};
double[] xData = new double[]
{
-2, -1.555555555555556, -1.111111111111111, -0.6666666666666667, -0.2222222222222223, 0.2222222222222223, 0.6666666666666665, 1.111111111111111,
1.555555555555555, 2
};
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { xVec[1], xVec[4], xVec[8] };
IKernelFunction kf = new SquaredExponential(System.Math.Log(2.0));
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <double> y = Variable.Array <double>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = Variable.GaussianFromMeanAndVariance(h[item], 0.1);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { -3.250044160725389, 4.579296091435270, -2.227005562666341 });
PositiveDefiniteMatrix betaExpected = new PositiveDefiniteMatrix(new double[, ]
{
{ 3.187555652658986, -3.301824438047169, 1.227566907279797 },
{ -3.30182443804717, 5.115027119603418, -2.373085083966294 },
{ 1.227566907279797, -2.373085083966294, 2.156308696222915 }
});
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
Console.WriteLine(StringUtil.JoinColumns("beta = ", sgp.Beta, " should be ", betaExpected));
double[] xTest = new double[]
{
-2, -1, 0.0
};
Vector[] xTestVec = Array.ConvertAll(xTest, v => Vector.Constant(1, v));
// computed by matlab/MNT/GP/test_gpr.m
double[] yMeanTest = new double[]
{
-0.544583265595561, 0.134323399801302, 0.503623822120711
};
double[] yVarTest = new double[]
{
0.058569682375201, 0.022695532903985, 0.024439582002951
};
for (int i = 0; i < xTestVec.Length; i++)
{
Gaussian pred = sgp.Marginal(xTestVec[i]);
Gaussian predExpected = new Gaussian(yMeanTest[i], yVarTest[i]);
Console.WriteLine("f({0}) = {1} should be {2}", xTest[i], pred, predExpected);
Assert.True(predExpected.MaxDiff(pred) < 1e-4);
}
double evExpected = -13.201173794945003;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-6) < 1e-4);
}