private void SparseGPModel()
{
// Basis Vector
Vector[] basis = new Vector[]
{
Vector.FromArray(new double[2] {
0.2, 0.2
}),
Vector.FromArray(new double[2] {
0.2, 0.8
}),
Vector.FromArray(new double[2] {
0.8, 0.2
}),
Vector.FromArray(new double[2] {
0.8, 0.8
})
};
// The kernel
double[] kp = new double[2];
kp[0] = 0.0;
kp[1] = 0.0;
IKernelFunction kf = new NNKernel(kp, 0.0);
// The fixed parameters
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
SparseGP agp = new SparseGP(sgpf);
IFunction a = Factor.Random(agp);
Vector b = Vector.FromArray(0.1, 0.2);
double c = Factor.FunctionEvaluate(a, b);
InferNet.Infer(c, nameof(c));
}
public void GPClassificationTest4()
{
bool[] yData = new bool[] { false, false, true, false };
double[] xData = new double[]
{
-1.555555555555556, -1.111111111111111, -0.2222222222222223, 1.555555555555555
};
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { Vector.Zero(1) };
IKernelFunction kf = new SquaredExponential(0.0);
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <bool> y = Variable.Array <bool>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = (h[item] > 0);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { 0.409693797629808 });
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
}
/// <summary>EP message to <c>func</c>.</summary>
/// <param name="y">Incoming message from <c>y</c>. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="func">Incoming message from <c>func</c>. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="x">Constant value for <c>x</c>.</param>
/// <param name="result">Modified to contain the outgoing message.</param>
/// <returns>
/// <paramref name="result" />
/// </returns>
/// <remarks>
/// <para>The outgoing message is a distribution matching the moments of <c>func</c> as the random arguments are varied. The formula is <c>proj[p(func) sum_(y) p(y) factor(y,func,x)]/p(func)</c>.</para>
/// </remarks>
/// <exception cref="ImproperMessageException">
/// <paramref name="y" /> is not a proper distribution.</exception>
/// <exception cref="ImproperMessageException">
/// <paramref name="func" /> is not a proper distribution.</exception>
public static SparseGP FuncAverageConditional([SkipIfUniform] Gaussian y, [SkipIfUniform] SparseGP func, Vector x, SparseGP result)
{
if (y.IsUniform() || func.IsUniform())
{
result.SetToUniform();
return(result);
}
result.FixedParameters = func.FixedParameters;
result.IncludePrior = false;
double vf = func.Variance(x);
double my, vy;
y.GetMeanAndVariance(out my, out vy);
Vector kbx = func.FixedParameters.KernelOf_X_B(x);
Vector proj = func.FixedParameters.InvKernelOf_B_B * kbx;
double prec = 1.0 / (vy + vf - func.Var_B_B.QuadraticForm(proj));
result.InducingDist.Precision.SetToOuter(proj, proj);
result.InducingDist.Precision.Scale(prec);
result.InducingDist.MeanTimesPrecision.SetTo(proj);
result.InducingDist.MeanTimesPrecision.Scale(prec * my);
result.ClearCachedValues();
return(result);
}
public void GPClassificationTest2()
{
bool[] yData = new bool[] { false, true, false };
double[] xData = new double[]
{
-1.555555555555556, -0.2222222222222223, 1.555555555555555
};
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { Vector.Zero(1) };
IKernelFunction kf = new SquaredExponential(0.0);
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <bool> y = Variable.Array <bool>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = (h[item] > 0);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { 0.573337393823702 });
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
double[] xTest = new double[]
{
-2, -1, 0.0
};
Vector[] xTestVec = Array.ConvertAll(xTest, v => Vector.Constant(1, v));
double[] yMeanTest = new double[]
{
0.077592778583272, 0.347746707713812, 0.573337393823702
};
double[] yVarTest = new double[]
{
0.986784459962251, 0.734558782611933, 0.278455962249970
};
for (int i = 0; i < xTestVec.Length; i++)
{
Gaussian pred = sgp.Marginal(xTestVec[i]);
Gaussian predExpected = new Gaussian(yMeanTest[i], yVarTest[i]);
Console.WriteLine("f({0}) = {1} should be {2}", xTest[i], pred, predExpected);
Assert.True(predExpected.MaxDiff(pred) < 1e-4);
}
double evExpected = -2.463679892165236;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-6) < 1e-4);
}
public void GPClassificationTest1()
{
bool[] yData = new bool[] { true };
double[] xData = new double[] { -0.2222222222222223 };
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { Vector.Zero(1) };
IKernelFunction kf = new SquaredExponential(0.0);
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <bool> y = Variable.Array <bool>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = (h[item] > 0);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { 0.778424938343491 });
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
double[] xTest = new double[]
{
-2, -1, 0.0
};
Vector[] xTestVec = Array.ConvertAll(xTest, v => Vector.Constant(1, v));
double[] yMeanTest = new double[]
{
0.105348359509159, 0.472138591390244, 0.778424938343491
};
double[] yVarTest = new double[]
{
0.988901723148729, 0.777085150520037, 0.394054615364932
};
for (int i = 0; i < xTestVec.Length; i++)
{
Gaussian pred = sgp.Marginal(xTestVec[i]);
Gaussian predExpected = new Gaussian(yMeanTest[i], yVarTest[i]);
Console.WriteLine("f({0}) = {1} should be {2}", xTest[i], pred, predExpected);
Assert.True(predExpected.MaxDiff(pred) < 1e-4);
}
double evExpected = -0.693147180559945;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-6) < 1e-4);
}
public void MP_GetItemSparseGP()
{
InferenceEngine engine = new InferenceEngine();
ModelDefinitionMethod meth = new ModelDefinitionMethod(GetItemSparseGPModel);
var ca = engine.Compiler.CompileWithoutParams(
declaringType, meth.Method, new AttributeRegistry <object, ICompilerAttribute>(true));
ca.Execute(1);
SparseGP cMarg = ca.Marginal <SparseGP>("c");
DistributionArray <SparseGP> dMarg = ca.Marginal <DistributionArray <SparseGP> >("d");
}
public static (Vector[] dataX, double[] dataY) GenerateRandomData(int numData, double proportionCorrupt)
{
int randomSeed = 9876;
Random rng = new Random(randomSeed);
Rand.Restart(randomSeed);
InferenceEngine engine = Utilities.GetInferenceEngine();
// The points to evaluate
Vector[] randomInputs = Utilities.VectorRange(0, 1, numData, null);
var gaussianProcessGenerator = new GaussianProcessRegressor(randomInputs);
// The basis
Vector[] basis = Utilities.VectorRange(0, 1, 6, rng);
// The kernel
var kf = new SummationKernel(new SquaredExponential(-1)) + new WhiteNoise();
// Fill in the sparse GP prior
GaussianProcess gp = new GaussianProcess(new ConstantFunction(0), kf);
gaussianProcessGenerator.Prior.ObservedValue = new SparseGP(new SparseGPFixed(gp, basis));
// Infer the posterior Sparse GP, and sample a random function from it
SparseGP sgp = engine.Infer <SparseGP>(gaussianProcessGenerator.F);
var randomFunc = sgp.Sample();
double[] randomOutputs = new double[randomInputs.Length];
int numCorrupted = (int)Math.Ceiling(numData * proportionCorrupt);
var subset = Enumerable.Range(0, randomInputs.Length + 1).OrderBy(x => rng.Next()).Take(numCorrupted);
// get random data
for (int i = 0; i < randomInputs.Length; i++)
{
double post = randomFunc.Evaluate(randomInputs[i]);
// corrupt data point if it we haven't exceed the proportion we wish to corrupt
if (subset.Contains(i))
{
double sign = rng.NextDouble() > 0.5 ? 1 : -1;
double distance = rng.NextDouble() * 1;
post = (sign * distance) + post;
}
randomOutputs[i] = post;
}
Console.WriteLine("Model complete: Generated {0} points with {1} corrupted", numData, numCorrupted);
return(randomInputs, randomOutputs);
}
public void SparseGPTest()
{
// The basis
Vector[] basis = new Vector[]
{
Vector.FromArray(new double[2] {
0.2, 0.2
}),
Vector.FromArray(new double[2] {
0.2, 0.8
}),
Vector.FromArray(new double[2] {
0.8, 0.2
}),
Vector.FromArray(new double[2] {
0.8, 0.8
})
};
// The kernel
IKernelFunction kf = new NNKernel(new double[] { 0.0, 0.0 }, 0.0);
// The fixed parameters
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
// Alpha and beta
Vector alpha = Vector.Zero(basis.Length);
PositiveDefiniteMatrix beta = new PositiveDefiniteMatrix(basis.Length, basis.Length);
for (int i = 0; i < alpha.Count; i++)
{
alpha[i] = i;
for (int j = 0; j < alpha.Count; j++)
{
beta[i, j] = alpha.Count - System.Math.Abs(i - j);
}
}
SparseGP a = new SparseGP(sgpf);
#if false
Rank1Pot r1p = new Rank1Pot();
r1p.Xi = basis[0]; // must be a basis point for this test
r1p.Yi = 2.2;
r1p.LambdaInv = 0.7;
SparseGP b = new SparseGP(sgpf, r1p);
DistributionTests.SettableToRatioTest(a, b);
#endif
DistributionTests.ProductWithUniformTest(a);
DistributionTests.RatioWithUniformTest(a);
DistributionTests.SettableToTest(a);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="SparseGPOp"]/message_doc[@name="FuncAverageConditional(Gaussian, SparseGP, Vector, SparseGP)"]/*'/>
public static SparseGP FuncAverageConditional([SkipIfUniform] Gaussian y, [SkipIfUniform] SparseGP func, Vector x, SparseGP result)
{
if (y.IsUniform() || func.IsUniform())
{
result.SetToUniform();
return(result);
}
result.FixedParameters = func.FixedParameters;
result.IncludePrior = false;
Vector kbx = func.FixedParameters.KernelOf_X_B(x);
Vector proj = func.FixedParameters.InvKernelOf_B_B * kbx;
// To avoid computing Var_B_B:
// vf - func.Var_B_B.QuadraticForm(proj) = kxx - kbx'*beta*kbx - kbx'*inv(K)*Var_B_B*inv(K)*kbx
// = kxx - kbx'*(beta + inv(K)*Var_B_B*inv(K))*kbx
// = kxx - kbx'*(beta + inv(K)*(K - K*Beta*K)*inv(K))*kbx
// = kxx - kbx'*inv(K)*kbx
// Since Var_B_B = K - K*Beta*K
double kxx = func.FixedParameters.Prior.Variance(x);
if (y.Precision == 0)
{
result.InducingDist.Precision.SetAllElementsTo(0);
result.InducingDist.MeanTimesPrecision.SetTo(proj);
result.InducingDist.MeanTimesPrecision.Scale(y.MeanTimesPrecision);
}
else
{
double my, vy;
y.GetMeanAndVariance(out my, out vy);
double prec = 1.0 / (vy + kxx - kbx.Inner(proj));
//Console.WriteLine($"{vf - func.Var_B_B.QuadraticForm(proj)} {func.FixedParameters.Prior.Variance(x) - func.FixedParameters.InvKernelOf_B_B.QuadraticForm(kbx)}");
if (prec > double.MaxValue || prec < 0)
{
int i = proj.IndexOfMaximum();
result.InducingDist.Precision.SetAllElementsTo(0);
result.InducingDist.Precision[i, i] = double.PositiveInfinity;
result.InducingDist.MeanTimesPrecision.SetAllElementsTo(0);
result.InducingDist.MeanTimesPrecision[i] = my;
}
else
{
result.InducingDist.Precision.SetToOuter(proj, proj);
result.InducingDist.Precision.Scale(prec);
result.InducingDist.MeanTimesPrecision.SetTo(proj);
result.InducingDist.MeanTimesPrecision.Scale(prec * my);
}
}
result.ClearCachedValues();
return(result);
}
public void GPRegressionTest1()
{
double[] yData = new double[] { 1.036659040456137 };
double[] xData = new double[] { -0.2222222222222223 };
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { xVec[0] };
IKernelFunction kf = new SquaredExponential(System.Math.Log(2.0));
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <double> y = Variable.Array <double>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = Variable.GaussianFromMeanAndVariance(h[item], 0.1);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { 0.942417309505579 });
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
double[] xTest = new double[]
{
-2, -1, 0.0
};
Vector[] xTestVec = Array.ConvertAll(xTest, v => Vector.Constant(1, v));
// computed by matlab/MNT/GP/test_gpr.m
double[] yTest = new double[]
{
0.634848540665472, 0.873781982196160, 0.936617836728720
};
for (int i = 0; i < xTestVec.Length; i++)
{
double pred = sgp.Mean(xTestVec[i]);
Console.WriteLine("Ef({0}) = {1} should be {2}", xTest[i], pred, yTest[i]);
Assert.True(MMath.AbsDiff(pred, yTest[i], 1e-6) < 1e-4);
}
double evExpected = -1.455076334997490;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-6) < 1e-4);
}
static void FitDataset(bool useSynthetic)
{
Vector[] trainingInputs;
double[] trainingOutputs;
if (!useSynthetic)
{
var trainingData = Utilities.LoadAISDataset();
trainingInputs = trainingData.Select(tup => Vector.FromArray(new double[1] {
tup.x
})).ToArray();
trainingOutputs = trainingData.Select(tup => tup.y).ToArray();
}
else
{
(trainingInputs, trainingOutputs) = GaussianProcessDataGenerator.GenerateRandomData(30, 0.3);
}
InferenceEngine engine = Utilities.GetInferenceEngine();
// First fit standard GP, then fit Student-T GP
foreach (var useStudentTLikelihood in new[] { false, true })
{
var gaussianProcessRegressor = new GaussianProcessRegressor(trainingInputs, useStudentTLikelihood, trainingOutputs);
// Log length scale estimated as -1
var noiseVariance = 0.8;
var kf = new SummationKernel(new SquaredExponential(-1)) + new WhiteNoise(Math.Log(noiseVariance) / 2);
GaussianProcess gp = new GaussianProcess(new ConstantFunction(0), kf);
// Convert SparseGP to full Gaussian Process by evaluating at all the training points
gaussianProcessRegressor.Prior.ObservedValue = new SparseGP(new SparseGPFixed(gp, trainingInputs.ToArray()));
double logOdds = engine.Infer <Bernoulli>(gaussianProcessRegressor.Evidence).LogOdds;
Console.WriteLine("{0} evidence = {1}", kf, logOdds.ToString("g4"));
// Infer the posterior Sparse GP
SparseGP sgp = engine.Infer <SparseGP>(gaussianProcessRegressor.F);
#if NETFULL
string datasetName = useSynthetic ? "Synthetic" : "AIS";
Utilities.PlotPredictions(sgp, trainingInputs, trainingOutputs, useStudentTLikelihood, datasetName);
#endif
}
}
private void GetItemSparseGPModel()
{
// Basis Vector
Vector[] basis = new Vector[]
{
Vector.FromArray(new double[2] {
0.2, 0.2
}),
Vector.FromArray(new double[2] {
0.2, 0.8
}),
Vector.FromArray(new double[2] {
0.8, 0.2
}),
Vector.FromArray(new double[2] {
0.8, 0.8
})
};
// The kernel
double[] kp = new double[2];
kp[0] = 0.0;
kp[1] = 0.0;
IKernelFunction kf = new NNKernel(kp, 0.0);
// The fixed parameters
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
SparseGP sgp = new SparseGP(sgpf);
IFunction[] array = new IFunction[3];
for (int i = 0; i < 3; i++)
{
array[i] = Factor.Random(sgp);
}
IFunction c = Factor.GetItem(array, 1);
IFunction[] d = new IFunction[2];
if (true) // preserve the previous assignment
{
d = Factor.GetItems(array, new int[] { 0, 2 });
}
InferNet.Infer(c, nameof(c));
InferNet.Infer(d, nameof(d));
}
/// <summary>
/// EP message to 'x'.
/// </summary>
/// <param name="y">Incoming message from 'y'.</param>
/// <param name="func">Incoming message from 'func'.</param>
/// <param name="x">Constant value for 'x'.</param>
/// <param name="result">Modified to contain the outgoing message.</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the integral of the factor times incoming messages, over all arguments except 'x'.
/// The formula is <c>int f(x,x) q(x) dx</c> where <c>x = (y,func)</c>.
/// </para></remarks>
public static VectorGaussian XAverageConditional(Gaussian y, SparseGP func, Vector x, VectorGaussian result)
{
// Doesn't matter what message we return as we are only supporting a point x
result.SetToUniform();
return(result);
}
/// <summary>EP message to <c>func</c>.</summary>
/// <param name="y">Constant value for <c>y</c>.</param>
/// <param name="func">Incoming message from <c>func</c>. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="x">Constant value for <c>x</c>.</param>
/// <param name="result">Modified to contain the outgoing message.</param>
/// <returns>
/// <paramref name="result" />
/// </returns>
/// <remarks>
/// <para>The outgoing message is the factor viewed as a function of <c>func</c> conditioned on the given values.</para>
/// </remarks>
/// <exception cref="ImproperMessageException">
/// <paramref name="func" /> is not a proper distribution.</exception>
public static SparseGP FuncAverageConditional(double y, [SkipIfUniform] SparseGP func, Vector x, SparseGP result)
{
return(FuncAverageConditional(Gaussian.PointMass(y), func, x, result));
}
public void Initialize()
{
// DO NOT make this a constructor, because it makes the test not notice complete lack of serialization as an empty object is set up exactly as the thing
// you are trying to deserialize.
this.pareto = new Pareto(1.2, 3.5);
this.poisson = new Poisson(2.3);
this.wishart = new Wishart(20, new PositiveDefiniteMatrix(new double[, ] {
{ 22, 21 }, { 21, 23 }
}));
this.vectorGaussian = new VectorGaussian(Vector.FromArray(13, 14), new PositiveDefiniteMatrix(new double[, ] {
{ 16, 15 }, { 15, 17 }
}));
this.unnormalizedDiscrete = UnnormalizedDiscrete.FromLogProbs(DenseVector.FromArray(5.1, 5.2, 5.3));
this.pointMass = PointMass <double> .Create(1.1);
this.gaussian = new Gaussian(11.0, 12.0);
this.nonconjugateGaussian = new NonconjugateGaussian(1.2, 2.3, 3.4, 4.5);
this.gamma = new Gamma(9.0, 10.0);
this.gammaPower = new GammaPower(5.6, 2.8, 3.4);
this.discrete = new Discrete(6.0, 7.0, 8.0);
this.conjugateDirichlet = new ConjugateDirichlet(1.2, 2.3, 3.4, 4.5);
this.dirichlet = new Dirichlet(3.0, 4.0, 5.0);
this.beta = new Beta(2.0, 1.0);
this.binomial = new Binomial(5, 0.8);
this.bernoulli = new Bernoulli(0.6);
this.sparseBernoulliList = SparseBernoulliList.Constant(4, new Bernoulli(0.1));
this.sparseBernoulliList[1] = new Bernoulli(0.9);
this.sparseBernoulliList[3] = new Bernoulli(0.7);
this.sparseBetaList = SparseBetaList.Constant(5, new Beta(2.0, 2.0));
this.sparseBetaList[0] = new Beta(3.0, 4.0);
this.sparseBetaList[1] = new Beta(5.0, 6.0);
this.sparseGaussianList = SparseGaussianList.Constant(6, Gaussian.FromMeanAndPrecision(0.1, 0.2));
this.sparseGaussianList[4] = Gaussian.FromMeanAndPrecision(0.3, 0.4);
this.sparseGaussianList[5] = Gaussian.FromMeanAndPrecision(0.5, 0.6);
this.sparseGammaList = SparseGammaList.Constant(1, Gamma.FromShapeAndRate(1.0, 2.0));
this.truncatedGamma = new TruncatedGamma(1.2, 2.3, 3.4, 4.5);
this.truncatedGaussian = new TruncatedGaussian(1.2, 3.4, 5.6, 7.8);
this.wrappedGaussian = new WrappedGaussian(1.2, 2.3, 3.4);
ga = Distribution <double> .Array(new[] { this.gaussian, this.gaussian });
vga = Distribution <Vector> .Array(new[] { this.vectorGaussian, this.vectorGaussian });
ga2D = Distribution <double> .Array(new[, ] {
{ this.gaussian, this.gaussian }, { this.gaussian, this.gaussian }
});
vga2D = Distribution <Vector> .Array(new[, ] {
{ this.vectorGaussian, this.vectorGaussian }, { this.vectorGaussian, this.vectorGaussian }
});
gaJ = Distribution <double> .Array(new[] { new[] { this.gaussian, this.gaussian }, new[] { this.gaussian, this.gaussian } });
vgaJ = Distribution <Vector> .Array(new[] { new[] { this.vectorGaussian, this.vectorGaussian }, new[] { this.vectorGaussian, this.vectorGaussian } });
var gp = new GaussianProcess(new ConstantFunction(0), new SquaredExponential(0));
var basis = Util.ArrayInit(2, i => Vector.FromArray(1.0 * i));
this.sparseGp = new SparseGP(new SparseGPFixed(gp, basis));
this.quantileEstimator = new QuantileEstimator(0.01);
this.quantileEstimator.Add(5);
}
/// <summary>EP message to <c>y</c>.</summary>
/// <param name="func">Incoming message from <c>func</c>. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="x">Constant value for <c>x</c>.</param>
/// <returns>The outgoing EP message to the <c>y</c> argument.</returns>
/// <remarks>
/// <para>The outgoing message is a distribution matching the moments of <c>y</c> as the random arguments are varied. The formula is <c>proj[p(y) sum_(func) p(func) factor(y,func,x)]/p(y)</c>.</para>
/// </remarks>
/// <exception cref="ImproperMessageException">
/// <paramref name="func" /> is not a proper distribution.</exception>
public static Gaussian YAverageConditional([SkipIfUniform] SparseGP func, Vector x)
{
return(func.Marginal(x));
}
internal void HeteroscedasticGPR()
{
// This model is based on the paper "Most Likely Heteroscedastic Gaussian Process Regression" by Kersting et al, ICML 2007
// Silverman's motorcycle benchmark dataset
double[] inputs = new double[]
{
2.4, 2.6, 3.2, 3.6, 4, 6.2, 6.6, 6.8, 7.8, 8.199999999999999, 8.800000000000001, 8.800000000000001,
9.6, 10, 10.2, 10.6, 11, 11.4, 13.2, 13.6, 13.8, 14.6, 14.6, 14.6, 14.6, 14.6, 14.6, 14.8, 15.4, 15.4,
15.4, 15.4, 15.6, 15.6, 15.8, 15.8, 16, 16, 16.2, 16.2, 16.2, 16.4, 16.4, 16.6, 16.8, 16.8, 16.8, 17.6,
17.6, 17.6, 17.6, 17.8, 17.8, 18.6, 18.6, 19.2, 19.4, 19.4, 19.6, 20.2, 20.4, 21.2, 21.4, 21.8, 22, 23.2,
23.4, 24, 24.2, 24.2, 24.6, 25, 25, 25.4, 25.4, 25.6, 26, 26.2, 26.2, 26.4, 27, 27.2, 27.2, 27.2, 27.6,
28.2, 28.4, 28.4, 28.6, 29.4, 30.2, 31, 31.2, 32, 32, 32.8, 33.4, 33.8, 34.4, 34.8, 35.2, 35.2, 35.4, 35.6,
35.6, 36.2, 36.2, 38, 38, 39.2, 39.4, 40, 40.4, 41.6, 41.6, 42.4, 42.8, 42.8, 43, 44, 44.4, 45, 46.6, 47.8,
47.8, 48.8, 50.6, 52, 53.2, 55, 55, 55.4, 57.6
};
double[] outputs = new double[]
{
0, -1.3, -2.7, 0, -2.7, -2.7, -2.7, -1.3, -2.7, -2.7, -1.3, -2.7, -2.7, -2.7, -5.4,
-2.7, -5.4, 0, -2.7, -2.7, 0, -13.3, -5.4, -5.4, -9.300000000000001, -16, -22.8, -2.7, -22.8, -32.1, -53.5,
-54.9, -40.2, -21.5, -21.5, -50.8, -42.9, -26.8, -21.5, -50.8, -61.7, -5.4, -80.40000000000001, -59, -71,
-91.09999999999999, -77.7, -37.5, -85.59999999999999, -123.1, -101.9, -99.09999999999999, -104.4, -112.5,
-50.8, -123.1, -85.59999999999999, -72.3, -127.2, -123.1, -117.9, -134, -101.9, -108.4, -123.1, -123.1, -128.5,
-112.5, -95.09999999999999, -81.8, -53.5, -64.40000000000001, -57.6, -72.3, -44.3, -26.8, -5.4, -107.1, -21.5,
-65.59999999999999, -16, -45.6, -24.2, 9.5, 4, 12, -21.5, 37.5, 46.9, -17.4, 36.2, 75, 8.1, 54.9, 48.2, 46.9,
16, 45.6, 1.3, 75, -16, -54.9, 69.59999999999999, 34.8, 32.1, -37.5, 22.8, 46.9, 10.7, 5.4, -1.3, -21.5, -13.3,
30.8, -10.7, 29.4, 0, -10.7, 14.7, -1.3, 0, 10.7, 10.7, -26.8, -14.7, -13.3, 0, 10.7, -14.7, -2.7, 10.7, -2.7, 10.7
};
Range j = new Range(inputs.Length);
Vector[] inputsVec = Util.ArrayInit(inputs.Length, i => Vector.FromArray(inputs[i]));
VariableArray <Vector> x = Variable.Observed(inputsVec, j).Named("x");
VariableArray <double> y = Variable.Observed(outputs, j).Named("y");
// Set up the GP prior, which will be filled in later
Variable <SparseGP> prior = Variable.New <SparseGP>().Named("prior");
Variable <SparseGP> prior2 = Variable.New <SparseGP>().Named("prior2");
// The sparse GP variable - a distribution over functions
Variable <IFunction> f = Variable <IFunction> .Random(prior).Named("f");
Variable <IFunction> r = Variable <IFunction> .Random(prior2).Named("r");
Variable <double> mean = Variable.FunctionEvaluate(f, x[j]).Named("mean");
Variable <double> logVariance = Variable.FunctionEvaluate(r, x[j]).Named("logVariance");
Variable <double> variance = Variable.Exp(logVariance);
y[j] = Variable.GaussianFromMeanAndVariance(mean, variance);
InferenceEngine engine = new InferenceEngine();
GaussianProcess gp = new GaussianProcess(new ConstantFunction(0), new SquaredExponential(0));
GaussianProcess gp2 = new GaussianProcess(new ConstantFunction(0), new SquaredExponential(0));
// Fill in the sparse GP prior
//Vector[] basis = Util.ArrayInit(120, i => Vector.FromArray(0.5*i));
Vector[] basis = Util.ArrayInit(60, i => Vector.FromArray(1.0 * i));
prior.ObservedValue = new SparseGP(new SparseGPFixed(gp, basis));
prior2.ObservedValue = new SparseGP(new SparseGPFixed(gp2, basis));
// Infer the posterior Sparse GP
SparseGP sgp = engine.Infer <SparseGP>(f);
// Check that training set is classified correctly
Console.WriteLine();
Console.WriteLine("Predictions on training set:");
for (int i = 0; i < outputs.Length; i++)
{
Gaussian post = sgp.Marginal(inputsVec[i]);
//double postMean = post.GetMean();
Console.WriteLine("f({0}) = {1}", inputs[i], post);
}
//TODO: change path for cross platform using
using (MatlabWriter writer = new MatlabWriter(@"..\..\HGPR.mat"))
{
int n = outputs.Length;
double[] m = new double[n];
double[] s = new double[n];
for (int i = 0; i < n; i++)
{
Gaussian post = sgp.Marginal(inputsVec[i]);
double mi, vi;
post.GetMeanAndVariance(out mi, out vi);
m[i] = mi;
s[i] = System.Math.Sqrt(vi);
}
writer.Write("mean", m);
writer.Write("std", s);
}
}
public void Initialize(bool skipStringDistributions = false)
{
// DO NOT make this a constructor, because it makes the test not notice complete lack of serialization as an empty object is set up exactly as the thing
// you are trying to deserialize.
this.pareto = new Pareto(1.2, 3.5);
this.poisson = new Poisson(2.3);
this.wishart = new Wishart(20, new PositiveDefiniteMatrix(new double[, ] {
{ 22, 21 }, { 21, 23 }
}));
this.vectorGaussian = new VectorGaussian(Vector.FromArray(13, 14), new PositiveDefiniteMatrix(new double[, ] {
{ 16, 15 }, { 15, 17 }
}));
this.unnormalizedDiscrete = UnnormalizedDiscrete.FromLogProbs(DenseVector.FromArray(5.1, 5.2, 5.3));
this.pointMass = PointMass <double> .Create(1.1);
this.gaussian = new Gaussian(11.0, 12.0);
this.nonconjugateGaussian = new NonconjugateGaussian(1.2, 2.3, 3.4, 4.5);
this.gamma = new Gamma(9.0, 10.0);
this.gammaPower = new GammaPower(5.6, 2.8, 3.4);
this.discrete = new Discrete(6.0, 7.0, 8.0);
this.conjugateDirichlet = new ConjugateDirichlet(1.2, 2.3, 3.4, 4.5);
this.dirichlet = new Dirichlet(3.0, 4.0, 5.0);
this.beta = new Beta(2.0, 1.0);
this.binomial = new Binomial(5, 0.8);
this.bernoulli = new Bernoulli(0.6);
this.sparseBernoulliList = SparseBernoulliList.Constant(4, new Bernoulli(0.1));
this.sparseBernoulliList[1] = new Bernoulli(0.9);
this.sparseBernoulliList[3] = new Bernoulli(0.7);
this.sparseBetaList = SparseBetaList.Constant(5, new Beta(2.0, 2.0));
this.sparseBetaList[0] = new Beta(3.0, 4.0);
this.sparseBetaList[1] = new Beta(5.0, 6.0);
this.sparseGaussianList = SparseGaussianList.Constant(6, Gaussian.FromMeanAndPrecision(0.1, 0.2));
this.sparseGaussianList[4] = Gaussian.FromMeanAndPrecision(0.3, 0.4);
this.sparseGaussianList[5] = Gaussian.FromMeanAndPrecision(0.5, 0.6);
this.sparseGammaList = SparseGammaList.Constant(1, Gamma.FromShapeAndRate(1.0, 2.0));
this.truncatedGamma = new TruncatedGamma(1.2, 2.3, 3.4, 4.5);
this.truncatedGaussian = new TruncatedGaussian(1.2, 3.4, 5.6, 7.8);
this.wrappedGaussian = new WrappedGaussian(1.2, 2.3, 3.4);
ga = Distribution <double> .Array(new[] { this.gaussian, this.gaussian });
vga = Distribution <Vector> .Array(new[] { this.vectorGaussian, this.vectorGaussian });
ga2D = Distribution <double> .Array(new[, ] {
{ this.gaussian, this.gaussian }, { this.gaussian, this.gaussian }
});
vga2D = Distribution <Vector> .Array(new[, ] {
{ this.vectorGaussian, this.vectorGaussian }, { this.vectorGaussian, this.vectorGaussian }
});
gaJ = Distribution <double> .Array(new[] { new[] { this.gaussian, this.gaussian }, new[] { this.gaussian, this.gaussian } });
vgaJ = Distribution <Vector> .Array(new[] { new[] { this.vectorGaussian, this.vectorGaussian }, new[] { this.vectorGaussian, this.vectorGaussian } });
var gp = new GaussianProcess(new ConstantFunction(0), new SquaredExponential(0));
var basis = Util.ArrayInit(2, i => Vector.FromArray(1.0 * i));
this.sparseGp = new SparseGP(new SparseGPFixed(gp, basis));
this.quantileEstimator = new QuantileEstimator(0.01);
this.quantileEstimator.Add(5);
this.outerQuantiles = OuterQuantiles.FromDistribution(3, this.quantileEstimator);
this.innerQuantiles = InnerQuantiles.FromDistribution(3, this.outerQuantiles);
if (!skipStringDistributions)
{
// String distributions can not be serialized by some formatters (namely BinaryFormatter)
// That is fine because this combination is never used in practice
this.stringDistribution1 = StringDistribution.String("aa")
.Append(StringDistribution.OneOf("b", "ccc")).Append("dddd");
this.stringDistribution2 = new StringDistribution();
this.stringDistribution2.SetToProduct(StringDistribution.OneOf("a", "b"),
StringDistribution.OneOf("b", "c"));
}
}
/// <summary>
/// EP message to 'func'
/// </summary>
/// <param name="y">Incoming message from 'y'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="func">Incoming message from 'func'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="x">Constant value for 'x'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'func' as the random arguments are varied.
/// The formula is <c>proj[p(func) sum_(y) p(y) factor(y,func,x)]/p(func)</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="y"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="func"/> is not a proper distribution</exception>
public static SparseGP FuncAverageConditional([SkipIfUniform] Gaussian y, [SkipIfUniform] SparseGP func, Vector x, SparseGP result)
{
if (y.IsUniform() || func.IsUniform()) { result.SetToUniform(); return result; }
result.FixedParameters = func.FixedParameters;
result.IncludePrior = false;
double vf = func.Variance(x);
double my, vy;
y.GetMeanAndVariance(out my, out vy);
Vector kbx = func.FixedParameters.KernelOf_X_B(x);
Vector proj = func.FixedParameters.InvKernelOf_B_B * kbx;
double prec = 1.0/(vy + vf - func.Var_B_B.QuadraticForm(proj));
result.InducingDist.Precision.SetToOuter(proj, proj);
result.InducingDist.Precision.Scale(prec);
result.InducingDist.MeanTimesPrecision.SetTo(proj);
result.InducingDist.MeanTimesPrecision.Scale(prec*my);
result.ClearCachedValues();
return result;
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="y">Constant value for 'y'.</param>
/// <param name="func">Incoming message from 'func'.</param>
/// <param name="x">Constant value for 'x'.</param>
/// <returns>Logarithm of the factor's contribution the EP model evidence</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(func) p(func) factor(y,func,x))</c>.
/// Adding up these values across all factors and variables gives the log-evidence estimate for EP.
/// </para></remarks>
public static double LogEvidenceRatio(double y, SparseGP func, Vector x) { return LogAverageFactor(y, func, x); }
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="y">Constant value for 'y'.</param>
/// <param name="func">Incoming message from 'func'.</param>
/// <param name="x">Constant value for 'x'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(func) p(func) factor(y,func,x))</c>.
/// </para></remarks>
public static double LogAverageFactor(double y, SparseGP func, Vector x)
{
Gaussian to_y = YAverageConditional(func, x);
return to_y.GetLogProb(y);
}
/// <summary>
/// EP message to 'x'.
/// </summary>
/// <param name="y">Incoming message from 'y'.</param>
/// <param name="func">Incoming message from 'func'.</param>
/// <param name="x">Constant value for 'x'.</param>
/// <param name="result">Modified to contain the outgoing message.</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the integral of the factor times incoming messages, over all arguments except 'x'.
/// The formula is <c>int f(x,x) q(x) dx</c> where <c>x = (y,func)</c>.
/// </para></remarks>
public static VectorGaussian XAverageConditional(Gaussian y, SparseGP func, Vector x, VectorGaussian result)
{
// Doesn't matter what message we return as we are only supporting a point x
result.SetToUniform();
return result;
}
/// <summary>
/// EP message to 'func'
/// </summary>
/// <param name="y">Constant value for 'y'.</param>
/// <param name="func">Incoming message from 'func'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="x">Constant value for 'x'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'func' conditioned on the given values.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="func"/> is not a proper distribution</exception>
public static SparseGP FuncAverageConditional(double y, [SkipIfUniform] SparseGP func, Vector x, SparseGP result)
{
return FuncAverageConditional(Gaussian.PointMass(y), func, x, result);
}
public void GPClassificationTest()
{
bool[] yData = new bool[] { false, false, false, true, true, true, true, false, false, false };
double[] xData = new double[]
{
-2, -1.555555555555556, -1.111111111111111, -0.6666666666666667, -0.2222222222222223, 0.2222222222222223, 0.6666666666666665, 1.111111111111111,
1.555555555555555, 2
};
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { xVec[1], xVec[4], xVec[8] };
//basis = xVec;
IKernelFunction kf = new SquaredExponential(0.0);
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <bool> y = Variable.Array <bool>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = (h[item] > 0);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { -1.410457563120709, 1.521306076273262, -1.008600221619413 });
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
double[] xTest = new double[]
{
-2, -1, 0.0
};
Vector[] xTestVec = Array.ConvertAll(xTest, v => Vector.Constant(1, v));
// computed by matlab/MNT/GP/test_gpc.m
double[] yMeanTest = new double[]
{
-0.966351175090184, -0.123034591744284, 0.762757400008960
};
double[] yVarTest = new double[]
{
0.323871157983366, 0.164009511251333, 0.162068482365962
};
for (int i = 0; i < xTestVec.Length; i++)
{
Gaussian pred = sgp.Marginal(xTestVec[i]);
Gaussian predExpected = new Gaussian(yMeanTest[i], yVarTest[i]);
Console.WriteLine("f({0}) = {1} should be {2}", xTest[i], pred, predExpected);
Assert.True(predExpected.MaxDiff(pred) < 1e-4);
}
double evExpected = -4.907121241357144;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-6) < 1e-4);
}
/// <summary>Evidence message for EP.</summary>
/// <param name="y">Constant value for <c>y</c>.</param>
/// <param name="func">Incoming message from <c>func</c>.</param>
/// <param name="x">Constant value for <c>x</c>.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions.</returns>
/// <remarks>
/// <para>The formula for the result is <c>log(sum_(func) p(func) factor(y,func,x))</c>.</para>
/// </remarks>
public static double LogAverageFactor(double y, SparseGP func, Vector x)
{
Gaussian to_y = YAverageConditional(func, x);
return(to_y.GetLogProb(y));
}
public void GPRegressionTest()
{
double[] yData = new double[]
{
-0.06416828853982412, -0.6799959810206935, -0.4541652863622044, 0.155770359928991, 1.036659040456137, 0.7353821980830825, 0.8996680933259047,
-0.05368704705684217, -0.7905775695015919, -0.1436284683992815
};
double[] xData = new double[]
{
-2, -1.555555555555556, -1.111111111111111, -0.6666666666666667, -0.2222222222222223, 0.2222222222222223, 0.6666666666666665, 1.111111111111111,
1.555555555555555, 2
};
Vector[] xVec = Array.ConvertAll(xData, v => Vector.Constant(1, v));
Vector[] basis = new Vector[] { xVec[1], xVec[4], xVec[8] };
IKernelFunction kf = new SquaredExponential(System.Math.Log(2.0));
SparseGPFixed sgpf = new SparseGPFixed(kf, basis);
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Variable <IFunction> f = Variable.Random <IFunction>(new SparseGP(sgpf)).Named("f");
Range item = new Range(xVec.Length).Named("item");
VariableArray <Vector> x = Variable.Array <Vector>(item).Named("x");
x.ObservedValue = xVec;
VariableArray <double> y = Variable.Array <double>(item).Named("y");
y.ObservedValue = yData;
VariableArray <double> h = Variable.Array <double>(item).Named("h");
h[item] = Variable.FunctionEvaluate(f, x[item]);
y[item] = Variable.GaussianFromMeanAndVariance(h[item], 0.1);
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
SparseGP sgp = engine.Infer <SparseGP>(f);
Vector alphaExpected = Vector.FromArray(new double[] { -3.250044160725389, 4.579296091435270, -2.227005562666341 });
PositiveDefiniteMatrix betaExpected = new PositiveDefiniteMatrix(new double[, ]
{
{ 3.187555652658986, -3.301824438047169, 1.227566907279797 },
{ -3.30182443804717, 5.115027119603418, -2.373085083966294 },
{ 1.227566907279797, -2.373085083966294, 2.156308696222915 }
});
Console.WriteLine("alpha = {0} should be {1}", sgp.Alpha, alphaExpected);
Console.WriteLine(StringUtil.JoinColumns("beta = ", sgp.Beta, " should be ", betaExpected));
double[] xTest = new double[]
{
-2, -1, 0.0
};
Vector[] xTestVec = Array.ConvertAll(xTest, v => Vector.Constant(1, v));
// computed by matlab/MNT/GP/test_gpr.m
double[] yMeanTest = new double[]
{
-0.544583265595561, 0.134323399801302, 0.503623822120711
};
double[] yVarTest = new double[]
{
0.058569682375201, 0.022695532903985, 0.024439582002951
};
for (int i = 0; i < xTestVec.Length; i++)
{
Gaussian pred = sgp.Marginal(xTestVec[i]);
Gaussian predExpected = new Gaussian(yMeanTest[i], yVarTest[i]);
Console.WriteLine("f({0}) = {1} should be {2}", xTest[i], pred, predExpected);
Assert.True(predExpected.MaxDiff(pred) < 1e-4);
}
double evExpected = -13.201173794945003;
double evActual = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("evidence = {0} should be {1}", evActual, evExpected);
Assert.True(MMath.AbsDiff(evExpected, evActual, 1e-6) < 1e-4);
}
public void Run()
{
InferenceEngine engine = new InferenceEngine();
if (!(engine.Algorithm is Algorithms.ExpectationPropagation))
{
Console.WriteLine("This example only runs with Expectation Propagation");
return;
}
// The data
Vector[] inputs = new Vector[]
{
Vector.FromArray(new double[2] {
0, 0
}),
Vector.FromArray(new double[2] {
0, 1
}),
Vector.FromArray(new double[2] {
1, 0
}),
Vector.FromArray(new double[2] {
0, 0.5
}),
Vector.FromArray(new double[2] {
1.5, 0
}),
Vector.FromArray(new double[2] {
0.5, 1.0
})
};
bool[] outputs = { true, true, false, true, false, false };
// Open an evidence block to allow model scoring
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
// Set up the GP prior, a distribution over functions, which will be filled in later
Variable <SparseGP> prior = Variable.New <SparseGP>().Named("prior");
// The sparse GP variable - a random function
Variable <IFunction> f = Variable <IFunction> .Random(prior).Named("f");
// The locations to evaluate the function
VariableArray <Vector> x = Variable.Observed(inputs).Named("x");
Range j = x.Range.Named("j");
// The observation model
VariableArray <bool> y = Variable.Observed(outputs, j).Named("y");
Variable <double> score = Variable.FunctionEvaluate(f, x[j]).Named("score");
y[j] = (Variable.GaussianFromMeanAndVariance(score, 0.1) > 0);
// Close the evidence block
block.CloseBlock();
// The basis
Vector[] basis = new Vector[]
{
Vector.FromArray(new double[2] {
0.2, 0.2
}),
Vector.FromArray(new double[2] {
0.2, 0.8
}),
Vector.FromArray(new double[2] {
0.8, 0.2
}),
Vector.FromArray(new double[2] {
0.8, 0.8
})
};
for (int trial = 0; trial < 3; trial++)
{
// The kernel
IKernelFunction kf;
if (trial == 0)
{
kf = new SquaredExponential(-0.0);
}
else if (trial == 1)
{
kf = new SquaredExponential(-0.5);
}
else
{
kf = new NNKernel(new double[] { 0.0, 0.0 }, -1.0);
}
// Fill in the sparse GP prior
GaussianProcess gp = new GaussianProcess(new ConstantFunction(0), kf);
prior.ObservedValue = new SparseGP(new SparseGPFixed(gp, basis));
// Model score
double NNscore = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("{0} evidence = {1}", kf, NNscore.ToString("g4"));
}
// Infer the posterior Sparse GP
SparseGP sgp = engine.Infer <SparseGP>(f);
// Check that training set is classified correctly
Console.WriteLine("");
Console.WriteLine("Predictions on training set:");
for (int i = 0; i < outputs.Length; i++)
{
Gaussian post = sgp.Marginal(inputs[i]);
double postMean = post.GetMean();
string comment = (outputs[i] == (postMean > 0.0)) ? "correct" : "incorrect";
Console.WriteLine("f({0}) = {1} ({2})", inputs[i], post, comment);
}
}
/// <summary>Evidence message for EP.</summary>
/// <param name="y">Constant value for <c>y</c>.</param>
/// <param name="func">Incoming message from <c>func</c>.</param>
/// <param name="x">Constant value for <c>x</c>.</param>
/// <returns>Logarithm of the factor's contribution the EP model evidence.</returns>
/// <remarks>
/// <para>The formula for the result is <c>log(sum_(func) p(func) factor(y,func,x))</c>. Adding up these values across all factors and variables gives the log-evidence estimate for EP.</para>
/// </remarks>
public static double LogEvidenceRatio(double y, SparseGP func, Vector x)
{
return(LogAverageFactor(y, func, x));
}
public void GPClassificationExample()
{
// The data
bool[] ydata = { true, true, false, true, false, false };
Vector[] xdata = new Vector[]
{
Vector.FromArray(new double[2] {
0, 0
}),
Vector.FromArray(new double[2] {
0, 1
}),
Vector.FromArray(new double[2] {
1, 0
}),
Vector.FromArray(new double[2] {
0, 0.5
}),
Vector.FromArray(new double[2] {
1.5, 0
}),
Vector.FromArray(new double[2] {
0.5, 1.0
})
};
// Open an evidence block to allow model scoring
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
// Set up the GP prior, which will be filled in later
Variable <SparseGP> prior = Variable.New <SparseGP>().Named("prior");
// The sparse GP variable - a distribution over functions
Variable <IFunction> f = Variable <IFunction> .Random(prior).Named("f");
// The locations to evaluate the function
VariableArray <Vector> x = Variable.Constant(xdata).Named("x");
Range j = x.Range.Named("j");
// The observation model
VariableArray <bool> y = Variable.Array <bool>(j).Named("y");
y[j] = (Variable.GaussianFromMeanAndVariance(Variable.FunctionEvaluate(f, x[j]), 0.1) > 0);
// Attach the observations
y.ObservedValue = ydata;
// Close the evidence block
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
// The basis
Vector[] basis = new Vector[]
{
Vector.FromArray(new double[2] {
0.2, 0.2
}),
Vector.FromArray(new double[2] {
0.2, 0.8
}),
Vector.FromArray(new double[2] {
0.8, 0.2
}),
Vector.FromArray(new double[2] {
0.8, 0.8
})
};
for (int trial = 0; trial < 3; trial++)
{
// The kernel
IKernelFunction kf;
if (trial == 0)
{
kf = new SquaredExponential(-0.0);
//kf = new LinearKernel(new double[] { 0.0, 0.0 });
}
else if (trial == 1)
{
kf = new SquaredExponential(-0.5);
}
else
{
kf = new NNKernel(new double[] { 0.0, 0.0 }, -1.0);
}
// Fill in the sparse GP prior
prior.ObservedValue = new SparseGP(new SparseGPFixed(kf, basis));
// Model score
double NNscore = engine.Infer <Bernoulli>(evidence).LogOdds;
Console.WriteLine("{0} evidence = {1}", kf, NNscore.ToString("g4"));
}
// Infer the posterior Sparse GP
SparseGP sgp = engine.Infer <SparseGP>(f);
// Check that training set is classified correctly
Console.WriteLine();
Console.WriteLine("Predictions on training set:");
for (int i = 0; i < ydata.Length; i++)
{
Gaussian post = sgp.Marginal(xdata[i]);
double postMean = post.GetMean();
string comment = (ydata[i] == (postMean > 0.0)) ? "correct" : "incorrect";
Console.WriteLine("f({0}) = {1} ({2})", xdata[i], post, comment);
Assert.True(ydata[i] == (postMean > 0.0));
}
}
