public static void GenData(int n, Vector w, double b,
out Vector[] X, out bool[] Y, int seed = 1)
{
Rand.Restart(seed);
int d = w.Count;
X = new Vector[n];
Y = new bool[n];
if (w == null)
{
throw new ArgumentException("coefficient vector w cannot be null");
}
// X = new Vector[n];
// Y = new double[n];
for (int i = 0; i < n; i++)
{
X[i] = Vector.Zero(d);
// samples are from standard multivariate normal
Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.IdentityScaledBy(d, 1), X[i]);
// Gamma random noise to each dimension
// X[i] = Rand.Gamma(1)*X[i];
double inner = w.Inner(X[i]);
double p = MMath.Logistic(inner + b);
// Y[i] = p >= 0.5 ? 1.0 - ep : 0.0 + ep;
Y[i] = Bernoulli.Sample(p);
// Y[i] = p >= 0.5;
}
}
/// <summary>
/// Returns the community score matrix prior.
/// </summary>
/// <returns>The community score matrix prior.</returns>
private VectorGaussian[] GetScoreMatrixPrior()
{
var dim = new Range(LabelCount);
var mean = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(LabelCount), PositiveDefiniteMatrix.IdentityScaledBy(LabelCount, 1));
var prec = Variable.WishartFromShapeAndRate(1.0, PositiveDefiniteMatrix.IdentityScaledBy(LabelCount, 1));
var score = Variable.VectorGaussianFromMeanAndPrecision(mean, prec);
var confusion = Variable.Softmax(score);
confusion.SetValueRange(dim);
var confusionConstraint = Variable.New <Dirichlet>();
Variable.ConstrainEqualRandom(confusion, confusionConstraint);
var engine = new InferenceEngine(new VariationalMessagePassing())
{
ShowProgress = false
};
engine.Compiler.WriteSourceFiles = false;
var scorePrior = new VectorGaussian[LabelCount];
for (int d = 0; d < LabelCount; d++)
{
confusionConstraint.ObservedValue = new Dirichlet(Util.ArrayInit(LabelCount, i => i == d ? (InitialWorkerBelief / (1 - InitialWorkerBelief)) * (LabelCount - 1) : 1.0));
scorePrior[d] = engine.Infer <VectorGaussian>(score);
}
return(scorePrior);
}
/// <summary>
/// Initializes the CBCC model with a number of communities.
/// </summary>
/// <param name="taskCount">The number of tasks.</param>
/// <param name="labelCount">The number of labels.</param>
/// <param name="communityCount">The number of communities.</param>
public virtual void CreateModel(int taskCount, int labelCount, int communityCount)
{
Evidence = Variable <bool> .Random(this.EvidencePrior);
var evidenceBlock = Variable.If(Evidence);
CommunityCount = communityCount;
CommunityProbPriorObserved = Dirichlet.Symmetric(communityCount, CommunityPseudoCount);
DefineVariablesAndRanges(taskCount, labelCount);
DefineGenerativeProcess();
DefineInferenceEngine();
evidenceBlock.CloseBlock();
if (ScoreMeanParameters == null)
{
var scoreMatrixPrior = GetScoreMatrixPrior();
CommunityScoreMatrixPriorObserved = Util.ArrayInit(CommunityCount, comm => Util.ArrayInit(labelCount, lab => new VectorGaussian(scoreMatrixPrior[lab])));
}
else
{
CommunityScoreMatrixPriorObserved = Util.ArrayInit(
CommunityCount,
comm => Util.ArrayInit(
labelCount, lab => VectorGaussian.FromMeanAndPrecision(
Vector.FromArray(
Util.ArrayInit(labelCount, lab1 => lab == lab1 ? ScoreMeanParameters[comm].Item1 : ScoreMeanParameters[comm].Item2)),
PositiveDefiniteMatrix.IdentityScaledBy(LabelCount, ScorePrecisionParameters[comm]))));
}
}
/// <summary>
/// For the multinomial regression model: generate synthetic data,
/// infer the model parameters and calculate the RMSE between the true
/// and mean inferred coefficients.
/// </summary>
/// <param name="numSamples">Number of samples</param>
/// <param name="numFeatures">Number of input features</param>
/// <param name="numClasses">Number of classes</param>
/// <param name="countPerSample">Total count per sample</param>
/// <returns>RMSE between the true and mean inferred coefficients</returns>
public double MultinomialRegressionSynthetic(
int numSamples, int numFeatures, int numClasses, int countPerSample, double noiseVar = 0.0)
{
var features = new Vector[numSamples];
var counts = new int[numSamples][];
var coefficients = new Vector[numClasses];
var bias = Vector.Zero(numClasses);
Rand.Restart(1);
for (int i = 0; i < numClasses - 1; i++)
{
bias[i] = Rand.Normal();
coefficients[i] = Vector.Zero(numFeatures);
Rand.Normal(Vector.Zero(numFeatures), PositiveDefiniteMatrix.Identity(numFeatures), coefficients[i]);
}
bias[numClasses - 1] = 0;
coefficients[numClasses - 1] = Vector.Zero(numFeatures);
var noiseDistribution = new VectorGaussian(Vector.Zero(numClasses), PositiveDefiniteMatrix.IdentityScaledBy(numClasses, noiseVar));
for (int i = 0; i < numSamples; i++)
{
features[i] = Vector.Zero(numFeatures);
Rand.Normal(Vector.Zero(numFeatures), PositiveDefiniteMatrix.Identity(numFeatures), features[i]);
var temp = Vector.FromArray(coefficients.Select(o => o.Inner(features[i])).ToArray());
if (noiseVar != 0.0)
{
temp += noiseDistribution.Sample();
}
var p = MMath.Softmax(temp + bias);
counts[i] = Rand.Multinomial(countPerSample, p);
}
IList <VectorGaussian> weightsPost;
IList <Gaussian> biasPost;
bool trackLowerBound = true;
double ev = MultinomialRegression(features, counts, out weightsPost, out biasPost, trackLowerBound);
if (trackLowerBound)
{
Console.WriteLine("Log lower bound= " + ev);
}
double error = 0;
Console.WriteLine("Weights -------------- ");
for (int i = 0; i < numClasses; i++)
{
var bMean = weightsPost[i].GetMean();
error += (bMean - coefficients[i]).Sum(o => o * o);
Console.WriteLine("Class " + i + " True " + coefficients[i]);
Console.WriteLine("Class " + i + " Inferred " + bMean);
}
error = System.Math.Sqrt(error / (numClasses * numFeatures));
Console.WriteLine("RMSE " + error);
Console.WriteLine("Bias -------------- ");
Console.WriteLine("True " + bias);
Console.WriteLine("Inferred " + Vector.FromArray(biasPost.Select(o => o.GetMean()).ToArray()));
return(error);
}
public ISuff GetSuffStat()
{
Vector moment1 = GetMeanVector();
double m2 = variance + mean * mean;
PositiveDefiniteMatrix moment2 = PositiveDefiniteMatrix.IdentityScaledBy(1, m2);
SuffStat s = new SuffStat(moment1, moment2);
return(s);
}
public void Run()
{
// Define a range for the number of mixture components
Range k = new Range(2).Named("k");
// Mixture component means
VariableArray <Vector> means = Variable.Array <Vector>(k).Named("means");
means[k] = Variable.VectorGaussianFromMeanAndPrecision(
Vector.FromArray(0.0, 0.0),
PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// Mixture component precisions
VariableArray <PositiveDefiniteMatrix> precs = Variable.Array <PositiveDefiniteMatrix>(k).Named("precs");
precs[k] = Variable.WishartFromShapeAndScale(100.0, PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// Mixture weights
Variable <Vector> weights = Variable.Dirichlet(k, new double[] { 1, 1 }).Named("weights");
// Create a variable array which will hold the data
Range n = new Range(300).Named("n");
VariableArray <Vector> data = Variable.Array <Vector>(n).Named("x");
// Create latent indicator variable for each data point
VariableArray <int> z = Variable.Array <int>(n).Named("z");
// The mixture of Gaussians model
using (Variable.ForEach(n))
{
z[n] = Variable.Discrete(weights);
using (Variable.Switch(z[n]))
{
data[n] = Variable.VectorGaussianFromMeanAndPrecision(means[z[n]], precs[z[n]]);
}
}
// Attach some generated data
data.ObservedValue = GenerateData(n.SizeAsInt);
// Initialise messages randomly so as to break symmetry
Discrete[] zinit = new Discrete[n.SizeAsInt];
for (int i = 0; i < zinit.Length; i++)
{
zinit[i] = Discrete.PointMass(Rand.Int(k.SizeAsInt), k.SizeAsInt);
}
z.InitialiseTo(Distribution <int> .Array(zinit));
// The inference
InferenceEngine ie = new InferenceEngine();
Console.WriteLine("Dist over pi=" + ie.Infer(weights));
Console.WriteLine("Dist over means=\n" + ie.Infer(means));
Console.WriteLine("Dist over precs=\n" + ie.Infer(precs));
}
public double MulticlassRegressionSynthetic(int numSamples, object softmaxOperator, out int iterations, out double lowerBound, double noiseVar = 0.0)
{
int numFeatures = 6;
int numClasses = 4;
var features = new Vector[numSamples];
var counts = new int[numSamples];
var coefficients = new Vector[numClasses];
var mean = Vector.Zero(numClasses);
for (int i = 0; i < numClasses - 1; i++)
{
mean[i] = Rand.Normal();
coefficients[i] = Vector.Zero(numFeatures);
Rand.Normal(Vector.Zero(numFeatures), PositiveDefiniteMatrix.Identity(numFeatures), coefficients[i]);
}
mean[numClasses - 1] = 0;
coefficients[numClasses - 1] = Vector.Zero(numFeatures);
var noiseDistribution = new VectorGaussian(Vector.Zero(numClasses), PositiveDefiniteMatrix.IdentityScaledBy(numClasses, noiseVar));
for (int i = 0; i < numSamples; i++)
{
features[i] = Vector.Zero(numFeatures);
Rand.Normal(Vector.Zero(numFeatures), PositiveDefiniteMatrix.Identity(numFeatures), features[i]);
var temp = Vector.FromArray(coefficients.Select(o => o.Inner(features[i])).ToArray());
if (noiseVar != 0.0)
{
temp += noiseDistribution.Sample();
}
var p = MMath.Softmax(temp + mean);
counts[i] = Rand.Sample(p);
}
Rand.Restart(DateTime.Now.Millisecond);
VectorGaussian[] bPost;
Gaussian[] meanPost;
iterations = MulticlassRegression(features, counts, numClasses, out bPost, out meanPost, out lowerBound, softmaxOperator, true);
var bMeans = bPost.Select(o => o.GetMean()).ToArray();
var bVars = bPost.Select(o => o.GetVariance()).ToArray();
double error = 0;
Console.WriteLine("Coefficients -------------- ");
for (int i = 0; i < numClasses; i++)
{
error += (bMeans[i] - coefficients[i]).Sum(o => o * o);
Console.WriteLine("True " + coefficients[i]);
Console.WriteLine("Inferred " + bMeans[i]);
}
Console.WriteLine("Mean -------------- ");
Console.WriteLine("True " + mean);
Console.WriteLine("Inferred " + Vector.FromArray(meanPost.Select(o => o.GetMean()).ToArray()));
error = System.Math.Sqrt(error / (numClasses * numFeatures));
Console.WriteLine(numSamples + " " + error);
return(error);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="SumVectorGaussianOp"]/message_doc[@name="SumAverageConditional(IList{VectorGaussian}, VectorGaussian)"]/*'/>
public static VectorGaussian SumAverageConditional([SkipIfAnyUniform] IList <VectorGaussian> array, VectorGaussian result)
{
if (array == null)
{
throw new ArgumentNullException(nameof(array));
}
if (result == null)
{
throw new ArgumentNullException(nameof(result));
}
if (array.Count < 1)
{
result.Point = Vector.Zero(result.Dimension);
return(result);
}
if (array.Any(element => element == null))
{
throw new ArgumentNullException(nameof(array));
}
int dimension = result.Dimension;
if (array.Any(element => element.Dimension != dimension))
{
throw new ArgumentException("The result and all elements of the array must have the same number of dimensions.");
}
var sumMean = Vector.Zero(dimension);
var sumVariance = PositiveDefiniteMatrix.IdentityScaledBy(dimension, 0);
var elementMean = Vector.Zero(dimension);
var elementVariance = PositiveDefiniteMatrix.Identity(dimension);
foreach (var element in array)
{
if (!element.IsProper())
{
return(element);
}
element.GetMeanAndVariance(elementMean, elementVariance);
sumMean.SetToSum(sumMean, elementMean);
sumVariance.SetToSum(sumVariance, elementVariance);
}
result.SetMeanAndVariance(sumMean, sumVariance);
return(result);
}
/// <summary>
/// The entry point of the program, where the program control starts and ends.
/// </summary>
public static void Main()
{
const int T = 100;
const int K = 2;
const int N = 5;
const bool showFactorGraph = false;
TestHMM <ContinousHMM, double, double, Gaussian, Gaussian, double, Gamma, double>(
T,
K,
1,
Gaussian.FromMeanAndPrecision,
() => Gaussian.FromMeanAndVariance(0, 1000),
() => Gamma.FromShapeAndScale(1000, 0.001),
showFactorGraph);
// TModel, TEmit, TEmitDist, TEmitMeanDist, TEmitMean, TEmitPrecDist, TEmitPrec
TestHMM <MultivariateHMM, Vector, Vector, VectorGaussian, VectorGaussian, Vector, Wishart, PositiveDefiniteMatrix>(
T,
K,
1,
VectorGaussian.FromMeanAndPrecision,
() => VectorGaussian.FromMeanAndVariance(Vector.Zero(N), PositiveDefiniteMatrix.IdentityScaledBy(N, 1000)),
() => Wishart.FromShapeAndScale(N, PositiveDefiniteMatrix.IdentityScaledBy(N, 0.001)),
showFactorGraph);
TestHMM <BinaryHMM, bool, double, Bernoulli, Beta, double, Beta, double>(
T,
K,
1,
(m, p) => new Bernoulli(m),
() => new Beta(1, 1),
null,
showFactorGraph);
TestHMM <DiscreteHMM, int, double, Discrete, Dirichlet, Vector, Dirichlet, Vector>(
T,
K,
N,
(m, p) => new Discrete(m),
() => Dirichlet.Uniform(N),
null,
showFactorGraph);
// TestBinaryHiddenMarkovModel();
// TestDiscreteHiddenMarkovModel();
// TestMultivariateHMM();
}
/// <summary>
/// Sets the priors of CBCC.
/// </summary>
/// <param name="workerCount">The number of workers.</param>
/// <param name="priors">The priors.</param>
protected override void SetPriors(int workerCount, BCCPosteriors priors)
{
int communityCount = m.SizeAsInt;
int labelCount = c.SizeAsInt;
WorkerCount.ObservedValue = workerCount;
NoiseMatrix.ObservedValue = PositiveDefiniteMatrix.IdentityScaledBy(labelCount, NoisePrecision);
CBCCPosteriors cbccPriors = (CBCCPosteriors)priors;
if (cbccPriors == null || cbccPriors.BackgroundLabelProb == null)
{
BackgroundLabelProbPrior.ObservedValue = Dirichlet.Uniform(labelCount);
}
else
{
BackgroundLabelProbPrior.ObservedValue = cbccPriors.BackgroundLabelProb;
}
if (cbccPriors == null || cbccPriors.CommunityProb == null)
{
CommunityProbPrior.ObservedValue = CommunityProbPriorObserved;
}
else
{
CommunityProbPrior.ObservedValue = cbccPriors.CommunityProb;
}
if (cbccPriors == null || cbccPriors.CommunityScoreMatrix == null)
{
CommunityScoreMatrixPrior.ObservedValue = CommunityScoreMatrixPriorObserved;
}
else
{
CommunityScoreMatrixPrior.ObservedValue = cbccPriors.CommunityScoreMatrix;
}
if (cbccPriors == null || cbccPriors.TrueLabelConstraint == null)
{
TrueLabelConstraint.ObservedValue = Util.ArrayInit(TaskCount, t => Discrete.Uniform(labelCount));
}
else
{
TrueLabelConstraint.ObservedValue = cbccPriors.TrueLabelConstraint;
}
}
private static void GenerateRandomMixtureComponent(
MicrosoftResearch.Infer.Maths.Vector min, MicrosoftResearch.Infer.Maths.Vector max, out MicrosoftResearch.Infer.Maths.Vector mean, out PositiveDefiniteMatrix covariance)
{
Debug.Assert(min != null && max != null);
Debug.Assert(min.Count == max.Count);
MicrosoftResearch.Infer.Maths.Vector diff = max - min;
mean = MicrosoftResearch.Infer.Maths.Vector.Zero(min.Count);
for (int i = 0; i < min.Count; ++i)
{
mean[i] = min[i] + diff[i] * Random.Double();
}
covariance = PositiveDefiniteMatrix.IdentityScaledBy(
min.Count,
MicrosoftResearch.Infer.Maths.Vector.InnerProduct(diff, diff) / 16);
}
public static VectorGaussian AAverageConditional(
Vector sumMean, PositiveDefiniteMatrix sumVariance,
Vector bMean, PositiveDefiniteMatrix bVariance,
VectorGaussian result)
{
if (result == null)
{
throw new ArgumentNullException(nameof(result));
}
int dimension = result.Dimension;
var aMean = Vector.Zero(dimension);
var aVariance = PositiveDefiniteMatrix.IdentityScaledBy(dimension, 0);
aMean.SetToDifference(sumMean, bMean);
aVariance.SetToSum(sumVariance, bVariance);
result.SetMeanAndVariance(aMean, aVariance);
return(result);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="SumVectorGaussianOp"]/message_doc[@name="ArrayAverageConditional{TVectorGaussianList}(VectorGaussian, VectorGaussian, IList{VectorGaussian}, TVectorGaussianList)"]/*'/>
public static TVectorGaussianList ArrayAverageConditional <TVectorGaussianList>(
[SkipIfUniform] VectorGaussian sum,
[Fresh] VectorGaussian to_sum,
IList <VectorGaussian> array,
TVectorGaussianList result)
where TVectorGaussianList : IList <VectorGaussian>, SettableToUniform
{
// Check inputs for consistency
int dimension = CheckArgumentConsistency(sum, to_sum, array, result);
if (array.Count == 0)
{
return(result);
}
// It is tempting to put SkipIfAllUniform on array but this isn't correct if the array has one element
if (array.Count == 1)
{
result[0].SetTo(sum);
return(result);
}
if (!sum.IsProper())
{
foreach (VectorGaussian element in result)
{
element.SetTo(sum);
}
return(result);
}
var elementMean = Vector.Zero(dimension);
var elementVariance = PositiveDefiniteMatrix.Identity(dimension);
// Check if an element of the array is uniform
int indexOfUniform = -1;
for (int i = 0; i < array.Count; i++)
{
array[i].GetMeanAndVariance(elementMean, elementVariance);
// Instead of testing IsUniform, we need to test the more strict requirement that all diagonal
// elements of variance are infinite due to the way we are doing the computations
if (IsUniform(elementVariance))
{
if (indexOfUniform >= 0)
{
// More than one element of array is uniform
result.SetToUniform();
return(result);
}
indexOfUniform = i;
}
}
Vector sumMean = Vector.Zero(dimension);
PositiveDefiniteMatrix sumVariance = PositiveDefiniteMatrix.Identity(dimension);
sum.GetMeanAndVariance(sumMean, sumVariance);
Vector totalMean = Vector.Zero(sum.Dimension);
PositiveDefiniteMatrix totalVariance = PositiveDefiniteMatrix.IdentityScaledBy(sum.Dimension, 0.0);
if (indexOfUniform >= 0)
{
// Exactly one element of array is uniform
for (int i = 0; i < array.Count; i++)
{
if (i == indexOfUniform)
{
continue;
}
array[i].GetMeanAndVariance(elementMean, elementVariance);
totalMean.SetToSum(totalMean, elementMean);
totalVariance.SetToSum(totalVariance, elementVariance);
result[i].SetToUniform();
}
// totalMean = sum_{i except indexOfUniform} array[i].GetMean()
// totalVariance = sum_{i except indexOfUniform} array[i].GetVariance()
totalMean.SetToDifference(sumMean, totalMean);
totalVariance.SetToSum(sumVariance, totalVariance);
result[indexOfUniform].SetMeanAndVariance(totalMean, totalVariance);
return(result);
}
// At this point, the array has no uniform elements
// Get the mean and variance of sum of all Gaussians
to_sum.GetMeanAndVariance(totalMean, totalVariance);
// Subtract it off from the mean and variance of the incoming Gaussian from Sum
totalMean.SetToDifference(sumMean, totalMean);
totalVariance.SetToSum(totalVariance, sumVariance);
for (int i = 0; i < array.Count; i++)
{
array[i].GetMeanAndVariance(elementMean, elementVariance);
elementMean.SetToSum(elementMean, totalMean);
elementVariance.SetToDifference(totalVariance, elementVariance);
result[i].SetMeanAndVariance(elementMean, elementVariance);
}
return(result);
}
public Matrix GetCovarianceMatrix()
{
return(PositiveDefiniteMatrix.IdentityScaledBy(1, gam.GetVariance()));
}
/// <summary>
/// Run our VB implementation of the Semi Parametric Latent Factor Model of
/// Teh, Y., Seeger, M., and Jordan, M. (AISTATS 2005).
/// </summary>
public SPLFM_VMP RunSPLFM_VMP(Vector[] inputs,
double[,] data,
bool[,] isMissing,
Settings settings,
string[] errorMeasureNames = null,
Converter <IPredictionSPLFMModel, double[]> modelAssessor = null,
string swfn = null)
{
var model = new SPLFM_VMP();
var nodeOptimiser = new KernelOptimiser(settings);
nodeOptimiser.xData = inputs;
nodeOptimiser.kernel = ObjectCloner.Clone(settings.node_kernel);
nodeOptimiser.hypersToOptimise = settings.nodeHypersToOptimise;
var nodeFunctionsInit = Enumerable.Range(0, settings.Q).Select(i =>
VectorGaussian.FromMeanAndVariance(
VectorGaussian.Sample(Vector.Zero(data.GetLength(1)), PositiveDefiniteMatrix.IdentityScaledBy(data.GetLength(1), 100)),
PositiveDefiniteMatrix.IdentityScaledBy(data.GetLength(1), settings.init_precision))).ToArray(); // should put this manually in generated code
var distArray = Distribution <Vector> .Array(nodeFunctionsInit);
double inputsRange
= inputs.Select(i => i[0]).Max() - inputs.Select(i => i[0]).Min();
Console.WriteLine("Init node kernel {0}", settings.node_kernel);
model.SetObservedValue("D", data.GetLength(0));
model.SetObservedValue("Q", settings.Q);
model.SetObservedValue("N", data.GetLength(1));
model.SetObservedValue("observedData", data);
model.SetObservedValue("nodeFunctionsInitVar", distArray);
model.SetObservedValue("K_node_inverse", Utils.GramMatrix(nodeOptimiser.kernel, inputs).Inverse());
model.SetObservedValue("noisePrecisionPrior", settings.noisePrecisionPrior);
//model.SetObservedValue("nodeNoisePrecisionPrior", settings.nodeNoisePrecisionPrior);
model.SetObservedValue("nodeSignalPrecisionsPrior", Enumerable.Range(0, settings.Q).Select(o => settings.nodeSignalPrecisionsPrior).ToArray());
model.SetObservedValue("isMissing", isMissing);
model.nodeKernelOptimiser = nodeOptimiser;
model.Reset();
var start = DateTime.Now;
if (swfn != null)
{
using (var sw = new StreamWriter(swfn, true))
{
sw.Write("{0} {1} {2}", "it", "time", "ml");
if (errorMeasureNames != null)
{
sw.Write(" " + errorMeasureNames.Aggregate((p, q) => p + " " + q));
}
sw.Write(" " + Utils.KernelHyperNames(nodeOptimiser.kernel).Select(o => "node_" + o).Aggregate((p, q) => p + " " + q));
sw.Write(" noise");
for (int i = 0; i < settings.Q; i++)
{
sw.Write(" signal" + i);
}
sw.WriteLine();
}
}
double oldML = double.NegativeInfinity;
double ml = 0;
int it = 0;
for (; it < settings.max_iterations; it++)
{
model.Update(1);
ml = model.Marginal <Bernoulli>("ev").LogOdds;
var noisePrecisionPost = model.Marginal <Gamma>("noisePrecision");
var assessment = (modelAssessor != null) ? modelAssessor(model).Select(o => o.ToString()).Aggregate((p, q) => p + " " + q) : "";
Console.WriteLine("It " + it + " node " + nodeOptimiser.kernel + " ml " + ml + " err " + assessment);
if (Math.Abs(oldML - ml) < settings.ml_tolerance)
{
break;
}
oldML = ml;
if (swfn != null)
{
using (var sw = new StreamWriter(swfn, true))
{
var nodeSignalPrecisionsPost = model.Marginal <Gamma[]>("nodeSignalPrecisions");
sw.Write("{0} {1} {2}", it, (DateTime.Now - start).TotalMilliseconds, ml);
if (modelAssessor != null)
{
sw.Write(" " + assessment);
}
sw.Write(" " + Utils.KernelToArray(nodeOptimiser.kernel).Select(o => o.ToString()).Aggregate((p, q) => p + " " + q));
sw.Write(" " + noisePrecisionPost.GetMeanInverse());
for (int i = 0; i < settings.Q; i++)
{
sw.Write(" " + nodeSignalPrecisionsPost[i].GetMeanInverse());
}
sw.WriteLine();
}
}
}
Console.WriteLine("Finished after " + it);
return(model);
}
public void MixtureOfMultivariateGaussians()
{
// Define a range for the number of mixture components
Range k = new Range(2).Named("k");
// Mixture component means
VariableArray <Vector> means = Variable.Array <Vector>(k).Named("means");
means[k] = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(2), PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// Mixture component precisions
VariableArray <PositiveDefiniteMatrix> precs = Variable.Array <PositiveDefiniteMatrix>(k).Named("precs");
precs[k] = Variable.WishartFromShapeAndScale(100.0, PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// Mixture weights
Variable <Vector> weights = Variable.Dirichlet(k, new double[] { 1, 1 }).Named("weights");
// Create a variable array which will hold the data
Range n = new Range(300).Named("n");
VariableArray <Vector> data = Variable.Array <Vector>(n).Named("x");
// Create latent indicator variable for each data point
VariableArray <int> z = Variable.Array <int>(n).Named("z");
// The mixture of Gaussians model
using (Variable.ForEach(n))
{
z[n] = Variable.Discrete(weights);
using (Variable.Switch(z[n]))
{
data[n] = Variable.VectorGaussianFromMeanAndPrecision(means[z[n]], precs[z[n]]);
}
}
// Attach some generated data
double truePi = 0.6;
data.ObservedValue = GenerateData(n.SizeAsInt, truePi);
// Initialise messages randomly so as to break symmetry
Discrete[] zinit = new Discrete[n.SizeAsInt];
for (int i = 0; i < zinit.Length; i++)
{
zinit[i] = Discrete.PointMass(Rand.Int(k.SizeAsInt), k.SizeAsInt);
}
z.InitialiseTo(Distribution <int> .Array(zinit));
// The inference
InferenceEngine ie = new InferenceEngine();
ie.Algorithm = new VariationalMessagePassing();
//ie.Compiler.GenerateInMemory = false;
//ie.NumberOfIterations = 200;
Dirichlet wDist = (Dirichlet)ie.Infer(weights);
Vector wEstMean = wDist.GetMean();
object meansActual = ie.Infer(means);
Console.WriteLine("means = ");
Console.WriteLine(meansActual);
var precsActual = ie.Infer <IList <Wishart> >(precs);
Console.WriteLine("precs = ");
Console.WriteLine(precsActual);
Console.WriteLine("w = {0} should be {1}", wEstMean, Vector.FromArray(truePi, 1 - truePi));
//Console.WriteLine(StringUtil.JoinColumns("z = ", ie.Infer(z)));
Assert.True(
MMath.AbsDiff(wEstMean[0], truePi) < 0.05 ||
MMath.AbsDiff(wEstMean[1], truePi) < 0.05);
}
static void Main(string[] args)
{
// x_0, x_1
double[][] data = new double[40][] {
new double [] { 1.579866, 5.113884 },
new double [] { 2.596718, 5.731915 },
new double [] { 2.250065, 5.431592 },
new double [] { 1.670613, 4.528363 },
new double [] { 1.819446, 4.941831 },
new double [] { 2.662797, 5.065860 },
new double [] { 1.999470, 5.029127 },
new double [] { 0.871543, 4.512580 },
new double [] { 1.751953, 4.926783 },
new double [] { 2.765513, 5.743687 },
new double [] { 1.855384, 4.789793 },
new double [] { 3.096498, 6.281974 },
new double [] { 2.071130, 4.893247 },
new double [] { 1.920525, 4.957070 },
new double [] { 1.040348, 3.779229 },
new double [] { 2.128256, 4.971110 },
new double [] { 1.865504, 4.909423 },
new double [] { 2.214581, 5.388266 },
new double [] { 2.679222, 5.509920 },
new double [] { 2.511456, 5.758537 },
new double [] { 1.373862, 3.160759 },
new double [] { 1.089823, 3.321712 },
new double [] { 1.179304, 3.063109 },
new double [] { 1.222694, 3.319575 },
new double [] { 1.583625, 3.028068 },
new double [] { 1.813981, 2.564715 },
new double [] { 1.764641, 2.498540 },
new double [] { 1.942168, 2.343563 },
new double [] { 1.496726, 3.097716 },
new double [] { 1.248391, 3.171456 },
new double [] { 1.167128, 3.165683 },
new double [] { 1.776658, 2.666594 },
new double [] { 1.024596, 3.529081 },
new double [] { 1.495685, 3.055696 },
new double [] { 0.863904, 3.651468 },
new double [] { 1.814797, 2.607492 },
new double [] { 1.796724, 2.754365 },
new double [] { 0.927581, 3.627839 },
new double [] { 1.544352, 2.947286 },
new double [] { 1.400211, 3.180880 }
};
Vector[] normalData = new Vector[20];
Vector[] anomalousData = new Vector[20];
for (int i = 0; i < 20; i++)
{
double[] input = new double[2] {
data[i][0], data[i][1]
};
normalData[i] = Vector.FromArray(input);
}
for (int i = 0; i < 20; i++)
{
double[] input = new double[2] {
data[i + 19][0], data[i + 19][1]
};
anomalousData[i] = Vector.FromArray(input);
}
Range k = new Range(2);
VariableArray <Vector> means = Variable.Array <Vector>(k);
means[k] = Variable.VectorGaussianFromMeanAndPrecision(Vector.FromArray(0.0, 0.0), PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
VariableArray <PositiveDefiniteMatrix> precs = Variable.Array <PositiveDefiniteMatrix>(k);
precs[k] = Variable.WishartFromShapeAndScale(100.0, PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// define the Dirichlet prior over the normal data points (concentrate on class 0)
Variable <Vector> weightsNormalPoints = Variable.Dirichlet(k, new double[] { 1, 0.1 });
// define range for the normal data
Range n = new Range(20);
// define the x Gaussian random variables we will observe as normal data points
VariableArray <Vector> xNormalPoints = Variable.Array <Vector>(n);
// define latent z for all the normal data points
VariableArray <int> zNormalPoints = Variable.Array <int>(n);
using (Variable.ForEach(n))
{
zNormalPoints[n] = Variable.Discrete(weightsNormalPoints);
using (Variable.Switch(zNormalPoints[n]))
{
xNormalPoints[n] = Variable.VectorGaussianFromMeanAndPrecision(means[zNormalPoints[n]], precs[zNormalPoints[n]]);
}
}
// define range for the anomalous data
Range m = new Range(20);
// define the x Gaussian random variables we will observe as anomalous data points
VariableArray <Vector> xAnomalousPoints = Variable.Array <Vector>(m);
// define the Dirichlet prior over the anomalous data points (concentrate on class 1)
Variable <Vector> weightsAnomalousPoints = Variable.Dirichlet(k, new double[] { 0.1, 1 });
// define latent z for all the anomalous data points
VariableArray <int> zAnomalousPoints = Variable.Array <int>(m);
using (Variable.ForEach(m))
{
zAnomalousPoints[m] = Variable.Discrete(weightsAnomalousPoints);
using (Variable.Switch(zAnomalousPoints[m]))
{
xAnomalousPoints[m] = Variable.VectorGaussianFromMeanAndPrecision(means[zAnomalousPoints[m]], precs[zAnomalousPoints[m]]);
}
}
xNormalPoints.ObservedValue = normalData;
xAnomalousPoints.ObservedValue = anomalousData;
InferenceEngine ie = new InferenceEngine(new VariationalMessagePassing());
//var weightsPost = ie.Infer<Dirichlet>(weightsNormalPoints);
//var anomalyWeightsPost = ie.Infer<Dirichlet>(weightsAnomalousPoints);
//var meansPost = ie.Infer<VectorGaussian[]>(means);
//var precsPost = ie.Infer<Wishart[]>(precs);
//Console.WriteLine("Dist over pi=" + weightsPost.GetMean());
//Console.WriteLine("Dist over pi=" + anomalyWeightsPost.GetMean());
//Console.WriteLine("Dist over means 0 =\n" + meansPost[0]);
//Console.WriteLine("Dist over means 1 =\n" + meansPost[1]);
//Console.WriteLine("Dist over precs 0 =\n" + precsPost[0].GetMean().Inverse());
//Console.WriteLine("Dist over precs 1 =\n" + precsPost[1].GetMean().Inverse());
// now let's try making a prediction with an "unseen" data point x
Range xn = new Range(1);
VariableArray <Vector> xNew = Variable.Array <Vector>(xn);
VariableArray <int> zNew = Variable.Array <int>(xn);
// define a uniform Dirichlet prior over the latent z
Variable <Vector> zNewWeights = Variable.Dirichlet(k, new double[] { 1, 1 });
using (Variable.ForEach(xn))
{
zNew[xn] = Variable.Discrete(zNewWeights);
using (Variable.Switch(zNew[xn]))
{
xNew[xn] = Variable.VectorGaussianFromMeanAndPrecision(means[zNew[xn]], precs[zNew[xn]]);
}
}
Vector[] xNewData = new Vector[1];
for (int i = 0; i < 1; i++)
{
double[] input = new double[2] {
3, 6
};
xNewData[i] = Vector.FromArray(input);
}
xNew.ObservedValue = xNewData;
var zNewPost = ie.Infer <Discrete[]>(zNew);
Console.WriteLine(zNewPost[0]);
}
/// <summary>
/// Run GPRN without node noise
/// </summary>
/// <param name="inputs">Covariates X</param>
/// <param name="data">Outputs Y</param>
/// <param name="settings">Algorithm settings</param>
/// <param name="swfn">Filename for logging</param>
/// <returns>Fitted model</returns>
public GPRN_VMP NetworkModelCA(Vector[] inputs,
double[,] data,
Settings settings,
string swfn = null)
{
bool anyIsMissing = false; // AnyIsMissing(isMissing);
var model = new GPRN_VMP();
var nodeOptimiser = new KernelOptimiser(settings);
var weightOptimiser = new KernelOptimiser(settings);
nodeOptimiser.xData = inputs;
weightOptimiser.xData = inputs;
nodeOptimiser.kernel = ObjectCloner.Clone(settings.node_kernel);
nodeOptimiser.hypersToOptimise = settings.nodeHypersToOptimise;
weightOptimiser.kernel = ObjectCloner.Clone(settings.weight_kernel);
weightOptimiser.hypersToOptimise = settings.weightHypersToOptimise;
var nodeFunctionsInit = Enumerable.Range(0, settings.Q).Select(i =>
VectorGaussian.FromMeanAndVariance(
VectorGaussian.Sample(Vector.Zero(data.GetLength(1)), PositiveDefiniteMatrix.IdentityScaledBy(data.GetLength(1), 100)),
PositiveDefiniteMatrix.IdentityScaledBy(data.GetLength(1), settings.init_precision))).ToArray(); // should put this manually in generated code
var distArray = Distribution <Vector> .Array(nodeFunctionsInit);
double inputsRange
= inputs.Select(i => i[0]).Max() - inputs.Select(i => i[0]).Min();
Console.WriteLine("Init node kernel {0}\ninit weight kernel {1}", settings.node_kernel, settings.weight_kernel);
model.SetObservedValue("D", data.GetLength(0));
model.SetObservedValue("Q", settings.Q);
model.SetObservedValue("N", data.GetLength(1));
model.SetObservedValue("observedData", data);
model.SetObservedValue("nodeFunctionsInitVar", distArray);
model.SetObservedValue("K_node_inverse", Utils.GramMatrix(nodeOptimiser.kernel, inputs).Inverse());
model.SetObservedValue("K_weights_inverse", Utils.GramMatrix(weightOptimiser.kernel, inputs).Inverse());
model.SetObservedValue("noisePrecisionPrior", settings.noisePrecisionPrior);
//model.SetObservedValue("nodeNoisePrecisionPrior", settings.nodeNoisePrecisionPrior);
model.SetObservedValue("nodeSignalPrecisionsPrior", settings.nodeSignalPrecisionsPrior);
//model.SetObservedValue("isMissing", isMissing);
model.nodeKernelOptimiser = nodeOptimiser;
model.weightKernelOptimiser = weightOptimiser;
model.Reset();
var start = DateTime.Now;
if (swfn != null)
{
using (var sw = new StreamWriter(swfn, true))
{
sw.Write("{0} {1} {2}", "it", "time", "ml");
if (anyIsMissing)
{
sw.Write(" {0} {1}", "logProb", "error");
}
sw.Write(" " + Utils.KernelHyperNames(nodeOptimiser.kernel).Select(o => "node_" + o).Aggregate((p, q) => p + " " + q));
sw.Write(" " + Utils.KernelHyperNames(weightOptimiser.kernel).Select(o => "weight_" + o).Aggregate((p, q) => p + " " + q));
sw.Write(" noise");
for (int i = 0; i < settings.Q; i++)
{
sw.Write(" signal" + i);
}
sw.WriteLine();
}
}
double oldML = double.NegativeInfinity;
double ml = 0;
int it = 0;
for (; it < settings.max_iterations; it++)
{
model.Update(1);
ml = model.Marginal <Bernoulli>("ev").LogOdds;
var noisePrecisionPost = model.Marginal <Gamma>("noisePrecision");
double logProb = 0, error = 0, MSLL = 0, SMSE = 0;
Console.WriteLine("It {9} Time: {8:G3} Node ls=exp({0:G3})={1:G3} Weight ls=exp({2:G3})={3:G3} ml={4:G3} error={5:G3} msll={6:G3} smse={7:G3}", nodeOptimiser.kernel[0], Math.Exp(nodeOptimiser.kernel[0]),
weightOptimiser.kernel[0], Math.Exp(weightOptimiser.kernel[0]), ml, error, MSLL, SMSE, (DateTime.Now - start).TotalMilliseconds, it);
if (Math.Abs(oldML - ml) < settings.ml_tolerance)
{
break;
}
oldML = ml;
if (swfn != null)
{
using (var sw = new StreamWriter(swfn, true))
{
var nodeSignalPrecisionsPost = model.Marginal <Gamma[]>("nodeSignalPrecisions");
sw.Write("{0} {1} {2}", it, (DateTime.Now - start).TotalMilliseconds, ml);
if (anyIsMissing)
{
sw.Write(" {0} {1}", logProb, error);
}
sw.Write(" " + Utils.KernelToArray(nodeOptimiser.kernel).Select(o => o.ToString()).Aggregate((p, q) => p + " " + q));
sw.Write(" " + Utils.KernelToArray(weightOptimiser.kernel).Select(o => o.ToString()).Aggregate((p, q) => p + " " + q));
sw.Write(" " + noisePrecisionPost.GetMeanInverse());
for (int i = 0; i < settings.Q; i++)
{
sw.Write(" " + nodeSignalPrecisionsPost[i].GetMeanInverse());
}
sw.WriteLine();
}
}
}
Console.WriteLine("Finished after " + it);
return(model);
}
/// <summary>
/// An implementation of GPRN specialised for one step look ahead multivariate volatility experiments
/// </summary>
/// <param name="inputs">Covariates X</param>
/// <param name="data">Outputs Y</param>
/// <returns>Predicted covariance for the next time point</returns>
public VectorGaussian GPRN_MultivariateVolatility(
Vector[] inputs,
double[,] data,
double[] nodeSignalPrecs,
double[] nodeNoisePrecs,
double obsNoisePrec,
ref VectorGaussian[] finit,
ref VectorGaussian[,] winit,
KernelFunction nodeKernel,
KernelFunction weightKernel)
{
var missing = new bool[data.GetLength(0), data.GetLength(1)];
for (int i = 0; i < data.GetLength(0); i++)
{
missing[i, data.GetLength(1) - 1] = true; // last data point is missing
}
int Q = nodeSignalPrecs.Length;
var toInfer = new List <IVariable>();
var K_node = Utils.GramMatrix(nodeKernel, inputs);
var K_weights = Utils.GramMatrix(weightKernel, inputs);
var D = Variable.Observed <int>(data.GetLength(0)).Named("D");
var d = new Range(D).Named("d");
var Qvar = Variable.Observed <int>(Q).Named("Q");
var q = new Range(Qvar).Named("q");
var N = Variable.Observed <int>(data.GetLength(1)).Named("N");
var n = new Range(N).Named("n");
var ev = Variable.Bernoulli(.5).Named("ev");
var modelBlock = Variable.If(ev);
var nodeSignalPrecisions = Variable.Array <double>(q).Named("nodeSignalPrecisions");
nodeSignalPrecisions.ObservedValue = nodeSignalPrecs;
var nodeFunctions = Variable.Array <Vector>(q).Named("nodeFunctions");
var K_node_inverse = Variable.Observed(K_node.Inverse()).Named("K_node_inverse");
nodeFunctions[q] = Variable <Vector> .Factor(MyFactors.VectorGaussianScaled, nodeSignalPrecisions[q], K_node_inverse);
nodeFunctions.AddAttribute(new MarginalPrototype(new VectorGaussian(N.ObservedValue)));
var nodeFunctionValues = Variable.Array(Variable.Array <double>(n), q).Named("nodeFunctionValues");
var nodeFunctionValuesPredictive = Variable.Array(Variable.Array <double>(n), q).Named("nodeFunctionValuesPredictive");
var nodeFunctionValuesClean = Variable.Array(Variable.Array <double>(n), q).Named("nodeFunctionValuesClean");
nodeFunctionValuesClean[q] = Variable.ArrayFromVector(nodeFunctions[q], n);
var nodeNoisePrecisions = Variable.Array <double>(q).Named("nodeNoisePrecisions");
nodeNoisePrecisions.ObservedValue = nodeNoisePrecs;
nodeFunctionValues[q][n] = Variable.GaussianFromMeanAndPrecision(nodeFunctionValuesClean[q][n], nodeNoisePrecisions[q]);
nodeFunctionValuesPredictive[q][n] = Variable.GaussianFromMeanAndPrecision(nodeFunctionValuesClean[q][n], nodeNoisePrecisions[q]);
var weightFunctions = Variable.Array <Vector>(d, q).Named("weightFunctions");
var K_weights_inverse = Variable.Observed(K_weights.Inverse()).Named("K_weights_inverse");
weightFunctions[d, q] = Variable <Vector> .Factor(MyFactors.VectorGaussianScaled, Variable.Constant <double>(1), K_weights_inverse).ForEach(d, q);
weightFunctions.AddAttribute(new MarginalPrototype(new VectorGaussian(N.ObservedValue)));
var weightFunctionValues = Variable.Array(Variable.Array <double>(n), d, q).Named("weightFunctionValues");
var weightFunctionValuesPredictive = Variable.Array(Variable.Array <double>(n), d, q).Named("weightFunctionValuesPredictive");
weightFunctionValues[d, q] = Variable.ArrayFromVector(weightFunctions[d, q], n);
weightFunctionValuesPredictive[d, q] = Variable.ArrayFromVector(weightFunctions[d, q], n);
var observedData = Variable.Array <double>(d, n).Named("observedData");
var noisePrecision = Variable.Observed(obsNoisePrec).Named("noisePrecision");
var isMissing = Variable.Array <bool>(d, n).Named("isMissing");
isMissing.ObservedValue = missing;
var noiseLessY = Variable.Array <double>(d, n).Named("noiseLessY");
using (Variable.ForEach(n))
using (Variable.ForEach(d))
{
var temp = Variable.Array <double>(q).Named("temp");
temp[q] = weightFunctionValues[d, q][n] * nodeFunctionValues[q][n];
noiseLessY[d, n] = Variable.Sum(temp);
using (Variable.IfNot(isMissing[d, n]))
observedData[d, n] = Variable.GaussianFromMeanAndPrecision(noiseLessY[d, n], noisePrecision);
using (Variable.If(isMissing[d, n]))
observedData[d, n] = Variable.GaussianFromMeanAndPrecision(0, 1);
}
observedData.ObservedValue = data;
var nodeFunctionsInit = Enumerable.Range(0, Q).Select(i =>
VectorGaussian.FromMeanAndVariance(
VectorGaussian.Sample(Vector.Zero(N.ObservedValue), PositiveDefiniteMatrix.IdentityScaledBy(N.ObservedValue, 100)),
PositiveDefiniteMatrix.IdentityScaledBy(N.ObservedValue, 100))).ToArray(); // should put this manually in generated code
var distArray = Distribution <Vector> .Array(nodeFunctionsInit);
var nodeFunctionsInitVar = Variable.Observed(distArray).Named("nodeFunctionsInitVar");
nodeFunctions.InitialiseTo(nodeFunctionsInitVar);
var finitNew = finit.Select(i => Utils.extendByOneDimension(i, Gaussian.FromMeanAndVariance(0, 1))).ToArray();
nodeFunctions.InitialiseTo(Distribution <Vector> .Array(finitNew));
var winitNew = new VectorGaussian[data.GetLength(0), Q];
for (int i = 0; i < data.GetLength(0); i++)
{
for (int j = 0; j < Q; j++)
{
winitNew[i, j] = Utils.extendByOneDimension(winit[i, j], Gaussian.FromMeanAndVariance(0, 1));
}
}
weightFunctions.InitialiseTo(Distribution <Vector> .Array(winitNew));
modelBlock.CloseBlock();
toInfer.AddRange(new List <IVariable>()
{
ev, noiseLessY, nodeFunctions, weightFunctions, nodeFunctionValuesPredictive, weightFunctionValues, weightFunctionValuesPredictive /* is this redundant? */
});
var ie = new InferenceEngine(new VariationalMessagePassing());
var ca = ie.GetCompiledInferenceAlgorithm(toInfer.ToArray());
ca.SetObservedValue(K_node_inverse.NameInGeneratedCode, Utils.GramMatrix(nodeKernel, inputs).Inverse());
ca.SetObservedValue(K_weights_inverse.NameInGeneratedCode, Utils.GramMatrix(weightKernel, inputs).Inverse());
ca.Reset();
double oldML = double.NegativeInfinity;
double ml = 0;
int it = 0;
for (; it < 30; it++)
{
ca.Update(1);
ml = ca.Marginal <Bernoulli>(ev.NameInGeneratedCode).LogOdds;
Console.WriteLine(ml);
if (Math.Abs(oldML - ml) < .1)
{
break;
}
oldML = ml;
}
var f = ca.Marginal <Gaussian[][]>("nodeFunctionValuesPredictive");
var W = ca.Marginal <Gaussian[, ][]>("weightFunctionValuesPredictive");
finit = ca.Marginal <VectorGaussian[]>(nodeFunctions.NameInGeneratedCode);
winit = ca.Marginal <VectorGaussian[, ]>(weightFunctions.NameInGeneratedCode);
return(Utils.CorrelatedPredictionsHelper(f, W, Gamma.PointMass(obsNoisePrec), Q, data.GetLength(0), data.GetLength(1) - 1));
}
public void MixtureOfMultivariateGaussians()
{
// Define a range for the number of mixture components
Range k = new Range(2).Named("k");
// Mixture component means
VariableArray <Vector> means = Variable.Array <Vector>(k).Named("means");
means[k] = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(2), PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// Mixture component precisions
VariableArray <PositiveDefiniteMatrix> precs = Variable.Array <PositiveDefiniteMatrix>(k).Named("precs");
precs[k] = Variable.WishartFromShapeAndScale(100.0, PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// Mixture weights
Variable <Vector> weights = Variable.Dirichlet(k, new double[] { 1, 1 }).Named("weights");
// Create a variable array which will hold the data
Range n = new Range(300).Named("n");
VariableArray <Vector> data = Variable.Array <Vector>(n).Named("x");
// Create latent indicator variable for each data point
VariableArray <int> z = Variable.Array <int>(n).Named("z");
// The mixture of Gaussians model
using (Variable.ForEach(n))
{
z[n] = Variable.Discrete(weights);
using (Variable.Switch(z[n]))
{
data[n] = Variable.VectorGaussianFromMeanAndPrecision(means[z[n]], precs[z[n]]);
}
}
// Attach some generated data
double truePi = 0.6;
data.ObservedValue = GenerateData(n.SizeAsInt, truePi);
// Initialise messages randomly to break symmetry
VariableArray <Discrete> zInit = Variable.Array <Discrete>(n).Named("zInit");
bool useObservedValue = true;
if (useObservedValue)
{
zInit.ObservedValue = Util.ArrayInit(n.SizeAsInt, i => Discrete.PointMass(Rand.Int(k.SizeAsInt), k.SizeAsInt));
}
else
{
// This approach doesn't work, because Infer.NET notices that Rand.Int is stochastic and thinks that it should perform message-passing here.
using (Variable.ForEach(n))
{
var randk = Variable <int> .Factor(new Func <int, int>(Rand.Int), (Variable <int>) k.Size);
randk.SetValueRange(k);
zInit[n] = Variable <Discrete> .Factor(Discrete.PointMass, randk, (Variable <int>) k.Size);
}
}
z[n].InitialiseTo(zInit[n]);
// The inference
InferenceEngine ie = new InferenceEngine();
ie.Algorithm = new VariationalMessagePassing();
//ie.Compiler.GenerateInMemory = false;
//ie.NumberOfIterations = 200;
Dirichlet wDist = (Dirichlet)ie.Infer(weights);
Vector wEstMean = wDist.GetMean();
object meansActual = ie.Infer(means);
Console.WriteLine("means = ");
Console.WriteLine(meansActual);
var precsActual = ie.Infer <IList <Wishart> >(precs);
Console.WriteLine("precs = ");
Console.WriteLine(precsActual);
Console.WriteLine("w = {0} should be {1}", wEstMean, Vector.FromArray(truePi, 1 - truePi));
//Console.WriteLine(StringUtil.JoinColumns("z = ", ie.Infer(z)));
Assert.True(
MMath.AbsDiff(wEstMean[0], truePi) < 0.05 ||
MMath.AbsDiff(wEstMean[1], truePi) < 0.05);
}
public Matrix GetCovarianceMatrix()
{
return(PositiveDefiniteMatrix.IdentityScaledBy(1, this.variance));
}
/// <summary>
/// Primary definition of the GPRN model as an Infer.NET model.
/// </summary>
/// <param name="inputs">Covariates X</param>
/// <param name="data">Outputs Y</param>
/// <param name="Q">Number of latent functions</param>
/// <param name="missing">Which elements of Y are missing</param>
/// <param name="nodeFunctionNoise">Whether to include node noise</param>
/// <param name="constrainWpositive">Whether to constrain W to be positive [experimental]</param>
/// <param name="isotropicNoise">Whether to use isotropic observation noise</param>
/// <param name="meanFunctions">Whether to include a per output mean function</param>
/// <param name="initLoglengthscales">Initial values for the length scales of the kernels</param>
/// <param name="sw">An output file for logging</param>
public void GPRN_InferNET_model(Vector[] inputs,
double[,] data,
int Q,
bool grid = false,
bool[,] missing = null,
bool nodeFunctionNoise = false,
bool constrainWpositive = false,
bool isotropicNoise = true,
bool meanFunctions = false,
double[] initLoglengthscales = null,
StreamWriter sw = null)
{
var toInfer = new List <IVariable>();
SummationKernel kf_node = new SummationKernel(new SquaredExponential(0)) + new WhiteNoise(-3);
var K_node = Utils.GramMatrix(kf_node, inputs);
SummationKernel kf_weights = new SummationKernel(new SquaredExponential(1)) + new WhiteNoise(-3);
var K_weights = Utils.GramMatrix(kf_weights, inputs);
var D = Variable.Observed <int>(data.GetLength(0)).Named("D");
var d = new Range(D).Named("d");
var Qvar = Variable.Observed <int>(Q).Named("Q");
var q = new Range(Qvar).Named("q");
var N = Variable.Observed <int>(data.GetLength(1)).Named("N");
var n = new Range(N).Named("n");
if (missing == null)
{
missing = new bool[D.ObservedValue, N.ObservedValue]; // check this is all false
}
var ev = Variable.Bernoulli(.5).Named("ev");
var modelBlock = Variable.If(ev);
var nodeSignalPrecisions = Variable.Array <double>(q).Named("nodeSignalPrecisions");
// set this to 1 if not learning signal variance
var nodeSignalPrecisionsPrior = Variable.Observed(Enumerable.Range(0, Q).Select(_ => Gamma.FromShapeAndRate(.1, .1)).ToArray(), q).Named("nodeSignalPrecisionsPrior");
nodeSignalPrecisions[q] = Variable.Random <double, Gamma>(nodeSignalPrecisionsPrior[q]);
var nodeFunctions = Variable.Array <Vector>(q).Named("nodeFunctions");
var K_node_inverse = Variable.Observed(K_node.Inverse()).Named("K_node_inverse");
nodeFunctions[q] = Variable <Vector> .Factor(MyFactors.VectorGaussianScaled, nodeSignalPrecisions[q], K_node_inverse);
nodeFunctions.AddAttribute(new MarginalPrototype(new VectorGaussian(N.ObservedValue)));
var nodeFunctionValues = Variable.Array(Variable.Array <double>(n), q).Named("nodeFunctionValues");
var nodeFunctionValuesPredictive = Variable.Array(Variable.Array <double>(n), q).Named("nodeFunctionValuesPredictive");
VariableArray <double> nodeNoisePrecisions = null;
if (nodeFunctionNoise)
{
var nodeFunctionValuesClean = Variable.Array(Variable.Array <double>(n), q).Named("nodeFunctionValuesClean");
nodeFunctionValuesClean[q] = Variable.ArrayFromVector(nodeFunctions[q], n);
nodeNoisePrecisions = Variable.Array <double>(q).Named("nodeNoisePrecisions");
var nodeNoisePrecisionPrior = Variable.Observed(Enumerable.Range(0, Q).Select(_ => Gamma.FromShapeAndRate(.1, .01)).ToArray(), q).Named("nodeNoisePrecisionPrior");
nodeNoisePrecisions[q] = Variable.Random <double, Gamma>(nodeNoisePrecisionPrior[q]);
toInfer.Add(nodeNoisePrecisions);
nodeFunctionValues[q][n] = Variable.GaussianFromMeanAndPrecision(nodeFunctionValuesClean[q][n], nodeNoisePrecisions[q]);
nodeFunctionValuesPredictive[q][n] = Variable.GaussianFromMeanAndPrecision(nodeFunctionValuesClean[q][n], nodeNoisePrecisions[q]);
}
else
{
nodeFunctionValues[q] = Variable.ArrayFromVector(nodeFunctions[q], n);
nodeFunctionValuesPredictive[q] = Variable.ArrayFromVector(nodeFunctions[q], n);
}
var weightFunctions = Variable.Array <Vector>(d, q).Named("weightFunctions");
var K_weights_inverse = Variable.Observed(K_weights.Inverse()).Named("K_weights_inverse");
weightFunctions[d, q] = Variable <Vector> .Factor(MyFactors.VectorGaussianScaled, Variable.Constant <double>(1), K_weights_inverse).ForEach(d, q);
weightFunctions.AddAttribute(new MarginalPrototype(new VectorGaussian(N.ObservedValue)));
var weightFunctionValues = Variable.Array(Variable.Array <double>(n), d, q).Named("weightFunctionValues");
var weightFunctionValues2 = Variable.Array(Variable.Array <double>(n), d, q).Named("weightFunctionValuesPredictive");
weightFunctionValues[d, q] = Variable.ArrayFromVector(weightFunctions[d, q], n);
if (constrainWpositive)
{
var weightFunctionValuesCopy = Variable.Array(Variable.Array <double>(n), d, q).Named("weightFunctionValuesCopy");
weightFunctionValuesCopy[d, q][n] = Variable.GaussianFromMeanAndPrecision(weightFunctionValues[d, q][n], 100);
Variable.ConstrainPositive(weightFunctionValuesCopy[d, q][n]);
}
weightFunctionValues2[d, q] = Variable.ArrayFromVector(weightFunctions[d, q], n);
var observedData = Variable.Array <double>(d, n).Named("observedData");
var noisePrecisionPrior = Variable.Observed(Gamma.FromShapeAndRate(1, .1)).Named("noisePrecisionPrior");
Variable <double> noisePrecision = null;
VariableArray <double> noisePrecisionArray = null;
if (isotropicNoise)
{
noisePrecision = Variable.Random <double, Gamma>(noisePrecisionPrior).Named("noisePrecision");
toInfer.Add(noisePrecision);
}
else
{
noisePrecisionArray = Variable.Array <double>(d).Named("noisePrecision");
noisePrecisionArray[d] = Variable.Random <double, Gamma>(noisePrecisionPrior).ForEach(d);
toInfer.Add(noisePrecisionArray);
}
var isMissing = Variable.Array <bool>(d, n).Named("isMissing");
isMissing.ObservedValue = missing;
var noiseLessY = Variable.Array <double>(d, n).Named("noiseLessY");
VariableArray <VariableArray <double>, double[][]> meanFunctionValues = null;
if (meanFunctions)
{
GPFactor.settings = new Settings
{
solverMethod = Settings.SolverMethod.GradientDescent,
};
VariableArray <KernelFunction> kf = Variable.Array <KernelFunction>(d);
kf.ObservedValue = Enumerable.Range(0, D.ObservedValue).Select(
o => new SummationKernel(new SquaredExponential()) + new WhiteNoise(-3)).ToArray();
var mf = Variable.Array <Vector>(d).Named("meanFunctions");
mf[d] = Variable <Vector> .Factor <double, Vector[], int[], KernelFunction>(MyFactors.GP, 1.0 /*Variable.GammaFromShapeAndRate(1,1)*/, inputs, new int[] { 0 },
kf[d]);
mf.AddAttribute(new MarginalPrototype(new VectorGaussian(N.ObservedValue)));
meanFunctionValues = Variable.Array(Variable.Array <double>(n), d).Named("meanFunctionValues");
meanFunctionValues[d] = Variable.ArrayFromVector(mf[d], n);
toInfer.Add(meanFunctionValues);
}
using (Variable.ForEach(n))
using (Variable.ForEach(d))
{
var temp = Variable.Array <double>(q).Named("temp");
temp[q] = weightFunctionValues[d, q][n] * nodeFunctionValues[q][n];
if (meanFunctions)
{
noiseLessY[d, n] = Variable.Sum(temp) + meanFunctionValues[d][n];
}
else
{
noiseLessY[d, n] = Variable.Sum(temp);
}
using (Variable.IfNot(isMissing[d, n]))
if (isotropicNoise)
{
observedData[d, n] = Variable.GaussianFromMeanAndPrecision(noiseLessY[d, n], noisePrecision);
}
else
{
observedData[d, n] = Variable.GaussianFromMeanAndPrecision(noiseLessY[d, n], noisePrecisionArray[d]);
}
using (Variable.If(isMissing[d, n]))
observedData[d, n] = Variable.GaussianFromMeanAndPrecision(0, 1);
}
observedData.ObservedValue = data;
var nodeFunctionsInit = Enumerable.Range(0, Q).Select(i =>
VectorGaussian.FromMeanAndVariance(
VectorGaussian.Sample(Vector.Zero(N.ObservedValue), PositiveDefiniteMatrix.IdentityScaledBy(N.ObservedValue, 100)),
PositiveDefiniteMatrix.IdentityScaledBy(N.ObservedValue, 100))).ToArray(); // should put this manually in generated code
var distArray = Distribution <Vector> .Array(nodeFunctionsInit);
var nodeFunctionsInitVar = Variable.Observed(distArray).Named("nodeFunctionsInitVar");
nodeFunctions.InitialiseTo(nodeFunctionsInitVar);
modelBlock.CloseBlock();
toInfer.AddRange(new List <IVariable>()
{
ev, noiseLessY, nodeFunctionValues, nodeSignalPrecisions, nodeFunctionValuesPredictive, weightFunctionValues, weightFunctionValues2
});
var infer = new InferenceEngine(new VariationalMessagePassing());
infer.ModelName = "MeanFunction";
var ca = infer.GetCompiledInferenceAlgorithm(toInfer.ToArray());
var kernel = new SummationKernel(new SquaredExponential(initLoglengthscales[0]));
kernel += new WhiteNoise(-3);
ca.SetObservedValue(K_node_inverse.NameInGeneratedCode, Utils.GramMatrix(kernel, inputs).Inverse());
kernel = new SummationKernel(new SquaredExponential(initLoglengthscales[1]));
kernel += new WhiteNoise(-3);
ca.SetObservedValue(K_weights_inverse.NameInGeneratedCode, Utils.GramMatrix(kernel, inputs).Inverse());
ca.Reset();
double oldML = double.NegativeInfinity;
double ml = 0;
int it = 0;
for (; it < 100; it++)
{
ca.Update(1);
ml = ca.Marginal <Bernoulli>(ev.NameInGeneratedCode).LogOdds;
Console.WriteLine(ml);
if (Math.Abs(oldML - ml) < .1)
{
break;
}
oldML = ml;
}
Console.WriteLine("Finished after " + it);
}
/// <summary>
/// Infer.NET definition of the Semi Parametric Latent Factor Model of
/// Teh, Y., Seeger, M., and Jordan, M. (AISTATS 2005).
/// </summary>
/// <param name="inputs">Covariates X</param>
/// <param name="data">Outputs Y</param>
/// <param name="Q">Number of latent functions</param>
/// <param name="missing">Which elements of Y are missing</param>
/// <param name="nodeFunctionNoise">Whether to include node noise</param>
public void SPLFM(
Vector[] inputs,
double[,] data,
int Q,
bool[,] missing = null,
bool nodeFunctionNoise = false)
{
var toInfer = new List <IVariable>();
SummationKernel kf_node = new SummationKernel(new SquaredExponential(0));
var K_node = Utils.GramMatrix(kf_node, inputs);
var D = Variable.Observed <int>(data.GetLength(0)).Named("D");
var d = new Range(D).Named("d");
var Qvar = Variable.Observed <int>(Q).Named("Q");
var q = new Range(Qvar).Named("q");
var N = Variable.Observed <int>(data.GetLength(1)).Named("N");
var n = new Range(N).Named("n");
if (missing == null)
{
missing = new bool[D.ObservedValue, N.ObservedValue]; // check this is all false
}
var ev = Variable.Bernoulli(.5).Named("ev");
var modelBlock = Variable.If(ev);
var nodeSignalPrecisions = Variable.Array <double>(q).Named("nodeSignalPrecisions");
// set this to 1 if not learning signal variance
var nodeSignalPrecisionsPrior = Variable.Observed(Enumerable.Range(0, Q).Select(_ => Gamma.FromShapeAndRate(.1, .1)).ToArray(), q).Named("nodeSignalPrecisionsPrior");
nodeSignalPrecisions[q] = Variable.Random <double, Gamma>(nodeSignalPrecisionsPrior[q]);
var nodeFunctions = Variable.Array <Vector>(q).Named("nodeFunctions");
var K_node_inverse = Variable.Observed(K_node.Inverse()).Named("K_node_inverse");
nodeFunctions[q] = Variable <Vector> .Factor(MyFactors.VectorGaussianScaled, nodeSignalPrecisions[q], K_node_inverse);
nodeFunctions.AddAttribute(new MarginalPrototype(new VectorGaussian(N.ObservedValue)));
var nodeFunctionValues = Variable.Array(Variable.Array <double>(n), q).Named("nodeFunctionValues");
var nodeFunctionValuesPredictive = Variable.Array(Variable.Array <double>(n), q).Named("nodeFunctionValuesPredictive");
VariableArray <double> nodeNoisePrecisions = null;
if (nodeFunctionNoise)
{
var nodeFunctionValuesClean = Variable.Array(Variable.Array <double>(n), q).Named("nodeFunctionValuesClean");
nodeFunctionValuesClean[q] = Variable.ArrayFromVector(nodeFunctions[q], n);
nodeNoisePrecisions = Variable.Array <double>(q).Named("nodeNoisePrecisions");
var nodeNoisePrecisionPrior = Variable.Observed(Enumerable.Range(0, Q).Select(_ => Gamma.FromShapeAndRate(.1, .01)).ToArray(), q).Named("nodeNoisePrecisionPrior");
nodeNoisePrecisions[q] = Variable.Random <double, Gamma>(nodeNoisePrecisionPrior[q]);
toInfer.Add(nodeNoisePrecisions);
nodeFunctionValues[q][n] = Variable.GaussianFromMeanAndPrecision(nodeFunctionValuesClean[q][n], nodeNoisePrecisions[q]);
nodeFunctionValuesPredictive[q][n] = Variable.GaussianFromMeanAndPrecision(nodeFunctionValuesClean[q][n], nodeNoisePrecisions[q]);
}
else
{
nodeFunctionValues[q] = Variable.ArrayFromVector(nodeFunctions[q], n);
nodeFunctionValuesPredictive[q] = Variable.ArrayFromVector(nodeFunctions[q], n);
}
var weights = Variable.Array <double>(d, q).Named("weights");
weights[d, q] = Variable.GaussianFromMeanAndPrecision(0, 1).ForEach(d, q);
var observedData = Variable.Array <double>(d, n).Named("observedData");
var noisePrecisionPrior = Variable.Observed(Gamma.FromShapeAndRate(1, .1)).Named("noisePrecisionPrior");
var noisePrecision = Variable.Random <double, Gamma>(noisePrecisionPrior).Named("noisePrecision");
var isMissing = Variable.Array <bool>(d, n).Named("isMissing");
isMissing.ObservedValue = missing;
var noiseLessY = Variable.Array <double>(d, n).Named("noiseLessY");
using (Variable.ForEach(n))
using (Variable.ForEach(d))
{
var temp = Variable.Array <double>(q).Named("temp");
temp[q] = weights[d, q] * nodeFunctionValues[q][n];
noiseLessY[d, n] = Variable.Sum(temp);
using (Variable.IfNot(isMissing[d, n]))
observedData[d, n] = Variable.GaussianFromMeanAndPrecision(noiseLessY[d, n], noisePrecision);
using (Variable.If(isMissing[d, n]))
observedData[d, n] = Variable.GaussianFromMeanAndPrecision(0, 1);
}
observedData.ObservedValue = data;
var nodeFunctionsInit = Enumerable.Range(0, Q).Select(i =>
VectorGaussian.FromMeanAndVariance(
VectorGaussian.Sample(Vector.Zero(N.ObservedValue), PositiveDefiniteMatrix.IdentityScaledBy(N.ObservedValue, 100)),
PositiveDefiniteMatrix.IdentityScaledBy(N.ObservedValue, 100))).ToArray(); // should put this manually in generated code
var distArray = Distribution <Vector> .Array(nodeFunctionsInit);
var nodeFunctionsInitVar = Variable.Observed(distArray).Named("nodeFunctionsInitVar");
nodeFunctions.InitialiseTo(nodeFunctionsInitVar);
modelBlock.CloseBlock();
toInfer.AddRange(new List <IVariable>()
{
ev, noiseLessY, noisePrecision, nodeFunctionValues, nodeSignalPrecisions, nodeFunctionValuesPredictive, weights
});
var ie = new InferenceEngine(new VariationalMessagePassing());
ie.ModelName = "SPLFM";
var ca = ie.GetCompiledInferenceAlgorithm(toInfer.ToArray());
ca.Execute(100);
var fvals = ca.Marginal <Gaussian[][]>(nodeFunctionValues.NameInGeneratedCode)[0]; // [q][n]
var x = inputs.Select(i => i[0]).ToArray();
var mplWrapper = new MatplotlibWrapper();
mplWrapper.AddArray("x", x);
mplWrapper.AddArray("y", fvals.Select(i => i.GetMean()).ToArray());
mplWrapper.AddArray("s", fvals.Select(i => Math.Sqrt(i.GetVariance())).ToArray());
mplWrapper.Plot(new string[] {
"fill_between(x,y-s,y+s,color=\"gray\")",
"ylabel(\"node (fitted)\")"
});
}
