public SoftmaxCategoricalOneParentNonShared(ModelNodeNonShared node)
{
this.node = node;
Range parentRange = null;
foreach (var parent in node.parents)
{
if (parent.distributionType == DistributionType.Categorical)
{
parentRange = parent.states;
}
}
Bprior = Variable.Array(Variable.Array <VectorGaussian>(parentRange), node.states).Named(node.name + "CoefficientsPrior");
Bprior.ObservedValue = (node.original.distributions as SoftmaxOneParentCategorical).Bprior;
B = Variable.Array(Variable.Array <Vector>(parentRange), node.states).Named(node.name + "B");
B[node.states][parentRange] = Variable <Vector> .Random(Bprior[node.states][parentRange]);
mPrior = Variable.Array(Variable.Array <Gaussian>(parentRange), node.states).Named(node.name + "mPrior");
mPrior.ObservedValue = (node.original.distributions as SoftmaxOneParentCategorical).mPrior;
m = Variable.Array(Variable.Array <double>(parentRange), node.states).Named(node.name + "m");
m[node.states][parentRange] = Variable <double> .Random(mPrior[node.states][parentRange]);
Variable.ConstrainEqualRandom(B[node.states.SizeAsInt - 1][parentRange], VectorGaussian.PointMass(Vector.Zero(node.parents.Count - 1)));
Variable.ConstrainEqualRandom(m[node.states.SizeAsInt - 1][parentRange], Gaussian.PointMass(0));
}
private void GaussianProductOp_APointMass(double aMean, Gaussian Product, Gaussian B)
{
bool isProper = Product.IsProper();
Gaussian A = Gaussian.PointMass(aMean);
Gaussian result = GaussianProductOp.AAverageConditional(Product, A, B);
Console.WriteLine("{0}: {1}", A, result);
Gaussian result2 = isProper ? GaussianProductOp_Slow.AAverageConditional(Product, A, B) : result;
Console.WriteLine("{0}: {1}", A, result2);
Assert.True(result.MaxDiff(result2) < 1e-6);
var Amsg = InnerProductOp_PointB.BAverageConditional(Product, DenseVector.FromArray(B.GetMean()), new PositiveDefiniteMatrix(new double[, ] {
{ B.GetVariance() }
}), VectorGaussian.PointMass(aMean), VectorGaussian.Uniform(1));
//Console.WriteLine("{0}: {1}", A, Amsg);
Assert.True(result.MaxDiff(Amsg.GetMarginal(0)) < 1e-6);
double prevDiff = double.PositiveInfinity;
for (int i = 3; i < 40; i++)
{
double v = System.Math.Pow(0.1, i);
A = Gaussian.FromMeanAndVariance(aMean, v);
result2 = isProper ? GaussianProductOp.AAverageConditional(Product, A, B) : result;
double diff = result.MaxDiff(result2);
Console.WriteLine("{0}: {1} diff={2}", A, result2, diff.ToString("g4"));
//Assert.True(diff <= prevDiff || diff < 1e-6);
result2 = isProper ? GaussianProductOp_Slow.AAverageConditional(Product, A, B) : result;
diff = result.MaxDiff(result2);
Console.WriteLine("{0}: {1} diff={2}", A, result2, diff.ToString("g4"));
Assert.True(diff <= prevDiff || diff < 1e-6);
prevDiff = diff;
}
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="SumVectorGaussianOp"]/message_doc[@name="ArrayAverageConditional{TVectorGaussianList}(Vector, IList{VectorGaussian}, TVectorGaussianList)"]/*'/>
/// <typeparam name="TVectorGaussianList">A list of <see cref="VectorGaussian"/> distributions which may be set to uniform.</typeparam>
public static TVectorGaussianList ArrayAverageConditional <TVectorGaussianList>(
[SkipIfUniform] Vector sum,
IList <VectorGaussian> array,
TVectorGaussianList result)
where TVectorGaussianList : IList <VectorGaussian>, SettableToUniform
{
if (sum == null)
{
throw new ArgumentNullException(nameof(sum));
}
VectorGaussian to_sum = SumAverageConditional(array, new VectorGaussian(sum.Count));
return(ArrayAverageConditional(VectorGaussian.PointMass(sum), to_sum, array, result));
}
//Addition by Guy Templeton, get log evidence from learned mixing coeff.
public double GetLogEvidence()
{
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
Range classes = new Range(numOfClasses);
IfBlock block = Variable.If(evidence);
VectorGaussian[] wObserved = trainModel.wPrior.ObservedValue;
VectorGaussian[] empty = Util.ArrayInit(numOfClasses, c => (c == 0) ?
VectorGaussian.PointMass(Vector.Zero(numOfFeatures)) :
VectorGaussian.FromMeanAndPrecision(Vector.Zero(numOfFeatures), PositiveDefiniteMatrix.Identity(numOfFeatures)));
block.CloseBlock();
InferenceEngine engine = new InferenceEngine();
return(engine.Infer <Bernoulli>(evidence).LogOdds);
}
public SoftmaxCategoricalNonShared(ModelNodeNonShared node)
{
this.node = node;
Bprior = Variable.Array <VectorGaussian>(node.states).Named(node.name + "CoefficientsPrior");
Bprior.ObservedValue = (node.original.distributions as SoftmaxCategorical).Bprior;
B = Variable.Array <Vector>(node.states).Named(node.name + "Coefficients");
// The weight vector for each class.
B[node.states] = Variable <Vector> .Random(Bprior[node.states]);
mPrior = Variable.Array <Gaussian>(node.states).Named(node.name + "BiasPrior");
mPrior.ObservedValue = (node.original.distributions as SoftmaxCategorical).mPrior;
m = Variable.Array <double>(node.states).Named(node.name + "Bias");
m[node.states] = Variable <double> .Random(mPrior[node.states]);
Variable.ConstrainEqualRandom(B[node.states.SizeAsInt - 1], VectorGaussian.PointMass(Vector.Zero(node.parents.Count)));
Variable.ConstrainEqualRandom(m[node.states.SizeAsInt - 1], Gaussian.PointMass(0));
}
/// <summary>
/// Build and run a multinomial regression model.
/// </summary>
/// <param name="xObs">An array of vectors of observed inputs.
/// The length of the array is the number of samples, and the
/// length of the vectors is the number of input features. </param>
/// <param name="yObs">An array of array of counts, where the first index is the sample,
/// and the second index is the class. </param>
/// <param name="bPost">The returned posterior over the coefficients.</param>
/// <param name="meanPost">The returned posterior over the means.</param>
public void MultinomialRegression(Vector[] xObs, int[][] yObs,
out VectorGaussian[] bPost, out Gaussian[] meanPost)
{
int C = yObs[0].Length;
int N = xObs.Length;
int K = xObs[0].Count;
var c = new Range(C).Named("c");
var n = new Range(N).Named("n");
// model
var B = Variable.Array <Vector>(c).Named("coefficients");
B[c] = Variable.VectorGaussianFromMeanAndPrecision(
Vector.Zero(K), PositiveDefiniteMatrix.Identity(K)).ForEach(c);
var m = Variable.Array <double>(c).Named("mean");
m[c] = Variable.GaussianFromMeanAndPrecision(0, 1).ForEach(c);
Variable.ConstrainEqualRandom(B[C - 1], VectorGaussian.PointMass(Vector.Zero(K)));
Variable.ConstrainEqualRandom(m[C - 1], Gaussian.PointMass(0));
var x = Variable.Array <Vector>(n);
x.ObservedValue = xObs;
var yData = Variable.Array(Variable.Array <int>(c), n);
yData.ObservedValue = yObs;
var trialsCount = Variable.Array <int>(n);
trialsCount.ObservedValue = yObs.Select(o => o.Sum()).ToArray();
var g = Variable.Array(Variable.Array <double>(c), n);
g[n][c] = Variable.InnerProduct(B[c], x[n]) + m[c];
var p = Variable.Array <Vector>(n);
p[n] = Variable.Softmax(g[n]);
using (Variable.ForEach(n))
yData[n] = Variable.Multinomial(trialsCount[n], p[n]);
// inference
// var ie = new InferenceEngine(new VariationalMessagePassing());
var ie = new InferenceEngine(new ExpectationPropagation());
// ie.Compiler.GivePriorityTo(typeof(SoftmaxOp_KM11_Sparse));
bPost = ie.Infer <VectorGaussian[]>(B);
meanPost = ie.Infer <Gaussian[]>(m);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="SumVectorGaussianOp"]/message_doc[@name="ArrayAverageLogarithm{TVectorGaussianList}(Vector, IList{VectorGaussian}, TVectorGaussianList)"]/*'/>
/// <typeparam name="TVectorGaussianList">A list of <see cref="VectorGaussian"/> distributions.</typeparam>
public static TVectorGaussianList ArrayAverageLogarithm <TVectorGaussianList>(
Vector sum, [Proper] IList <VectorGaussian> array, TVectorGaussianList result)
where TVectorGaussianList : IList <VectorGaussian>
{
return(ArrayAverageLogarithm(VectorGaussian.PointMass(sum), array, result));
}
/// <summary>
/// Build and run a multinomial regression model.
/// </summary>
/// <param name="xObs">An array of vectors of observed inputs.
/// The length of the array is the number of samples, and the
/// length of the vectors is the number of input features. </param>
/// <param name="yObs">An array of array of counts, where the first index is the sample,
/// and the second index is the class. </param>
/// <param name="weightsPost">The returned posterior over the coefficients.</param>
/// <param name="biasPost">The returned posterior over the means.</param>
public double MultinomialRegression(Vector[] xObs, int[][] yObs, out IList <VectorGaussian> weightsPost, out IList <Gaussian> biasPost, bool trackLowerBound = true)
{
int nClasses = yObs[0].Length;
int nPoints = xObs.Length;
int dimension = xObs[0].Count;
var c = new Range(nClasses).Named("c");
var n = new Range(nPoints).Named("n");
Variable <bool> ev = null;
IfBlock model = null;
if (trackLowerBound)
{
ev = Variable.Bernoulli(0.5).Named("evidence");
model = Variable.If(ev);
}
// model
var weights = Variable.Array <Vector>(c).Named("weights");
weights[c] = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(dimension), PositiveDefiniteMatrix.Identity(dimension)).ForEach(c);
var bias = Variable.Array <double>(c).Named("bias");
bias[c] = Variable.GaussianFromMeanAndPrecision(0, 1).ForEach(c);
Variable.ConstrainEqualRandom(weights[nClasses - 1], VectorGaussian.PointMass(Vector.Zero(dimension)));
Variable.ConstrainEqualRandom(bias[nClasses - 1], Gaussian.PointMass(0));
var x = Variable.Array <Vector>(n).Named("x");
x.ObservedValue = xObs;
var yData = Variable.Array(Variable.Array <int>(c), n).Named("y");
yData.ObservedValue = yObs;
var trialCounts = Variable.Array <int>(n).Named("trialCounts");
trialCounts.ObservedValue = yObs.Select(o => o.Sum()).ToArray();
var g = Variable.Array(Variable.Array <double>(c), n).Named("g");
g[n][c] = Variable.InnerProduct(weights[c], x[n]) + bias[c];
var p = Variable.Array <Vector>(n).Named("p");
p[n] = Variable.Softmax(g[n]);
using (Variable.ForEach(n))
yData[n] = Variable.Multinomial(trialCounts[n], p[n]);
if (trackLowerBound)
{
model.CloseBlock();
}
// inference
var ie = new InferenceEngine(new VariationalMessagePassing());
//ie.Compiler.GivePriorityTo(typeof(SaulJordanSoftmaxOp_LBFGS));
//ie.Compiler.GivePriorityTo(typeof(ProductOfLogisticsSoftmaxOp));
//ie.Compiler.GivePriorityTo(typeof(SaulJordanSoftmaxOp_NCVMP));
//ie.Compiler.GivePriorityTo(typeof(SoftmaxOp_Bouchard));
//ie.Compiler.GivePriorityTo(typeof(Blei06SoftmaxOp_NCVMP));
ie.OptimiseForVariables = (trackLowerBound ? new IVariable[] { weights, bias, ev } : new IVariable[] { weights, bias });
if (trackLowerBound)
{
ie.ShowProgress = false;
}
for (int iter = 1; iter <= 50; iter++)
{
ie.NumberOfIterations = iter;
if (trackLowerBound)
{
Console.WriteLine(ie.Infer <Bernoulli>(ev).LogOdds);
}
}
weightsPost = ie.Infer <IList <VectorGaussian> >(weights);
biasPost = ie.Infer <IList <Gaussian> >(bias);
return(trackLowerBound ? ie.Infer <Bernoulli>(ev).LogOdds : 0.0);
}
public int MulticlassRegression(Vector[] xObs, int[] yObs, int C, out VectorGaussian[] bPost, out Gaussian[] meanPost, out double lowerBound, object softmaxOperator,
bool trackLowerBound = false)
{
int N = xObs.Length;
int K = xObs[0].Count;
var c = new Range(C).Named("c");
var n = new Range(N).Named("n");
Variable <bool> ev = null;
IfBlock model = null;
if (trackLowerBound)
{
ev = Variable.Bernoulli(0.5).Named("evidence");
model = Variable.If(ev);
}
// model
var B = Variable.Array <Vector>(c).Named("coefficients");
B[c] = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(K), PositiveDefiniteMatrix.Identity(K)).ForEach(c);
var m = Variable.Array <double>(c).Named("mean");
m[c] = Variable.GaussianFromMeanAndPrecision(0, 1).ForEach(c);
Variable.ConstrainEqualRandom(B[C - 1], VectorGaussian.PointMass(Vector.Zero(K)));
Variable.ConstrainEqualRandom(m[C - 1], Gaussian.PointMass(0));
var x = Variable.Array <Vector>(n);
x.ObservedValue = xObs;
var yData = Variable.Array <int>(n);
yData.ObservedValue = yObs;
var g = Variable.Array(Variable.Array <double>(c), n);
g[n][c] = Variable.InnerProduct(B[c], x[n]) + m[c];
var p = Variable.Array <Vector>(n);
p[n] = Variable.Softmax(g[n]);
using (Variable.ForEach(n))
yData[n] = Variable.Discrete(p[n]);
if (trackLowerBound)
{
model.CloseBlock();
}
// inference
var ie = new InferenceEngine(new VariationalMessagePassing());
ie.Compiler.GivePriorityTo(softmaxOperator);
var ca = ie.GetCompiledInferenceAlgorithm(ev, B, m);
var start = DateTime.Now;
var initBound = double.NegativeInfinity;
int i = 0;
lowerBound = 0;
for (i = 1; i <= 50; i++)
{
ca.Update(1);
lowerBound = ca.Marginal <Bernoulli>(ev.NameInGeneratedCode).LogOdds;
Console.WriteLine(i + "," + lowerBound + "," + (DateTime.Now - start).TotalMilliseconds);
if (System.Math.Abs(initBound - lowerBound) < 1e-2)
{
break;
}
initBound = lowerBound;
start = DateTime.Now;
}
bPost = ca.Marginal <VectorGaussian[]>(B.NameInGeneratedCode);
meanPost = ca.Marginal <Gaussian[]>(m.NameInGeneratedCode);
return(i);
}
private VectorGaussian[] InitializePrior(int numClasses, int numFeatures)
{
return(Util.ArrayInit(numClasses, c => (c == 0) ?
VectorGaussian.PointMass(Vector.Zero(numFeatures)) :
VectorGaussian.FromMeanAndPrecision(Vector.Zero(numFeatures), PositiveDefiniteMatrix.Identity(numFeatures))));
}
public static void Test2()
{
double[] observed = new double[] { 21, 27, 41, 35, 47, 54, 59, 19, 19, 16, 23, 35, 19, 30, 31, 37, 66, 20, 27, 25, 35, 28, 62, 31,
49, 40, 31, 14, 20, 37, 38, 25, 31, 39 };
double[] pathObserved = new double[] { 22, 24, 32, 50, 62, 73, 69, 19, 13, 8, 23, 27, 32, 22,
25, 39, 39, 17, 22, 24, 26, 26, 53, 26, 36, 38, 34, 19, 20, 45, 37, 21, 45, 41 };
int[] classes = new int[] { 2, 2, 3, 5, 6, 7, 6, 1, 1, 0, 2, 2, 3, 2,
2, 3, 3, 1, 2, 2, 2, 2, 5, 2, 3, 3, 3, 1, 2, 4, 3, 2, 4, 4 };
int[][] classesCounts = new int[classes.Length][];
for (int ii = 0; ii < classes.Length; ii++)
{
classesCounts[ii] = new int[8];
classesCounts[ii][classes[ii]] = 1;
}
Vector[] xdata = new Vector[observed.Length];
for (int i = 0; i < xdata.Length; i++)
{
xdata[i] = Vector.FromArray(observed[i], pathObserved[i]);
}
Range range = new Range(observed.Length);
Variable <Gaussian> meanPrior = Gaussian.FromMeanAndVariance(0, 100);
Variable <double> roomsMean = Variable <double> .Random(meanPrior).Named("room Mean");
Variable <double> roomsStd = Variable.GammaFromMeanAndVariance(2, 2).Named("room sigma");
VariableArray <double> observedVar = Variable.Array <double>(range).Named("observedVar");
Variable <double> roomGauss = Variable.GaussianFromMeanAndPrecision(roomsMean, roomsStd).Named("roomGauss");
observedVar[range] = roomGauss.ForEach(range);
Variable <double> pathsMean = Variable.GaussianFromMeanAndVariance(0, 200).Named("path Mean");
Variable <double> pathsStd = Variable.GammaFromMeanAndVariance(1, 1.5).Named("path sigma");
VariableArray <double> observedPath = Variable.Array <double>(range).Named("observedPath");
observedPath[range] = observedVar[range] + (Variable.GaussianFromMeanAndPrecision(pathsMean, pathsStd)).ForEach(range);
observedVar.ObservedValue = observed;
observedPath.ObservedValue = pathObserved;
int C = 8;
var c = new Range(C);
int K = 2;
var B = Variable.Array <Vector>(c).Named("coefficients");
B[c] = Variable.VectorGaussianFromMeanAndPrecision(
Vector.Zero(K), PositiveDefiniteMatrix.Identity(K)).ForEach(c);
var m = Variable.Array <double>(c).Named("mean");
m[c] = Variable.GaussianFromMeanAndPrecision(0, 1).ForEach(c);
Variable.ConstrainEqualRandom(B[C - 1], VectorGaussian.PointMass(Vector.Zero(K)));
Variable.ConstrainEqualRandom(m[C - 1], Gaussian.PointMass(0));
var x = Variable.Array <Vector>(range).Named("x");
x.ObservedValue = xdata;
//var x = Variable.Array(Variable.Array<double>(c), range).Named("x");
// Variable.ConstrainEqual(x[range][0], observedVar[range]);
// Variable.ConstrainEqual(x[range][1], observedPath[range]);
// x[range][1] = observedPath[range];
// var x = xTemp;// Variable.Vector(xTemp[range]);
var yData = Variable.Array(Variable.Array <int>(c), range).Named("y");
yData.ObservedValue = classesCounts;
var trialsCount = Variable.Array <int>(range).Named("trialsCount");
trialsCount.ObservedValue = classesCounts.Select(o => o.Sum()).ToArray();
var g = Variable.Array(Variable.Array <double>(c), range).Named("g");
g[range][c] = Variable.InnerProduct(B[c], (x[range])) + m[c];
var p = Variable.Array <Vector>(range).Named("p");
p[range] = Variable.Softmax(g[range]);
using (Variable.ForEach(range))
yData[range] = Variable.Multinomial(trialsCount[range], p[range]);
// inference
// var ie = new InferenceEngine(new VariationalMessagePassing());
InferenceEngine Engine = new InferenceEngine(new VariationalMessagePassing());
Gaussian posterior = Engine.Infer <Gaussian>(roomsMean);
Gamma posteriorStd = Engine.Infer <Gamma>(roomsStd);
Gaussian posteriorPath = Engine.Infer <Gaussian>(pathsMean);
Gamma posteriorStdPath = Engine.Infer <Gamma>(pathsStd);
Console.WriteLine(posterior + " " + posteriorStd);
Console.WriteLine(posteriorPath + " " + posteriorStdPath);
var bPost = Engine.Infer <VectorGaussian[]>(B);
var meanPost = Engine.Infer <Gaussian[]>(m);
for (int i = 0; i < C; i++)
{
Console.WriteLine("C " + bPost[i].GetMean());
Console.WriteLine("m " + meanPost[i].GetMean());
}
for (int ii = 0; ii < classes.Length; ii++)
{
for (int i = 0; i < C; i++)
{
Console.WriteLine("C " + i + " " + (meanPost[i].GetMean() + bPost[i].GetMean()[0] * observed[ii] + bPost[i].GetMean()[1] * pathObserved[ii]) + " - " + classes[ii]);
}
}
// Engine.Compiler.GivePriorityTo(typeof(SaulJordanSoftmaxOp_NCVMP));
}