private static void Test_BPMIncremental(
int nClass, int totalFeatures, double noisePrec, int maxItemsInBatch,
int nChunks, string trainingFile, Vector[] testData)
{
Console.WriteLine("\n------- BPM Train Incremental -------");
VectorGaussian[] wInfer = new VectorGaussian[nClass];
BPM bpmIncremental = new BPM(nClass, totalFeatures, noisePrec);
bpmIncremental.TrainingEngine.ShowProgress = false;
bpmIncremental.TestEngine.ShowProgress = false;
int LocToStart = 0;
for (int c = 0; c < nChunks; c++)
{
List<Vector>[] dataChunks = DataFromFile.Read(trainingFile, nClass, maxItemsInBatch, ref LocToStart);
wInfer = bpmIncremental.TrainIncremental(dataChunks);
}
#if ShowWeights
for (int i = 0; i < wInfer.GetLength(0); i++)
{
Console.WriteLine(wInfer[i].ToString());
}
#endif
Console.WriteLine("\nPredictions:");
Discrete[] predictions = bpmIncremental.Test(testData);
foreach (Discrete pred in predictions)
Console.WriteLine(pred);
Console.WriteLine();
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="concat">Constant value for 'concat'.</param>
/// <param name="first">Constant value for 'first'.</param>
/// <param name="second">Incoming message from 'second'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(second) p(second) factor(concat,first,second))</c>.
/// </para></remarks>
public static double LogAverageFactor(Vector concat, Vector first, VectorGaussian second)
{
for (int i = 0; i < first.Count; i++) {
if (concat[i] != first[i]) return Double.NegativeInfinity;
}
Vector concat2 = Vector.Subvector(concat, first.Count, second.Dimension);
return second.GetLogProb(concat2);
}
/// <summary>
/// EP message to 'B'
/// </summary>
/// <param name="sum">Incoming message from 'Sum'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="A">Constant value for 'A'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'B' as the random arguments are varied.
/// The formula is <c>proj[p(B) sum_(Sum) p(Sum) factor(Sum,A,B)]/p(B)</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="sum"/> is not a proper distribution</exception>
public static VectorGaussian BAverageConditional([SkipIfUniform] Gaussian sum, bool[] A, VectorGaussian result)
{
if (result == default(VectorGaussian)) result = new VectorGaussian(A.Length);
// (m - a'b)^2/v = (a'bb'a - 2a'bm + m^2)/v
var ma = Vector.FromArray(A.Select(x => x ? 1.0 : 0.0).ToArray());
result.Precision.SetToOuter(ma, ma);
result.Precision.Scale(sum.Precision);
result.MeanTimesPrecision.SetToProduct(ma, sum.MeanTimesPrecision);
return result;
}
/// <summary>
/// Test the test data.
/// </summary>
/// <param name="priorValue">Prior distribution from training.</param>
/// <param name="testData">The test data.</param>
/// <returns>The prediction.</returns>
public Discrete[] Test(VectorGaussian[] priorValue, Vector[] testData)
{
// Set the prior of all classes
for (int i = 0; i < numOfClasses; i++)
{
this.classes[i].Prior.ObservedValue = priorValue[i];
}
// Set the observed test data
this.numOfVectors.ObservedValue = testData.Length;
this.featureVectors.ObservedValue = testData;
// Infer the test model outputs
return Distribution.ToArray<Discrete[]>(this.Engine.Infer(this.modelOutput));
}
public static VectorGaussianWishart Combine(VectorGaussian position, Wishart orientation, VectorGaussianWishart result)
{
if (orientation.IsUniform())
{
result.SetToUniform();
}
else if (position.IsUniform())
{
result.SetTo(orientation.Shape, orientation.Rate, Vector.Zero(2), 0);
}
else
{
PositiveDefiniteMatrix rateTimesPrecision = new PositiveDefiniteMatrix(2, 2);
rateTimesPrecision.SetToProduct(orientation.Rate, position.Precision);
double trace = MathHelpers.Invert(rateTimesPrecision).Trace();
Vector positionMean = position.MeanTimesPrecision * MathHelpers.Invert(position.Precision);
result.SetTo(orientation.Shape, orientation.Rate, positionMean, orientation.Dimension / (orientation.Shape * trace));
}
return result;
}
public static VectorGaussian RotateAverageLogarithm([SkipIfUniform] Gaussian x, [SkipIfUniform] Gaussian y, [Proper] WrappedGaussian angle, VectorGaussian result)
{
// for x ~ N(m,v):
// E[cos(x)] = cos(m)*exp(-v/2)
// E[sin(x)] = sin(m)*exp(-v/2)
if (angle.Period != 2*Math.PI) throw new ArgumentException("angle.Period ("+angle.Period+") != 2*PI ("+2*Math.PI+")");
double angleMean, angleVar;
angle.Gaussian.GetMeanAndVariance(out angleMean, out angleVar);
double expHalfVar = Math.Exp(-0.5*angleVar);
double mCos = Math.Cos(angleMean)*expHalfVar;
double mSin = Math.Sin(angleMean)*expHalfVar;
double mCos2 = mCos*mCos;
double mSin2 = mSin*mSin;
// E[cos(x)^2] = 0.5 E[1+cos(2x)] = 0.5 (1 + cos(2m) exp(-2v))
// E[sin(x)^2] = E[1 - cos(x)^2] = 0.5 (1 - cos(2m) exp(-2v))
double expVar = expHalfVar*expHalfVar;
// cos2m = cos(2m)*exp(-v)
double cos2m = 2*mCos2 - expVar;
double mCosSqr = 0.5*(1 + cos2m*expVar);
double mSinSqr = 1 - mCosSqr;
double mSinCos = mSin*mCos*expVar;
if (result.Dimension != 2) throw new ArgumentException("result.Dimension ("+result.Dimension+") != 2");
double mx, vx, my, vy;
x.GetMeanAndVariance(out mx, out vx);
y.GetMeanAndVariance(out my, out vy);
Vector mean = Vector.Zero(2);
mean[0] = mCos*mx - mSin*my;
mean[1] = mSin*mx + mCos*my;
double mx2 = mx*mx + vx;
double my2 = my*my + vy;
double mxy = mx*my;
PositiveDefiniteMatrix variance = new PositiveDefiniteMatrix(2, 2);
variance[0, 0] = mx2*mCosSqr - 2*mxy*mSinCos + my2*mSinSqr - mean[0]*mean[0];
variance[0, 1] = (mx2 - my2)*mSinCos + mxy*(mCosSqr - mSinSqr) - mean[0]*mean[1];
variance[1, 0] = variance[0, 1];
variance[1, 1] = mx2*mSinSqr + 2*mxy*mSinCos + my2*mCosSqr - mean[1]*mean[1];
result.SetMeanAndVariance(mean, variance);
return result;
}
private static IEnumerable<Vector> CreateTestData(int dimensions, int clusters)
{
var dimRange = new Range(dimensions);
var clusterRange = new Range(clusters);
var trueMeans = Variable.Array<Vector>(clusterRange);
var trueCovariance = Variable.Array<PositiveDefiniteMatrix>(clusterRange);
var trueVectorGaussian = new VectorGaussian[clusters];
for (var i = 0; i < clusters; i++)
{
var trueMeansVector = Vector.FromArray(CreateRandomArray(dimensions));
Console.WriteLine("True means {0}", trueMeansVector);
trueVectorGaussian[i] = VectorGaussian.FromMeanAndPrecision(trueMeansVector, PositiveDefiniteMatrix.Identity(dimensions));
}
var vectorGaussians = Variable.Array<VectorGaussian>(clusterRange);
var dirichletMixture = Dirichlet.Uniform(clusters).Sample();
Console.WriteLine("True mixture {0}", dirichletMixture);
while (true)
{
var index = Rand.Sample(dirichletMixture);
yield return trueVectorGaussian[index].Sample();
}
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="concat">Constant value for 'concat'.</param>
/// <param name="first">Incoming message from 'first'.</param>
/// <param name="second">Incoming message from 'second'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(first,second) p(first,second) factor(concat,first,second))</c>.
/// </para></remarks>
public static double LogAverageFactor(Vector concat, VectorGaussian first, VectorGaussian second)
{
Vector concat1 = Vector.Subvector(concat, 0, first.Dimension);
Vector concat2 = Vector.Subvector(concat, first.Dimension, second.Dimension);
return first.GetLogProb(concat1) + second.GetLogProb(concat2);
}
public static Vector BMeanInit([IgnoreDependency] VectorGaussian B)
{
return(Vector.Zero(B.Dimension));
}
/// <summary>
/// VMP message to 'B'
/// </summary>
/// <param name="sum">Incoming message from 'Sum'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="A">Incoming message from 'A'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the exponential of the average log-factor value, where the average is over all arguments except 'B'.
/// Because the factor is deterministic, 'Sum' is integrated out before taking the logarithm.
/// The formula is <c>exp(sum_(A) p(A) log(sum_Sum p(Sum) factor(Sum,A,B)))</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="sum"/> is not a proper distribution</exception>
public static VectorGaussian BAverageLogarithm([SkipIfUniform] Gaussian sum, DistributionStructArray<Bernoulli, bool> A, VectorGaussian result)
{
if (result == default(VectorGaussian)) result = new VectorGaussian(A.Count);
// E[log N(x; ab, 0)] = -0.5 E[(x-ab)^2]/0 = -0.5 (E[x^2] - 2 E[x] a' E[b] + trace(aa' E[bb']))/0
// message to a = N(a; E[x]*inv(var(b)+E[b]E[b]')*E[b], var(x)*inv(var(b)+E[b]E[b]'))
// result.Precision = (var(b)+E[b]*E[b]')/var(x)
// result.MeanTimesPrecision = E[x]/var(x)*E[b] = E[b]*X.MeanTimesPrecision
Vector ma = Vector.FromArray(A.Select(x => x.GetMean()).ToArray());
Vector va = Vector.FromArray(A.Select(x => x.GetVariance()).ToArray());
result.Precision.SetToDiagonal(va);
result.Precision.SetToSumWithOuter(result.Precision, 1, ma, ma);
result.Precision.SetToProduct(result.Precision, sum.Precision);
result.MeanTimesPrecision.SetToProduct(ma, sum.MeanTimesPrecision);
return result;
}
/// <summary>
/// EP message to 'b'
/// </summary>
/// <param name="product">Incoming message from 'product'.</param>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'b' as the random arguments are varied.
/// The formula is <c>proj[p(b) sum_(product) p(product) factor(product,a,b)]/p(b)</c>.
/// </para></remarks>
public static VectorGaussian BAverageConditional(VectorGaussian product, Matrix A, VectorGaussian result)
{
if (product.IsPointMass) return BAverageConditional(product.Point, A, result);
// (p.mean - A*B)'*p.prec*(p.mean - A*B)
// = B'*(A'*p.prec*A)*B - B'*A'*p.prec*p.mean - ...
// B.prec = A'*p.prec*A
// B.precTimesMean = A'*p.precTimesMean
Matrix temp = (product.Precision*A).Transpose();
result.Precision.SetToProduct(temp, A);
result.MeanTimesPrecision.SetToProduct(product.MeanTimesPrecision, A);
return result;
}
public static double AverageLogFactor(VectorGaussian product, Matrix A, VectorGaussian B) { return 0.0; }
/// <summary>
/// Runs the BPM using dense features and batched training.
/// </summary>
/// <param name="numClasses">The number of classes</param>
/// <param name="noisePrecision">The noise precision</param>
/// <param name="numFeatures">The number of features</param>
/// <param name="trainingSetFile">The file containing the training set</param>
/// <param name="testSet">The test set</param>
/// <param name="maxItemsPerChunk">The maximum number of items per chunk</param>
/// <param name="numChunks">The total number of chunks</param>
private static void RunSharedBPM(int numClasses, double noisePrecision, int numFeatures, string trainingSetFile, bool labelAtEnd, bool addBias, Vector[] testSet, int maxItemsPerChunk, int numChunks)
{
Console.WriteLine("\n------- Shared BPM -------");
BPMShared bpm = new BPMShared(numClasses, noisePrecision, numFeatures, numChunks, 1);
int[] labels;
Vector[] featureVectors;
VectorGaussian[] posteriorWeights = new VectorGaussian[numClasses];
// Several passes to achieve convergence.
for (int pass = 0; pass < 15; pass++)
{
int locationToStart = 0;
for (int c = 0; c < numChunks; c++)
{
featureVectors = DataUtils.Read(trainingSetFile, maxItemsPerChunk, labelAtEnd, addBias, out labels, ref locationToStart);
posteriorWeights = bpm.Train(featureVectors, labels, c);
}
}
#if ShowWeights
Console.WriteLine("Weights=" + StringUtil.ArrayToString(posteriorWeights));
#endif
Console.WriteLine("\nPredictions:");
Discrete[] predictions = bpm.Test(testSet, 0);
foreach (Discrete prediction in predictions)
Console.WriteLine(prediction);
Console.WriteLine();
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="product">Constant value for 'product'.</param>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="B">Incoming message from 'b'.</param>
/// <param name="BMean">Buffer 'BMean'.</param>
/// <param name="BVariance">Buffer 'BVariance'.</param>
/// <returns>Logarithm of the factor's contribution the EP model evidence</returns>
/// <remarks><para>
/// The formula for the result is <c>log(factor(product,a,b))</c>.
/// Adding up these values across all factors and variables gives the log-evidence estimate for EP.
/// </para></remarks>
public static double LogEvidenceRatio(Vector product, Matrix A, VectorGaussian B, [Fresh] Vector BMean, [Fresh] PositiveDefiniteMatrix BVariance)
{
return(LogAverageFactor(product, A, B, BMean, BVariance));
}
public static double AverageLogFactor(VectorGaussian product, Matrix A, VectorGaussian B)
{
return(0.0);
}
public static double LogEvidenceRatio(VectorGaussian product, Matrix A, VectorGaussian B)
{
return(0.0);
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="product">Constant value for 'product'.</param>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="BMean">Buffer 'BMean'.</param>
/// <param name="BVariance">Buffer 'BVariance'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(factor(product,a,b))</c>.
/// </para></remarks>
public static double LogAverageFactor(Vector product, Matrix A, VectorGaussian B, [Fresh] Vector BMean, [Fresh] PositiveDefiniteMatrix BVariance)
{
VectorGaussian toProduct = ProductAverageConditional(A, BMean, BVariance, new VectorGaussian(A.Rows));
return(toProduct.GetLogProb(product));
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="product">Incoming message from 'product'.</param>
/// <param name="to_product">Outgoing message to 'product'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(product) p(product) factor(product,a,b))</c>.
/// </para></remarks>
public static double LogAverageFactor(VectorGaussian product, [Fresh] VectorGaussian to_product)
{
return(to_product.GetLogAverageOf(product));
}
/// <summary>
/// Update the buffer 'BMean'
/// </summary>
/// <param name="B">Incoming message from 'b'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="BVariance">Buffer 'BVariance'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
///
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="B"/> is not a proper distribution</exception>
public static Vector BMean([Proper] VectorGaussian B, [Fresh] PositiveDefiniteMatrix BVariance, Vector result)
{
return(B.GetMean(result, BVariance));
}
/// <summary>
/// EP message to 'product'
/// </summary>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="BMean">Buffer 'BMean'.</param>
/// <param name="BVariance">Buffer 'BVariance'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'product' conditioned on the given values.
/// </para></remarks>
public static VectorGaussian ProductAverageConditional(Matrix A, [Fresh] Vector BMean, [Fresh] PositiveDefiniteMatrix BVariance, VectorGaussian result)
{
// P.mean = A*B.mean
// P.var = A*B.var*A'
// if A is invertible, then
// P.prec = inv(A)'*inv(B.var)*inv(A)
// P.precTimesMean = inv(A)'*B.precTimesMean
Vector rmean = A * BMean;
PositiveDefiniteMatrix rvariance = new PositiveDefiniteMatrix(result.Dimension, result.Dimension);
Matrix temp = (A * BVariance).Transpose();
rvariance.SetToProduct(A, temp);
result.SetMeanAndVariance(rmean, rvariance);
return(result);
}
public static VectorGaussian RotateAverageLogarithm([SkipIfUniform] Gaussian x, [SkipIfUniform] Gaussian y, double angle, VectorGaussian result)
{
return RotateAverageLogarithm(x, y, WrappedGaussian.PointMass(angle), result);
}
/// <summary>
/// EP message to 'b'
/// </summary>
/// <param name="product">Incoming message from 'product'.</param>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'b' as the random arguments are varied.
/// The formula is <c>proj[p(b) sum_(product) p(product) factor(product,a,b)]/p(b)</c>.
/// </para></remarks>
public static VectorGaussian BAverageConditional(VectorGaussian product, Matrix A, VectorGaussian result)
{
if (product.IsPointMass)
{
return(BAverageConditional(product.Point, A, result));
}
// (p.mean - A*B)'*p.prec*(p.mean - A*B)
// = B'*(A'*p.prec*A)*B - B'*A'*p.prec*p.mean - ...
// B.prec = A'*p.prec*A
// B.precTimesMean = A'*p.precTimesMean
Matrix temp = (product.Precision * A).Transpose();
result.Precision.SetToProduct(temp, A);
result.MeanTimesPrecision.SetToProduct(product.MeanTimesPrecision, A);
return(result);
}
/// <summary>
/// Performs inference on this Bayes point machine
/// </summary>
/// <param name="xValuesData">Lists of vectors for each component</param>
/// <returns></returns>
private VectorGaussian[] InferW(List<Vector>[] xValuesData)
{
// Set the observed data
for (int c = 0; c < nClass; c++)
{
trainModel.nItems[c].ObservedValue = xValuesData[c].Count;
trainModel.xValues[c].ObservedValue = xValuesData[c].ToArray();
}
// Infer the weights
VectorGaussian[] wInfer = new VectorGaussian[nClass];
// Reset the priors to support incremental training
for (int c = 0; c < nClass; c++)
{
wInfer[c] = (trainModel.ie).Infer<VectorGaussian>(trainModel.w[c]);
trainModel.wInit[c].ObservedValue = wInfer[c];
}
return wInfer;
}
/// <summary>
/// VMP message to 'b'
/// </summary>
/// <param name="product">Incoming message from 'product'.</param>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'b' with 'product' integrated out.
/// The formula is <c>sum_product p(product) factor(product,a,b)</c>.
/// </para></remarks>
public static VectorGaussian BAverageLogarithm(VectorGaussian product, Matrix A, VectorGaussian result)
{
return(BAverageConditional(product, A, result));
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="product">Constant value for 'product'.</param>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="BMean">Buffer 'BMean'.</param>
/// <param name="BVariance">Buffer 'BVariance'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(factor(product,a,b))</c>.
/// </para></remarks>
public static double LogAverageFactor(Vector product, Matrix A, VectorGaussian B, [Fresh] Vector BMean, [Fresh] PositiveDefiniteMatrix BVariance)
{
VectorGaussian toProduct = ProductAverageConditional(A, BMean, BVariance, new VectorGaussian(A.Rows));
return toProduct.GetLogProb(product);
}
/// <summary>Computations that depend on the observed value of vVector__7</summary>
private void Changed_vVector__7()
{
if (this.Changed_vVector__7_iterationsDone == 1)
{
return;
}
this.vVector__7_marginal = new PointMass <Vector[]>(this.VVector__7);
// The constant 'vVectorGaussian7'
VectorGaussian vVectorGaussian7 = VectorGaussian.FromNatural(DenseVector.FromArray(new double[3] {
1547829870.0, 525077980.0, 200270.0
}), new PositiveDefiniteMatrix(new double[3, 3] {
{ 4254590363351.0, 1127383488860.0, 433199230.0 }, { 1127383488860.0, 482723521821.0, 146764360.0 }, { 433199230.0, 146764360.0, 56221.0 }
}));
this.vVector21_marginal_F = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian7);
// Buffer for ReplicateOp_Divide.Marginal<VectorGaussian>
VectorGaussian vVector21_rep_B_toDef = default(VectorGaussian);
// Message to 'vVector21_rep' from Replicate factor
vVector21_rep_B_toDef = ReplicateOp_Divide.ToDefInit <VectorGaussian>(vVectorGaussian7);
// Message to 'vVector21_marginal' from Variable factor
this.vVector21_marginal_F = VariableOp.MarginalAverageConditional <VectorGaussian>(vVector21_rep_B_toDef, vVectorGaussian7, this.vVector21_marginal_F);
DistributionStructArray <Gaussian, double> vdouble__21_F = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__21' Forwards messages.
vdouble__21_F = new DistributionStructArray <Gaussian, double>(1);
for (int index7 = 0; index7 < 1; index7++)
{
vdouble__21_F[index7] = Gaussian.Uniform();
}
DistributionStructArray <Gaussian, double> vdouble__22_F = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__22' Forwards messages.
vdouble__22_F = new DistributionStructArray <Gaussian, double>(1);
for (int index7 = 0; index7 < 1; index7++)
{
vdouble__22_F[index7] = Gaussian.Uniform();
}
DistributionRefArray <VectorGaussian, Vector> vVector21_rep_F = default(DistributionRefArray <VectorGaussian, Vector>);
DistributionRefArray <VectorGaussian, Vector> vVector21_rep_B = default(DistributionRefArray <VectorGaussian, Vector>);
// Create array for 'vVector21_rep' Forwards messages.
vVector21_rep_F = new DistributionRefArray <VectorGaussian, Vector>(1);
// Create array for 'vVector21_rep' Backwards messages.
vVector21_rep_B = new DistributionRefArray <VectorGaussian, Vector>(1);
for (int index7 = 0; index7 < 1; index7++)
{
vVector21_rep_B[index7] = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian7);
vVector21_rep_F[index7] = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian7);
}
// Buffer for ReplicateOp_Divide.UsesAverageConditional<VectorGaussian>
VectorGaussian vVector21_rep_F_marginal = default(VectorGaussian);
// Message to 'vVector21_rep' from Replicate factor
vVector21_rep_F_marginal = ReplicateOp_Divide.MarginalInit <VectorGaussian>(vVectorGaussian7);
// Message to 'vVector21_rep' from Replicate factor
vVector21_rep_F_marginal = ReplicateOp_Divide.Marginal <VectorGaussian>(vVector21_rep_B_toDef, vVectorGaussian7, vVector21_rep_F_marginal);
// Buffer for InnerProductOp.InnerProductAverageConditional
// Create array for replicates of 'vVector21_rep_F_index7__AMean'
Vector[] vVector21_rep_F_index7__AMean = new Vector[1];
for (int index7 = 0; index7 < 1; index7++)
{
// Message to 'vdouble__22' from InnerProduct factor
vVector21_rep_F_index7__AMean[index7] = InnerProductOp.AMeanInit(vVector21_rep_F[index7]);
}
// Buffer for InnerProductOp.AMean
// Create array for replicates of 'vVector21_rep_F_index7__AVariance'
PositiveDefiniteMatrix[] vVector21_rep_F_index7__AVariance = new PositiveDefiniteMatrix[1];
for (int index7 = 0; index7 < 1; index7++)
{
// Message to 'vdouble__22' from InnerProduct factor
vVector21_rep_F_index7__AVariance[index7] = InnerProductOp.AVarianceInit(vVector21_rep_F[index7]);
// Message to 'vVector21_rep' from Replicate factor
vVector21_rep_F[index7] = ReplicateOp_Divide.UsesAverageConditional <VectorGaussian>(vVector21_rep_B[index7], vVector21_rep_F_marginal, index7, vVector21_rep_F[index7]);
}
// Create array for 'vdouble__22_marginal' Forwards messages.
this.vdouble__22_marginal_F = new DistributionStructArray <Gaussian, double>(1);
for (int index7 = 0; index7 < 1; index7++)
{
this.vdouble__22_marginal_F[index7] = Gaussian.Uniform();
}
// Message from use of 'vdouble__22'
DistributionStructArray <Gaussian, double> vdouble__22_use_B = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__22_use' Backwards messages.
vdouble__22_use_B = new DistributionStructArray <Gaussian, double>(1);
for (int index7 = 0; index7 < 1; index7++)
{
vdouble__22_use_B[index7] = Gaussian.Uniform();
// Message to 'vdouble__22' from InnerProduct factor
vVector21_rep_F_index7__AVariance[index7] = InnerProductOp.AVariance(vVector21_rep_F[index7], vVector21_rep_F_index7__AVariance[index7]);
// Message to 'vdouble__22' from InnerProduct factor
vVector21_rep_F_index7__AMean[index7] = InnerProductOp.AMean(vVector21_rep_F[index7], vVector21_rep_F_index7__AVariance[index7], vVector21_rep_F_index7__AMean[index7]);
// Message to 'vdouble__22' from InnerProduct factor
vdouble__22_F[index7] = InnerProductOp.InnerProductAverageConditional(vVector21_rep_F_index7__AMean[index7], vVector21_rep_F_index7__AVariance[index7], this.VVector__7[index7]);
// Message to 'vdouble__22_marginal' from DerivedVariable factor
this.vdouble__22_marginal_F[index7] = DerivedVariableOp.MarginalAverageConditional <Gaussian>(vdouble__22_use_B[index7], vdouble__22_F[index7], this.vdouble__22_marginal_F[index7]);
}
// Create array for 'vdouble__21_marginal' Forwards messages.
this.vdouble__21_marginal_F = new DistributionStructArray <Gaussian, double>(1);
for (int index7 = 0; index7 < 1; index7++)
{
this.vdouble__21_marginal_F[index7] = Gaussian.Uniform();
}
// Message from use of 'vdouble__21'
DistributionStructArray <Gaussian, double> vdouble__21_use_B = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__21_use' Backwards messages.
vdouble__21_use_B = new DistributionStructArray <Gaussian, double>(1);
for (int index7 = 0; index7 < 1; index7++)
{
vdouble__21_use_B[index7] = Gaussian.Uniform();
// Message to 'vdouble__21' from GaussianFromMeanAndVariance factor
vdouble__21_F[index7] = GaussianFromMeanAndVarianceOp.SampleAverageConditional(vdouble__22_F[index7], 0.1);
// Message to 'vdouble__21_marginal' from Variable factor
this.vdouble__21_marginal_F[index7] = VariableOp.MarginalAverageConditional <Gaussian>(vdouble__21_use_B[index7], vdouble__21_F[index7], this.vdouble__21_marginal_F[index7]);
}
this.Changed_vVector__7_iterationsDone = 1;
}
/// <summary>
/// EP message to 'product'
/// </summary>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="BMean">Buffer 'BMean'.</param>
/// <param name="BVariance">Buffer 'BVariance'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'product' conditioned on the given values.
/// </para></remarks>
public static VectorGaussian ProductAverageConditional(Matrix A, [Fresh] Vector BMean, [Fresh] PositiveDefiniteMatrix BVariance, VectorGaussian result)
{
// P.mean = A*B.mean
// P.var = A*B.var*A'
// if A is invertible, then
// P.prec = inv(A)'*inv(B.var)*inv(A)
// P.precTimesMean = inv(A)'*B.precTimesMean
Vector rmean = A * BMean;
PositiveDefiniteMatrix rvariance = new PositiveDefiniteMatrix(result.Dimension, result.Dimension);
Matrix temp = (A*BVariance).Transpose();
rvariance.SetToProduct(A, temp);
result.SetMeanAndVariance(rmean, rvariance);
return result;
}
/// <summary>Computations that depend on the observed value of vVector__46 and vdouble__138</summary>
private void Changed_vVector__46_vdouble__138()
{
if (this.Changed_vVector__46_vdouble__138_iterationsDone == 1)
{
return;
}
this.vVector__46_marginal = new PointMass <Vector[]>(this.VVector__46);
this.vdouble__138_marginal = new DistributionStructArray <Gaussian, double>(5622, delegate(int index46) {
return(Gaussian.Uniform());
});
this.vdouble__138_marginal = Distribution.SetPoint <DistributionStructArray <Gaussian, double>, double[]>(this.vdouble__138_marginal, this.Vdouble__138);
// The constant 'vVectorGaussian46'
VectorGaussian vVectorGaussian46 = VectorGaussian.FromNatural(DenseVector.FromArray(new double[3] {
0.0, 0.0, 0.0
}), new PositiveDefiniteMatrix(new double[3, 3] {
{ 1.0, 0.0, 0.0 }, { 0.0, 1.0, 0.0 }, { 0.0, 0.0, 1.0 }
}));
this.vVector139_marginal_F = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian46);
// Message from use of 'vdouble__139'
DistributionStructArray <Gaussian, double> vdouble__139_use_B = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__139_use' Backwards messages.
vdouble__139_use_B = new DistributionStructArray <Gaussian, double>(5622);
for (int index46 = 0; index46 < 5622; index46++)
{
vdouble__139_use_B[index46] = Gaussian.Uniform();
// Message to 'vdouble__139_use' from GaussianFromMeanAndVariance factor
vdouble__139_use_B[index46] = GaussianFromMeanAndVarianceOp.MeanAverageConditional(this.Vdouble__138[index46], 0.1);
}
DistributionRefArray <VectorGaussian, Vector> vVector139_rep_B = default(DistributionRefArray <VectorGaussian, Vector>);
// Create array for 'vVector139_rep' Backwards messages.
vVector139_rep_B = new DistributionRefArray <VectorGaussian, Vector>(5622);
for (int index46 = 0; index46 < 5622; index46++)
{
vVector139_rep_B[index46] = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian46);
// Message to 'vVector139_rep' from InnerProduct factor
vVector139_rep_B[index46] = InnerProductOp.AAverageConditional(vdouble__139_use_B[index46], this.VVector__46[index46], vVector139_rep_B[index46]);
}
// Buffer for ReplicateOp_Divide.Marginal<VectorGaussian>
VectorGaussian vVector139_rep_B_toDef = default(VectorGaussian);
// Message to 'vVector139_rep' from Replicate factor
vVector139_rep_B_toDef = ReplicateOp_Divide.ToDefInit <VectorGaussian>(vVectorGaussian46);
// Message to 'vVector139_rep' from Replicate factor
vVector139_rep_B_toDef = ReplicateOp_Divide.ToDef <VectorGaussian>(vVector139_rep_B, vVector139_rep_B_toDef);
// Message to 'vVector139_marginal' from Variable factor
this.vVector139_marginal_F = VariableOp.MarginalAverageConditional <VectorGaussian>(vVector139_rep_B_toDef, vVectorGaussian46, this.vVector139_marginal_F);
DistributionStructArray <Gaussian, double> vdouble__139_F = default(DistributionStructArray <Gaussian, double>);
// Create array for 'vdouble__139' Forwards messages.
vdouble__139_F = new DistributionStructArray <Gaussian, double>(5622);
for (int index46 = 0; index46 < 5622; index46++)
{
vdouble__139_F[index46] = Gaussian.Uniform();
}
DistributionRefArray <VectorGaussian, Vector> vVector139_rep_F = default(DistributionRefArray <VectorGaussian, Vector>);
// Create array for 'vVector139_rep' Forwards messages.
vVector139_rep_F = new DistributionRefArray <VectorGaussian, Vector>(5622);
for (int index46 = 0; index46 < 5622; index46++)
{
vVector139_rep_F[index46] = ArrayHelper.MakeUniform <VectorGaussian>(vVectorGaussian46);
}
// Buffer for ReplicateOp_Divide.UsesAverageConditional<VectorGaussian>
VectorGaussian vVector139_rep_F_marginal = default(VectorGaussian);
// Message to 'vVector139_rep' from Replicate factor
vVector139_rep_F_marginal = ReplicateOp_Divide.MarginalInit <VectorGaussian>(vVectorGaussian46);
// Message to 'vVector139_rep' from Replicate factor
vVector139_rep_F_marginal = ReplicateOp_Divide.Marginal <VectorGaussian>(vVector139_rep_B_toDef, vVectorGaussian46, vVector139_rep_F_marginal);
// Buffer for InnerProductOp.InnerProductAverageConditional
// Create array for replicates of 'vVector139_rep_F_index46__AMean'
Vector[] vVector139_rep_F_index46__AMean = new Vector[5622];
for (int index46 = 0; index46 < 5622; index46++)
{
// Message to 'vdouble__139' from InnerProduct factor
vVector139_rep_F_index46__AMean[index46] = InnerProductOp.AMeanInit(vVector139_rep_F[index46]);
}
// Buffer for InnerProductOp.AMean
// Create array for replicates of 'vVector139_rep_F_index46__AVariance'
PositiveDefiniteMatrix[] vVector139_rep_F_index46__AVariance = new PositiveDefiniteMatrix[5622];
for (int index46 = 0; index46 < 5622; index46++)
{
// Message to 'vdouble__139' from InnerProduct factor
vVector139_rep_F_index46__AVariance[index46] = InnerProductOp.AVarianceInit(vVector139_rep_F[index46]);
// Message to 'vVector139_rep' from Replicate factor
vVector139_rep_F[index46] = ReplicateOp_Divide.UsesAverageConditional <VectorGaussian>(vVector139_rep_B[index46], vVector139_rep_F_marginal, index46, vVector139_rep_F[index46]);
}
// Create array for 'vdouble__139_marginal' Forwards messages.
this.vdouble__139_marginal_F = new DistributionStructArray <Gaussian, double>(5622);
for (int index46 = 0; index46 < 5622; index46++)
{
this.vdouble__139_marginal_F[index46] = Gaussian.Uniform();
// Message to 'vdouble__139' from InnerProduct factor
vVector139_rep_F_index46__AVariance[index46] = InnerProductOp.AVariance(vVector139_rep_F[index46], vVector139_rep_F_index46__AVariance[index46]);
// Message to 'vdouble__139' from InnerProduct factor
vVector139_rep_F_index46__AMean[index46] = InnerProductOp.AMean(vVector139_rep_F[index46], vVector139_rep_F_index46__AVariance[index46], vVector139_rep_F_index46__AMean[index46]);
// Message to 'vdouble__139' from InnerProduct factor
vdouble__139_F[index46] = InnerProductOp.InnerProductAverageConditional(vVector139_rep_F_index46__AMean[index46], vVector139_rep_F_index46__AVariance[index46], this.VVector__46[index46]);
// Message to 'vdouble__139_marginal' from DerivedVariable factor
this.vdouble__139_marginal_F[index46] = DerivedVariableOp.MarginalAverageConditional <Gaussian>(vdouble__139_use_B[index46], vdouble__139_F[index46], this.vdouble__139_marginal_F[index46]);
}
this.Changed_vVector__46_vdouble__138_iterationsDone = 1;
}
public static VectorGaussian BAverageLogarithm(Vector product, Matrix A, VectorGaussian result)
{
throw new NotSupportedException(MatrixVectorProductOp.LowRankNotSupportedMessage);
}
internal void SoftmaxGWAS()
{
var v = Vector.Zero(1);
var x1 = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(1), PositiveDefiniteMatrix.Identity(1));
x1.ObservedValue = v;
var x2 = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(1), PositiveDefiniteMatrix.Identity(1));
v[0] = 1;
x2.ObservedValue = v;
Console.WriteLine(x1.ObservedValue);
Console.WriteLine(x2.ObservedValue);
int numExposures = 10;
int numClasses = 5;
int numSamples = 1000;
var c = new Range(numClasses);
var n = new Range(numSamples);
// model
var Bexposures = Variable.Array <Vector>(c);
Bexposures[c] = Variable.VectorGaussianFromMeanAndPrecision(
Vector.Zero(numExposures),
PositiveDefiniteMatrix.Identity(numExposures))
.ForEach(c);
var exposures = Variable.Array <Vector>(n);
var Bgenotype = Variable.Array <double>(c);
Bgenotype[c] = Variable.GaussianFromMeanAndPrecision(0, 1).ForEach(c);
var genotypes = Variable.Array <double>(n);
var mean = Variable.Array <double>(c);
mean[c] = Variable.GaussianFromMeanAndPrecision(0, 1).ForEach(c);
var g = Variable.Array(Variable.Array <double>(c), n);
g[n][c] = Bgenotype[c] * genotypes[n] + Variable.InnerProduct(Bexposures[c], exposures[n]) + mean[c];
var classProbs = Variable.Array <Vector>(n);
classProbs[n] = Variable.Softmax(g[n]);
var classAssignments = Variable.Array <int>(n);
classAssignments[n] = Variable.Discrete(classProbs[n]);
// generate data
var exposuresObs = new Vector[numSamples];
var genotypeObs = new double[numSamples];
var classObs = new int[numSamples];
var trueBexposures = new Vector[numClasses];
var trueBgenotype = new double[numClasses];
Rand.Restart(123);
for (int i = 0; i < numClasses; i++)
{
trueBexposures[i] = VectorGaussian.SampleFromMeanAndVariance(Vector.Zero(numExposures), PositiveDefiniteMatrix.Identity(numExposures));
trueBgenotype[i] = Rand.Normal();
}
for (int i = 0; i < numSamples; i++)
{
exposuresObs[i] = VectorGaussian.SampleFromMeanAndVariance(Vector.Zero(numExposures), PositiveDefiniteMatrix.Identity(numExposures));
genotypeObs[i] = Rand.Double() < .5 ? 0.0 : 1.0;
var gt = new double[numClasses];
for (int j = 0; j < numClasses; j++)
{
gt[j] = genotypeObs[i] * trueBgenotype[j] + exposuresObs[i].Inner(trueBexposures[j]);
}
var p = MMath.Softmax(gt);
classObs[i] = Discrete.Sample(p);
}
Rand.Restart(DateTime.Now.Millisecond);
exposures.ObservedValue = exposuresObs;
genotypes.ObservedValue = genotypeObs;
classAssignments.ObservedValue = classObs;
// inference
var ie = new InferenceEngine(new VariationalMessagePassing());
var bPost = ie.Infer <Gaussian[]>(Bgenotype);
for (int i = 0; i < numClasses; i++)
{
Console.WriteLine("Inferred: " + bPost[i] + " truth: " + trueBgenotype[i]);
}
}
public void Initialize(bool skipStringDistributions = false)
{
// DO NOT make this a constructor, because it makes the test not notice complete lack of serialization as an empty object is set up exactly as the thing
// you are trying to deserialize.
this.pareto = new Pareto(1.2, 3.5);
this.poisson = new Poisson(2.3);
this.wishart = new Wishart(20, new PositiveDefiniteMatrix(new double[, ] {
{ 22, 21 }, { 21, 23 }
}));
this.vectorGaussian = new VectorGaussian(Vector.FromArray(13, 14), new PositiveDefiniteMatrix(new double[, ] {
{ 16, 15 }, { 15, 17 }
}));
this.unnormalizedDiscrete = UnnormalizedDiscrete.FromLogProbs(DenseVector.FromArray(5.1, 5.2, 5.3));
this.pointMass = PointMass <double> .Create(1.1);
this.gaussian = new Gaussian(11.0, 12.0);
this.nonconjugateGaussian = new NonconjugateGaussian(1.2, 2.3, 3.4, 4.5);
this.gamma = new Gamma(9.0, 10.0);
this.gammaPower = new GammaPower(5.6, 2.8, 3.4);
this.discrete = new Discrete(6.0, 7.0, 8.0);
this.conjugateDirichlet = new ConjugateDirichlet(1.2, 2.3, 3.4, 4.5);
this.dirichlet = new Dirichlet(3.0, 4.0, 5.0);
this.beta = new Beta(2.0, 1.0);
this.binomial = new Binomial(5, 0.8);
this.bernoulli = new Bernoulli(0.6);
this.sparseBernoulliList = SparseBernoulliList.Constant(4, new Bernoulli(0.1));
this.sparseBernoulliList[1] = new Bernoulli(0.9);
this.sparseBernoulliList[3] = new Bernoulli(0.7);
this.sparseBetaList = SparseBetaList.Constant(5, new Beta(2.0, 2.0));
this.sparseBetaList[0] = new Beta(3.0, 4.0);
this.sparseBetaList[1] = new Beta(5.0, 6.0);
this.sparseGaussianList = SparseGaussianList.Constant(6, Gaussian.FromMeanAndPrecision(0.1, 0.2));
this.sparseGaussianList[4] = Gaussian.FromMeanAndPrecision(0.3, 0.4);
this.sparseGaussianList[5] = Gaussian.FromMeanAndPrecision(0.5, 0.6);
this.sparseGammaList = SparseGammaList.Constant(1, Gamma.FromShapeAndRate(1.0, 2.0));
this.truncatedGamma = new TruncatedGamma(1.2, 2.3, 3.4, 4.5);
this.truncatedGaussian = new TruncatedGaussian(1.2, 3.4, 5.6, 7.8);
this.wrappedGaussian = new WrappedGaussian(1.2, 2.3, 3.4);
ga = Distribution <double> .Array(new[] { this.gaussian, this.gaussian });
vga = Distribution <Vector> .Array(new[] { this.vectorGaussian, this.vectorGaussian });
ga2D = Distribution <double> .Array(new[, ] {
{ this.gaussian, this.gaussian }, { this.gaussian, this.gaussian }
});
vga2D = Distribution <Vector> .Array(new[, ] {
{ this.vectorGaussian, this.vectorGaussian }, { this.vectorGaussian, this.vectorGaussian }
});
gaJ = Distribution <double> .Array(new[] { new[] { this.gaussian, this.gaussian }, new[] { this.gaussian, this.gaussian } });
vgaJ = Distribution <Vector> .Array(new[] { new[] { this.vectorGaussian, this.vectorGaussian }, new[] { this.vectorGaussian, this.vectorGaussian } });
var gp = new GaussianProcess(new ConstantFunction(0), new SquaredExponential(0));
var basis = Util.ArrayInit(2, i => Vector.FromArray(1.0 * i));
this.sparseGp = new SparseGP(new SparseGPFixed(gp, basis));
this.quantileEstimator = new QuantileEstimator(0.01);
this.quantileEstimator.Add(5);
this.outerQuantiles = OuterQuantiles.FromDistribution(3, this.quantileEstimator);
this.innerQuantiles = InnerQuantiles.FromDistribution(3, this.outerQuantiles);
if (!skipStringDistributions)
{
// String distributions can not be serialized by some formatters (namely BinaryFormatter)
// That is fine because this combination is never used in practice
this.stringDistribution1 = StringDistribution.String("aa")
.Append(StringDistribution.OneOf("b", "ccc")).Append("dddd");
this.stringDistribution2 = new StringDistribution();
this.stringDistribution2.SetToProduct(StringDistribution.OneOf("a", "b"),
StringDistribution.OneOf("b", "c"));
}
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="AverageLogFactor(Vector, Matrix, VectorGaussian, Vector, PositiveDefiniteMatrix)"]/*'/>
public static double AverageLogFactor(Vector product, Matrix A, VectorGaussian B, Vector BMean, PositiveDefiniteMatrix BVariance)
{
return(LogAverageFactor(product, A, B, BMean, BVariance));
}
public static VectorGaussian RotateAverageLogarithm(double x, double y, [Proper] WrappedGaussian angle, VectorGaussian result)
{
return(RotateAverageLogarithm(Gaussian.PointMass(x), Gaussian.PointMass(y), angle, result));
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="ProductAverageConditional(Matrix, Vector, PositiveDefiniteMatrix, VectorGaussian)"]/*'/>
public static VectorGaussian ProductAverageConditional(Matrix A, Vector BMean, PositiveDefiniteMatrix BVariance, VectorGaussian result)
{
Vector mean = Vector.Zero(result.Dimension);
PositiveDefiniteMatrix variance = result.Precision;
GetProductMoments(A, BMean, BVariance, mean, variance);
result.SetMeanAndVariance(mean, variance);
return(result);
}
/// <summary>
/// VMP message to 'second'
/// </summary>
/// <param name="concat">Constant value for 'concat'.</param>
/// <param name="first">Incoming message from 'first'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the exponential of the average log-factor value, where the average is over all arguments except 'second'.
/// The formula is <c>exp(sum_(first) p(first) log(factor(concat,first,second)))</c>.
/// </para></remarks>
public static VectorGaussian SecondAverageLogarithm(Vector concat, VectorGaussian first, VectorGaussian result)
{
return SecondAverageConditional(concat, first, result);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="ProductAverageConditional(double[,], Vector, PositiveDefiniteMatrix, VectorGaussian)"]/*'/>
public static VectorGaussian ProductAverageConditional(DistributionArray2D <Gaussian, double> A, Vector BMean, PositiveDefiniteMatrix BVariance, VectorGaussian result)
{
if (!A.IsPointMass)
{
throw new ArgumentException("A is not a point mass");
}
return(ProductAverageConditional(new Matrix(A.Point), BMean, BVariance, result));
}
public static VectorGaussian RotateAverageLogarithm(double x, double y, [Proper] WrappedGaussian angle, VectorGaussian result)
{
return RotateAverageLogarithm(Gaussian.PointMass(x), Gaussian.PointMass(y), angle, result);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="ProductAverageLogarithm(double[,], Vector, PositiveDefiniteMatrix, VectorGaussian)"]/*'/>
public static VectorGaussian ProductAverageLogarithm(double[,] A, Vector BMean, PositiveDefiniteMatrix BVariance, VectorGaussian result)
{
return(ProductAverageConditional(A, BMean, BVariance, result));
}
public static double AverageLogFactor(VectorGaussian rotate) { return 0.0; }
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="BAverageConditional(VectorGaussian, Matrix, VectorGaussian)"]/*'/>
public static VectorGaussian BAverageConditional([SkipIfUniform] VectorGaussian product, Matrix A, VectorGaussian result)
{
if (product.IsPointMass)
{
return(BAverageConditional(product.Point, A, result));
}
// (p.mean - A*B)'*p.prec*(p.mean - A*B)
// = B'*(A'*p.prec*A)*B - B'*A'*p.prec*p.mean - ...
// B.prec = A'*p.prec*A
// B.precTimesMean = A'*p.precTimesMean
if (UseAccurateMethod)
{
// this method is slower but more numerically accurate
// L*L' = p.prec
int dim = product.Precision.Cols;
LowerTriangularMatrix L = new LowerTriangularMatrix(dim, dim);
L.SetToCholesky(product.Precision);
Matrix At = A.Transpose();
Matrix temp = At * L;
result.Precision.SetToOuter(temp);
}
else
{
Matrix temp = (product.Precision * A).Transpose();
result.Precision.SetToProduct(temp, A);
result.Precision.Symmetrize();
}
result.MeanTimesPrecision.SetToProduct(product.MeanTimesPrecision, A);
return(result);
}
/// <summary>
/// Train the machine with training data.
/// </summary>
/// <param name="trainingData">Training data</param>
private void TrainModel(IList<DataVector> trainingData)
{
// Create the array to store classification results
VariableArray<bool> trainResults =
Variable.Observed(trainingData.Select(r => r.ClassId == 1).ToArray()).Named("trainResults");
InitWeight();
// Create an array to represent the observed training data
Range resultRange = trainResults.Range.Named("relevance");
VariableArray<Vector> observedData =
Variable.Observed(trainingData.Select(r => r.FeatureVector).ToArray(), resultRange).Named("observedData");
// Create Bayes Point Machine
trainResults[resultRange] =
Variable.GaussianFromMeanAndVariance(
Variable.InnerProduct(weight, observedData[resultRange]).Named("innerProduct"), this.Noise) > 0;
// Using expectation propagation to infer the posterior over test data
this.Posterior = Engine.Infer<VectorGaussian>(this.weight);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="BAverageConditional(VectorGaussianMoments, Matrix, VectorGaussian)"]/*'/>
public static VectorGaussian BAverageConditional([SkipIfUniform] VectorGaussianMoments product, Matrix A, VectorGaussian result)
{
if (product.IsPointMass)
{
return(BAverageConditional(product.Point, A, result));
}
VectorGaussian productNatural = new VectorGaussian(product.Mean, product.Variance);
return(BAverageConditional(productNatural, A, result));
}
/// <summary>
/// Runs the BPM using dense features and incremental training.
/// </summary>
/// <param name="numClasses">The number of classes</param>
/// <param name="noisePrecision">The noise precision</param>
/// <param name="trainingSetFile">The file containing the training set</param>
/// <param name="testSet">The test set</param>
/// <param name="maxItemsPerChunk">The maximum number of items per chunk</param>
/// <param name="numChunks">The total number of chunks</param>
private static void RunIncrementalBPM(int numClasses, double noisePrecision, string trainingSetFile, bool labelAtEnd, bool addBias, Vector[] testSet, int maxItemsPerChunk, int numChunks)
{
Console.WriteLine("\n------- Incremental BPM -------");
BPM bpm = new BPM(numClasses, noisePrecision);
int[] labels;
Vector[] featureVectors;
VectorGaussian[] posteriorWeights = new VectorGaussian[numClasses];
int locationToStart = 0;
for (int c = 0; c < numChunks; c++)
{
featureVectors = DataUtils.Read(trainingSetFile, maxItemsPerChunk, labelAtEnd, addBias, out labels, ref locationToStart);
posteriorWeights = bpm.Train(featureVectors, labels);
}
#if ShowWeights
Console.WriteLine("Weights=" + StringUtil.ArrayToString(posteriorWeights));
#endif
Console.WriteLine("\nPredictions:");
Discrete[] predictions = bpm.Test(testSet);
foreach (Discrete pred in predictions)
Console.WriteLine(pred);
Console.WriteLine();
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="BAverageConditional(VectorGaussianMoments, double[,], VectorGaussian)"]/*'/>
public static VectorGaussian BAverageConditional([SkipIfUniform] VectorGaussianMoments product, double[,] A, VectorGaussian result)
{
return(BAverageConditional(product, new Matrix(A), result));
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="product">Incoming message from 'product'.</param>
/// <param name="to_product">Outgoing message to 'product'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(product) p(product) factor(product,a,b))</c>.
/// </para></remarks>
public static double LogAverageFactor(VectorGaussian product, [Fresh] VectorGaussian to_product)
{
return to_product.GetLogAverageOf(product);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="BAverageConditional(VectorGaussianMoments, DistributionArray2D{Gaussian,double}, VectorGaussian)"]/*'/>
public static VectorGaussian BAverageConditional([SkipIfUniform] VectorGaussianMoments product, DistributionArray2D <Gaussian, double> A, VectorGaussian result)
{
if (!A.IsPointMass)
{
throw new ArgumentException("A is not a point mass");
}
return(BAverageConditional(product, new Matrix(A.Point), result));
}
public static double LogEvidenceRatio(VectorGaussian product, Matrix A, VectorGaussian B) { return 0.0; }
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="BAverageLogarithm(VectorGaussianMoments, Matrix, VectorGaussian)"]/*'/>
public static VectorGaussian BAverageLogarithm([SkipIfUniform] VectorGaussianMoments product, Matrix A, VectorGaussian result)
{
return(BAverageConditional(product, A, result));
}
/// <summary>
/// Evidence message for VMP
/// </summary>
/// <param name="product">Constant value for 'product'.</param>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="B">Incoming message from 'b'.</param>
/// <param name="BMean">Buffer 'BMean'.</param>
/// <param name="BVariance">Buffer 'BVariance'.</param>
/// <returns>Zero</returns>
/// <remarks><para>
/// The formula for the result is <c>log(factor(product,a,b))</c>.
/// Adding up these values across all factors and variables gives the log-evidence estimate for VMP.
/// </para></remarks>
public static double AverageLogFactor(Vector product, Matrix A, VectorGaussian B, [Fresh] Vector BMean, [Fresh] PositiveDefiniteMatrix BVariance) { return LogAverageFactor(product, A, B, BMean, BVariance); }
public static VectorGaussian BAverageLogarithm(Vector product, Matrix A, VectorGaussian result)
{
throw new NotSupportedException(MatrixVectorProductOp.LowRankNotSupportedMessage);
}
/// <summary>
/// VMP message to 'product'
/// </summary>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="BMean">Buffer 'BMean'.</param>
/// <param name="BVariance">Buffer 'BVariance'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'product' conditioned on the given values.
/// </para></remarks>
public static VectorGaussian ProductAverageLogarithm(Matrix A, [Fresh] Vector BMean, [Fresh] PositiveDefiniteMatrix BVariance, VectorGaussian result)
{
return ProductAverageConditional(A, BMean, BVariance, result);
}
/// <summary>
/// EP message to 'A'
/// </summary>
/// <param name="matrixMultiply">Incoming message from 'matrixMultiply'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="A">Incoming message from 'A'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="B">Constant value for 'B'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'A' as the random arguments are varied.
/// The formula is <c>proj[p(A) sum_(matrixMultiply) p(matrixMultiply) factor(matrixMultiply,A,B)]/p(A)</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="matrixMultiply"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="A"/> is not a proper distribution</exception>
public static GaussianArray2D AAverageConditional([SkipIfUniform] GaussianArray2D matrixMultiply, [SkipIfUniform] GaussianArray2D A, double[,] B, GaussianArray2D result)
{
int rows = matrixMultiply.GetLength(0);
int cols = matrixMultiply.GetLength(1);
int inner = B.GetLength(0);
if (result == null) result = new GaussianArray2D(rows, inner);
// sum_{i,j} (m[i,j] - a[i,:]*b[:,j])^2/v[i,j] =
// sum_{i,j} (m[i,j]^2 - 2m[i,j]a[i,:]*b[:,j] + a[i,:]*(b[:,j] b[:,j]')*a[i,:]')/v[i,j]
// meanTimesPrec(a[i,:]) = sum_j (m[i,j]/v[i,j]) b[:,j]
// prec(a[i,:]) = sum_j b[:,j]*b[:,j]'/v[i,j]
Vector bj = Vector.Zero(inner);
Vector mean = Vector.Zero(inner);
VectorGaussian ai = new VectorGaussian(inner);
PositiveDefiniteMatrix variance = new PositiveDefiniteMatrix(inner, inner);
for (int i = 0; i < rows; i++) {
ai.Precision.SetAllElementsTo(0.0);
ai.MeanTimesPrecision.SetAllElementsTo(0.0);
// we are projecting from family of full covariance Gaussians to diagonal
// covariance, so we should include the context
for (int c = 0; c < inner; c++) {
ai.Precision[c, c] = A[i, c].Precision;
ai.MeanTimesPrecision[c] = A[i, c].MeanTimesPrecision;
}
for (int j = 0; j < cols; j++) {
Gaussian xij = matrixMultiply[i, j];
for (int k = 0; k < inner; k++) {
bj[k] = B[k, j];
}
if (xij.IsPointMass) throw new NotImplementedException(LowRankNotSupportedMessage);
ai.Precision.SetToSumWithOuter(ai.Precision, xij.Precision, bj, bj);
ai.MeanTimesPrecision.SetToSum(1.0, ai.MeanTimesPrecision, xij.MeanTimesPrecision, bj);
}
ai.GetMeanAndVariance(mean, variance);
for (int k = 0; k < inner; k++) {
Gaussian rik = result[i, k];
rik.SetMeanAndVariance(mean[k], variance[k, k]);
result[i, k] = rik / A[i, k];
}
}
return result;
}
/// <summary>
/// VMP message to 'b'
/// </summary>
/// <param name="product">Incoming message from 'product'.</param>
/// <param name="A">Constant value for 'a'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'b' with 'product' integrated out.
/// The formula is <c>sum_product p(product) factor(product,a,b)</c>.
/// </para></remarks>
public static VectorGaussian BAverageLogarithm(VectorGaussian product, Matrix A, VectorGaussian result)
{
return BAverageConditional(product, A, result);
}
public static VectorGaussian RotateAverageLogarithm([SkipIfUniform] Gaussian x, [SkipIfUniform] Gaussian y, double angle, VectorGaussian result)
{
return(RotateAverageLogarithm(x, y, WrappedGaussian.PointMass(angle), result));
}
/// <summary>
/// VMP message to 'B'
/// </summary>
/// <param name="sum">Incoming message from 'Sum'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="A">Constant value for 'A'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'B' with 'Sum' integrated out.
/// The formula is <c>sum_Sum p(Sum) factor(Sum,A,B)</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="sum"/> is not a proper distribution</exception>
public static VectorGaussian BAverageLogarithm([SkipIfUniform] Gaussian sum, bool[] A, VectorGaussian result)
{
if (result == default(VectorGaussian)) result = new VectorGaussian(A.Length);
// E[log N(x; ab, 0)] = -0.5 E[(x-ab)^2]/0 = -0.5 (E[x^2] - 2 E[x] a' E[b] + trace(aa' E[bb']))/0
// message to a = N(a; E[x]*inv(var(b)+E[b]E[b]')*E[b], var(x)*inv(var(b)+E[b]E[b]'))
// result.Precision = (var(b)+E[b]*E[b]')/var(x)
// result.MeanTimesPrecision = E[x]/var(x)*E[b] = E[b]*X.MeanTimesPrecision
Vector ma = Vector.FromArray(A.Select(x => x ? 1.0 : 0.0).ToArray());
// note this is exact if B is a point mass (vb=0).
result.Precision.SetToOuter(ma, ma);
result.Precision.Scale(sum.Precision);
result.MeanTimesPrecision.SetToProduct(ma, sum.MeanTimesPrecision);
return result;
}
public static PositiveDefiniteMatrix BVariance([Proper] VectorGaussian B, PositiveDefiniteMatrix result)
{
return(B.GetVariance(result));
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="AAverageConditional(VectorGaussian, DistributionArray2D{Gaussian, double}, Vector, PositiveDefiniteMatrix, DistributionStructArray2D{Gaussian, double})"]/*'/>
public static DistributionStructArray2D <Gaussian, double> AAverageConditional([SkipIfUniform] VectorGaussian product, DistributionArray2D <Gaussian, double> A, Vector BMean, PositiveDefiniteMatrix BVariance, DistributionStructArray2D <Gaussian, double> result)
{
if (product.IsUniform())
{
result.SetToUniform();
return(result);
}
if (!A.IsPointMass)
{
throw new ArgumentException("A is not a point mass");
}
// logZ = log N(mProduct; A*BMean, vProduct + A*BVariance*A')
// = -0.5 (mProduct - A*BMean)' inv(vProduct + A*BVariance*A') (mProduct - A*BMean) - 0.5 logdet(vProduct + A*BVariance*A')
// = -0.5 (mProduct - A*BMean)' pPrec inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (mProduct - A*BMean)
// - 0.5 logdet(pProduct + pProduct*A*BVariance*A'*pProduct) + logdet(pProduct)
// dlogZ = 0.5 (dA*BMean)' pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (mProduct - A*BMean)
// +0.5 (mProduct - A*BMean)' pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (dA*BMean)
// +0.5 (mProduct - A*BMean)' pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) (pProduct*dA*BVariance*A'*pProduct + pProduct*A*BVariance*dA'*pProduct) inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (mProduct - A*BMean)
// - 0.5 tr(inv(pProduct + pProduct*A*BVariance*A'*pProduct) (pProduct*dA*BVariance*A'*pProduct + pProduct*A*BVariance*dA'*pProduct))
// dlogZ/dA = pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (mProduct - A*BMean) BMean'
// + pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (mProduct - A*BMean) (mProduct - A*BMean)' pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct*A*BVariance
// - pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct A*BVariance
var Amatrix = new Matrix(A.Point);
var pProductA = product.Precision * Amatrix;
var pProductABV = pProductA * BVariance;
PositiveDefiniteMatrix prec = new PositiveDefiniteMatrix(product.Dimension, product.Dimension);
prec.SetToSum(product.Precision, pProductABV * pProductA.Transpose());
// pProductA is now free
for (int i = 0; i < prec.Rows; i++)
{
if (prec[i, i] == 0)
{
prec[i, i] = 1;
}
}
var v = prec.Inverse();
var ABM = Amatrix * BMean;
var pProductABM = product.Precision * ABM;
var diff = pProductABM;
diff.SetToDifference(product.MeanTimesPrecision, pProductABM);
// ABM is now free
var pProductV = product.Precision * v;
var pProductVdiff = ABM;
pProductVdiff.SetToProduct(pProductV, diff);
var Vdiff = v * diff;
pProductV.Scale(-1);
pProductV.SetToSumWithOuter(pProductV, 1, pProductVdiff, Vdiff);
Matrix dlogZ = pProductA;
dlogZ.SetToProduct(pProductV, pProductABV);
dlogZ.SetToSumWithOuter(dlogZ, 1, pProductVdiff, BMean);
int rows = A.GetLength(0);
int cols = A.GetLength(1);
for (int i = 0; i < rows; i++)
{
for (int j = 0; j < cols; j++)
{
double dlogp = dlogZ[i, j];
// for now, we don't compute the second derivative.
double ddlogp = -1;
result[i, j] = Gaussian.FromDerivatives(A[i, j].Point, dlogp, ddlogp, false);
}
}
return(result);
}
public static WrappedGaussian AngleAverageLogarithm([SkipIfUniform] VectorGaussian rotate, [Proper] Gaussian x, [Proper] Gaussian y)
{
return(AngleAverageLogarithm(rotate, x.GetMean(), y.GetMean()));
}
/// <summary>
/// EP message to 'B'
/// </summary>
/// <param name="matrixMultiply">Incoming message from 'matrixMultiply'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="A">Constant value for 'A'.</param>
/// <param name="B">Incoming message from 'B'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'B' as the random arguments are varied.
/// The formula is <c>proj[p(B) sum_(matrixMultiply) p(matrixMultiply) factor(matrixMultiply,A,B)]/p(B)</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="matrixMultiply"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="B"/> is not a proper distribution</exception>
public static GaussianArray2D BAverageConditional([SkipIfUniform] GaussianArray2D matrixMultiply, double[,] A, [SkipIfUniform] GaussianArray2D B, GaussianArray2D result)
{
int rows = matrixMultiply.GetLength(0);
int cols = matrixMultiply.GetLength(1);
int inner = A.GetLength(1);
if (result == null) result = new GaussianArray2D(inner, cols);
var ai = DenseVector.Zero(inner);
var mean = DenseVector.Zero(inner);
PositiveDefiniteMatrix variance = new
PositiveDefiniteMatrix(inner, inner);
var bj = new VectorGaussian(inner);
for (int j = 0; j < cols; j++) {
bj.Precision.SetAllElementsTo(0);
bj.MeanTimesPrecision.SetAllElementsTo(0);
// we are projecting from family of full covariance Gaussians to diagonal
// covariance, so we should include the context
for (int c = 0; c < inner; c++) {
bj.Precision[c, c] = B[c, j].Precision;
bj.MeanTimesPrecision[c] = B[c, j].MeanTimesPrecision;
}
for (int i = 0; i < rows; i++) {
Gaussian xij = matrixMultiply[i, j];
for (int k = 0; k < inner; k++) {
ai[k] = A[i, k];
}
if (xij.IsPointMass) throw new NotImplementedException(LowRankNotSupportedMessage);
bj.Precision.SetToSumWithOuter(bj.Precision, xij.Precision, ai, ai);
bj.MeanTimesPrecision.SetToSum(1.0, bj.MeanTimesPrecision, xij.MeanTimesPrecision, ai);
}
bj.GetMeanAndVariance(mean, variance);
for (int k = 0; k < inner; k++) {
Gaussian rkj = result[k, j];
rkj.SetMeanAndVariance(mean[k], variance[k, k]);
result[k, j] = rkj / B[k, j];
}
}
return result;
}
public static PositiveDefiniteMatrix BVarianceInit([IgnoreDependency] VectorGaussian B)
{
return(new PositiveDefiniteMatrix(B.Dimension, B.Dimension));
}
