/// <summary>
/// VMP message to 'matrixMultiply'
/// </summary>
/// <param name="A">Incoming message from 'A'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="B">Incoming message from 'B'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is a distribution matching the moments of 'matrixMultiply' as the random arguments are varied.
/// The formula is <c>proj[sum_(A,B) p(A,B) factor(matrixMultiply,A,B)]</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="A"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="B"/> is not a proper distribution</exception>
public static GaussianArray2D MatrixMultiplyAverageLogarithm([SkipIfUniform] DistributionArray2D <Gaussian> A, [SkipIfUniform] DistributionArray2D <Gaussian> B, GaussianArray2D result)
{
if (result == null)
{
result = new DistributionStructArray2D <Gaussian, double>(A.GetLength(0), B.GetLength(1));
}
// x[i,j] = sum_k a[i,k]*b[k,j]
// E(x[i,j]) = sum_k E(a[i,k])*E(b[k,j])
// var(x[i,j]) = sum_k E(a[i,k])^2*var(b[k,j]) + var(a[i,k])*E(b[k,j])^2 + var(a[i,k])*var(b[k,j])
int rows = result.GetLength(0);
int cols = result.GetLength(1);
int inner = A.GetLength(1);
for (int i = 0; i < rows; i++)
{
for (int j = 0; j < cols; j++)
{
double mean = 0; double var = 0;
for (int k = 0; k < inner; k++)
{
double Am, Av, Bm, Bv;
A[i, k].GetMeanAndVariance(out Am, out Av);
B[k, j].GetMeanAndVariance(out Bm, out Bv);
mean += Am * Bm;
var += Av * Bm * Bm + Bv * Am * Am + Av * Bv;
}
Gaussian rij = result[i, j];
rij.SetMeanAndVariance(mean, var);
result[i, j] = rij;
}
}
return(result);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="ProductAverageConditional(double[,], Vector, PositiveDefiniteMatrix, VectorGaussianMoments)"]/*'/>
public static VectorGaussianMoments ProductAverageConditional(DistributionArray2D <Gaussian, double> A, Vector BMean, PositiveDefiniteMatrix BVariance, VectorGaussianMoments result)
{
if (!A.IsPointMass)
{
throw new ArgumentException("A is not a point mass");
}
return(ProductAverageConditional(new Matrix(A.Point), BMean, BVariance, result));
}
public void FactorAnalysisMslTest()
{
InferenceEngine engine = new InferenceEngine(new VariationalMessagePassing());
engine.Compiler.DeclarationProvider = RoslynDeclarationProvider.Instance;
double[,] dataIn = MatlabReader.ReadMatrix(new double[400, 64],
Path.Combine(
#if NETCORE
Path.GetDirectoryName(typeof(VmpMslTests).Assembly.Location), // work dir is not the one with Microsoft.ML.Probabilistic.Tests.dll on netcore and neither is .Location on netfull
#endif
"Data", "pca.txt"),
' ');
int C = 8;
engine.ShowProgress = true;
engine.ShowTimings = true;
engine.NumberOfIterations = 20;
engine.Compiler.DeclarationProvider = RoslynDeclarationProvider.Instance;
var ca = engine.Compiler.Compile(FactorAnalysisModel, C, dataIn,
BioTests.RandomGaussianArray(C, dataIn.GetLength(1)),
BioTests.RandomGaussianArray(dataIn.GetLength(0), C));
ca.Execute(engine.NumberOfIterations);
DistributionArray2D <Gaussian> wMarginal = ca.Marginal <DistributionArray2D <Gaussian> >("W");
DistributionArray2D <Gaussian> xMarginal = ca.Marginal <DistributionArray2D <Gaussian> >("x");
//WriteMatrix(wMarginal.ToArray<Gaussian[,]>(), @"..\..\faresultsW.txt");
//WriteMatrix(xMarginal.ToArray<Gaussian[,]>(), @"..\..\faresultsX.txt");
// Reconstruct
DistributionArray2D <Gaussian> productMarginal = MatrixMultiplyOp.MatrixMultiplyAverageLogarithm(xMarginal, wMarginal, null);
double error = 0;
for (int i = 0; i < productMarginal.GetLength(0); i++)
{
for (int j = 0; j < productMarginal.GetLength(1); j++)
{
//Assert.True(productMarginal[i,j].GetLogProb(dataIn[i,j]) > -130);
error += System.Math.Abs(productMarginal[i, j].GetMean() - dataIn[i, j]);
}
}
error /= productMarginal.Count;
// error = 0.121231278712027
Console.WriteLine("error = {0}", error);
Assert.True(error < 0.15); // C=2: 0.15
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="AAverageConditional(VectorGaussian, DistributionArray2D{Gaussian, double}, Vector, PositiveDefiniteMatrix, DistributionStructArray2D{Gaussian, double})"]/*'/>
public static DistributionStructArray2D <Gaussian, double> AAverageConditional([SkipIfUniform] VectorGaussian product, DistributionArray2D <Gaussian, double> A, Vector BMean, PositiveDefiniteMatrix BVariance, DistributionStructArray2D <Gaussian, double> result)
{
if (product.IsUniform())
{
result.SetToUniform();
return(result);
}
if (!A.IsPointMass)
{
throw new ArgumentException("A is not a point mass");
}
// logZ = log N(mProduct; A*BMean, vProduct + A*BVariance*A')
// = -0.5 (mProduct - A*BMean)' inv(vProduct + A*BVariance*A') (mProduct - A*BMean) - 0.5 logdet(vProduct + A*BVariance*A')
// = -0.5 (mProduct - A*BMean)' pPrec inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (mProduct - A*BMean)
// - 0.5 logdet(pProduct + pProduct*A*BVariance*A'*pProduct) + logdet(pProduct)
// dlogZ = 0.5 (dA*BMean)' pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (mProduct - A*BMean)
// +0.5 (mProduct - A*BMean)' pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (dA*BMean)
// +0.5 (mProduct - A*BMean)' pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) (pProduct*dA*BVariance*A'*pProduct + pProduct*A*BVariance*dA'*pProduct) inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (mProduct - A*BMean)
// - 0.5 tr(inv(pProduct + pProduct*A*BVariance*A'*pProduct) (pProduct*dA*BVariance*A'*pProduct + pProduct*A*BVariance*dA'*pProduct))
// dlogZ/dA = pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (mProduct - A*BMean) BMean'
// + pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct (mProduct - A*BMean) (mProduct - A*BMean)' pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct*A*BVariance
// - pProduct inv(pProduct + pProduct*A*BVariance*A'*pProduct) pProduct A*BVariance
var Amatrix = new Matrix(A.Point);
var pProductA = product.Precision * Amatrix;
var pProductABV = pProductA * BVariance;
PositiveDefiniteMatrix prec = new PositiveDefiniteMatrix(product.Dimension, product.Dimension);
prec.SetToSum(product.Precision, pProductABV * pProductA.Transpose());
// pProductA is now free
for (int i = 0; i < prec.Rows; i++)
{
if (prec[i, i] == 0)
{
prec[i, i] = 1;
}
}
var v = prec.Inverse();
var ABM = Amatrix * BMean;
var pProductABM = product.Precision * ABM;
var diff = pProductABM;
diff.SetToDifference(product.MeanTimesPrecision, pProductABM);
// ABM is now free
var pProductV = product.Precision * v;
var pProductVdiff = ABM;
pProductVdiff.SetToProduct(pProductV, diff);
var Vdiff = v * diff;
pProductV.Scale(-1);
pProductV.SetToSumWithOuter(pProductV, 1, pProductVdiff, Vdiff);
Matrix dlogZ = pProductA;
dlogZ.SetToProduct(pProductV, pProductABV);
dlogZ.SetToSumWithOuter(dlogZ, 1, pProductVdiff, BMean);
int rows = A.GetLength(0);
int cols = A.GetLength(1);
for (int i = 0; i < rows; i++)
{
for (int j = 0; j < cols; j++)
{
double dlogp = dlogZ[i, j];
// for now, we don't compute the second derivative.
double ddlogp = -1;
result[i, j] = Gaussian.FromDerivatives(A[i, j].Point, dlogp, ddlogp, false);
}
}
return(result);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="MatrixVectorProductOp"]/message_doc[@name="BAverageConditional(VectorGaussianMoments, DistributionArray2D{Gaussian,double}, VectorGaussian)"]/*'/>
public static VectorGaussian BAverageConditional([SkipIfUniform] VectorGaussianMoments product, DistributionArray2D <Gaussian, double> A, VectorGaussian result)
{
if (!A.IsPointMass)
{
throw new ArgumentException("A is not a point mass");
}
return(BAverageConditional(product, new Matrix(A.Point), result));
}
public void BernoulliMixtureTest()
{
int N = 10, D = 2, K = 2;
Range n = new Range(N).Named("n");
Range k = new Range(K).Named("k");
Range d = new Range(D).Named("d");
VariableArray2D <double> p = Variable.Array <double>(k, d).Named("p");
p[k, d] = Variable.Beta(1, 1).ForEach(k, d);
VariableArray2D <bool> x = Variable.Array <bool>(n, d).Named("x");
VariableArray <int> c = Variable.Array <int>(n).Named("c");
using (Variable.ForEach(n))
{
c[n] = Variable.Discrete(k, 0.5, 0.5);
using (Variable.Switch(c[n]))
{
x[n, d] = Variable.Bernoulli(p[c[n], d]);
}
}
InferenceEngine engine = new InferenceEngine();
bool[,] data = new bool[N, D];
int N1 = N / 2;
int i = 0;
for (; i < N1; i++)
{
data[i, 0] = true;
data[i, 1] = false;
}
for (; i < N; i++)
{
data[i, 0] = false;
data[i, 1] = true;
}
x.ObservedValue = data;
Discrete[] cInit = new Discrete[N];
for (int j = 0; j < N; j++)
{
double r = Rand.Double();
cInit[j] = new Discrete(r, 1 - r);
}
c.InitialiseTo(Distribution <int> .Array(cInit));
engine.NumberOfIterations = 1;
var pExpected = engine.Infer(p);
engine.NumberOfIterations = engine.Algorithm.DefaultNumberOfIterations;
DistributionArray <Discrete> cPost = engine.Infer <DistributionArray <Discrete> >(c);
Console.WriteLine(cPost);
DistributionArray2D <Beta> pPost = engine.Infer <DistributionArray2D <Beta> >(p);
Console.WriteLine(pPost);
// test resetting inference
engine.NumberOfIterations = 1;
var pActual = engine.Infer <Diffable>(p);
Assert.True(pActual.MaxDiff(pExpected) < 1e-10);
}
public void BernoulliMixtureGaussianTest()
{
int N = 10, D = 2, K = 2;
Range n = new Range(N).Named("n");
Range k = new Range(K).Named("k");
Range d = new Range(D).Named("d");
VariableArray2D <double> p = Variable.Array <double>(k, d).Named("p");
p[k, d] = Variable.GaussianFromMeanAndVariance(0, 1).ForEach(k, d);
VariableArray2D <bool> x = Variable.Array <bool>(n, d).Named("x");
VariableArray <int> c = Variable.Array <int>(n).Named("c");
using (Variable.ForEach(n))
{
c[n] = Variable.Discrete(k, 0.5, 0.5);
using (Variable.Switch(c[n]))
{
x[n, d] = (Variable.GaussianFromMeanAndVariance(p[c[n], d], 1.0) > 0);
}
}
bool geForceProper = GateEnterOp <double> .ForceProper;
try
{
GateEnterOp <double> .ForceProper = true;
InferenceEngine engine = new InferenceEngine(); //new VariationalMessagePassing());
engine.Compiler.GivePriorityTo(typeof(IsPositiveOp_Proper)); // needed to avoid improper messages in EP
bool[,] data = new bool[N, D];
int N1 = N / 2;
int i = 0;
for (; i < N1; i++)
{
data[i, 0] = true;
data[i, 1] = false;
}
for (; i < N; i++)
{
data[i, 0] = false;
data[i, 1] = true;
}
x.ObservedValue = data;
Discrete[] cInit = new Discrete[N];
for (int j = 0; j < N; j++)
{
double r = Rand.Double();
cInit[j] = new Discrete(r, 1 - r);
}
c.InitialiseTo(Distribution <int> .Array(cInit));
engine.NumberOfIterations = 1;
var pExpected = engine.Infer(p);
engine.NumberOfIterations = engine.Algorithm.DefaultNumberOfIterations;
DistributionArray <Discrete> cPost = engine.Infer <DistributionArray <Discrete> >(c);
Console.WriteLine(cPost);
DistributionArray2D <Gaussian> pPost = engine.Infer <DistributionArray2D <Gaussian> >(p);
Console.WriteLine(pPost);
// test resetting inference
engine.NumberOfIterations = 1;
var pActual = engine.Infer <Diffable>(p);
Assert.True(pActual.MaxDiff(pExpected) < 1e-10);
}
finally
{
GateEnterOp <double> .ForceProper = geForceProper;
}
}
public static GaussianArray2D MatrixMultiplyAverageLogarithmInit([IgnoreDependency] DistributionArray2D <Gaussian> A, [IgnoreDependency] double[,] B)
{
return(new DistributionStructArray2D <Gaussian, double>(A.GetLength(0), B.GetLength(1)));
}
