/// Generate LDA data - returns an array of dictionaries mapping unique word index
/// to word count per document.
/// <param name="trueTheta">Known Theta</param>
/// <param name="truePhi">Known Phi</param>
/// <param name="averageNumWords">Average number of words to sample per doc</param>
/// <returns></returns>
public static Dictionary<int, int>[] GenerateLDAData(Dirichlet[] trueTheta, Dirichlet[] truePhi, int averageNumWords)
{
int numVocab = truePhi[0].Dimension;
int numTopics = truePhi.Length;
int numDocs = trueTheta.Length;
// Sample from the model
Vector[] topicDist = new Vector[numDocs];
Vector[] wordDist = new Vector[numTopics];
for (int i = 0; i < numDocs; i++)
topicDist[i] = trueTheta[i].Sample();
for (int i = 0; i < numTopics; i++)
wordDist[i] = truePhi[i].Sample();
var wordCounts = new Dictionary<int, int>[numDocs];
for (int i=0; i < numDocs; i++)
{
int LengthOfDoc = Poisson.Sample((double)averageNumWords);
var counts = new Dictionary<int, int>();
for (int j=0; j < LengthOfDoc; j++)
{
int topic = Discrete.Sample(topicDist[i]);
int w = Discrete.Sample(wordDist[topic]);
if (!counts.ContainsKey(w))
counts.Add(w, 1);
else
counts[w] = counts[w] + 1;
}
wordCounts[i] = counts;
}
return wordCounts;
}
public void CanGetAlpha()
{
var d = new Dirichlet(0.3, 10);
for (var i = 0; i < 10; i++)
{
Assert.AreEqual(0.3, d.Alpha[i]);
}
}
public void CanCreateSymmetricDirichlet()
{
var d = new Dirichlet(0.3, 5);
for (var i = 0; i < 5; i++)
{
Assert.AreEqual(0.3, d.Alpha[i]);
}
}
public void CanGetAlpha()
{
Dirichlet d = new Dirichlet(0.3, 10);
double[] alpha = new double[10];
for (int i = 0; i < 10; i++)
{
Assert.AreEqual(0.3, d.Alpha[i]);
}
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="sample">Incoming message from 'sample'.</param>
/// <param name="trialCount">Constant value for 'trialCount'.</param>
/// <param name="p">Incoming message from 'p'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(sample,p) p(sample,p) factor(sample,trialCount,p))</c>.
/// </para></remarks>
public static double LogAverageFactor(IList<int> sample, int trialCount, Dirichlet p)
{
double result = MMath.GammaLn(trialCount+1);
for (int i = 0; i < sample.Count; i++) {
result += MMath.GammaLn(sample[i]+p.PseudoCount[i]) + MMath.GammaLn(p.PseudoCount[i])
-MMath.GammaLn(sample[i]+1);
}
result += MMath.GammaLn(p.TotalCount) - MMath.GammaLn(p.TotalCount + trialCount);
return result;
}
public void CanCreateDirichlet()
{
var alpha = new double[10];
for (var i = 0; i < 10; i++)
{
alpha[i] = i;
}
var d = new Dirichlet(alpha);
for (var i = 0; i < 5; i++)
{
Assert.AreEqual(i, d.Alpha[i]);
}
}
/// <summary>
/// Randomly create true theta and phi arrays
/// </summary>
/// <param name="numVocab">Vocabulary size</param>
/// <param name="numTopics">Number of topics</param>
/// <param name="numDocs">Number of documents</param>
/// <param name="averageDocLength">Avarage document length</param>
/// <param name="trueTheta">Theta array (output)</param>
/// <param name="truePhi">Phi array (output)</param>
public static void CreateTrueThetaAndPhi(
int numVocab, int numTopics, int numDocs, int averageDocLength, int averageWordsPerTopic,
out Dirichlet[] trueTheta, out Dirichlet[] truePhi)
{
truePhi = new Dirichlet[numTopics];
for (int i=0; i < numTopics; i++)
{
truePhi[i] = Dirichlet.Uniform(numVocab);
truePhi[i].PseudoCount.SetAllElementsTo(0.0);
// Draw the number of unique words in the topic.
int numUniqueWordsPerTopic = Poisson.Sample((double)averageWordsPerTopic);
if (numUniqueWordsPerTopic >= numVocab) numUniqueWordsPerTopic = numVocab;
if (numUniqueWordsPerTopic < 1) numUniqueWordsPerTopic = 1;
double expectedRepeatOfWordInTopic =
((double)numDocs) * averageDocLength / numUniqueWordsPerTopic;
int[] shuffledWordIndices = Rand.Perm(numVocab);
for (int j = 0; j < numUniqueWordsPerTopic; j++)
{
int wordIndex = shuffledWordIndices[j];
// Draw the count for that word
int cnt = Poisson.Sample(expectedRepeatOfWordInTopic);
truePhi[i].PseudoCount[wordIndex] = cnt + 1.0;
}
}
trueTheta = new Dirichlet[numDocs];
for (int i=0; i < numDocs; i++)
{
trueTheta[i] = Dirichlet.Uniform(numTopics);
trueTheta[i].PseudoCount.SetAllElementsTo(0.0);
// Draw the number of unique topics in the doc.
int numUniqueTopicsPerDoc = Math.Min(1 + Poisson.Sample(1.0), numTopics);
double expectedRepeatOfTopicInDoc =
averageDocLength / numUniqueTopicsPerDoc;
int[] shuffledTopicIndices = Rand.Perm(numTopics);
for (int j = 0; j < numUniqueTopicsPerDoc; j++)
{
int topicIndex = shuffledTopicIndices[j];
// Draw the count for that topic
int cnt = Poisson.Sample(expectedRepeatOfTopicInDoc);
trueTheta[i].PseudoCount[topicIndex] = cnt + 1.0;
}
}
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="prob">Incoming message from 'prob'.</param>
/// <param name="mean">Constant value for 'mean'.</param>
/// <param name="to_prob">Previous outgoing message to 'prob'.</param>
/// <param name="totalCount">Constant value for 'totalCount'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(sum_(prob) p(prob) factor(prob,mean,totalCount))</c>.
/// </para></remarks>
public static double LogAverageFactor(Dirichlet prob, Vector mean, Dirichlet to_prob, double totalCount)
{
return to_prob.GetLogAverageOf(prob);
}
public void DiscreteFromDirichletOpTest()
{
Dirichlet probs4 = new Dirichlet(1.0, 2, 3, 4);
Discrete sample4 = new Discrete(0.4, 0.6, 0, 0);
Dirichlet result4 = DiscreteFromDirichletOp.ProbsAverageConditional(sample4, probs4, Dirichlet.Uniform(4));
Dirichlet probs3 = new Dirichlet(1.0, 2, 7);
Discrete sample3 = new Discrete(0.4, 0.6, 0);
Dirichlet result3 = DiscreteFromDirichletOp.ProbsAverageConditional(sample3, probs3, Dirichlet.Uniform(3));
for (int i = 0; i < 3; i++)
{
Assert.True(MMath.AbsDiff(result4.PseudoCount[i], result3.PseudoCount[i], 1e-6) < 1e-10);
}
Dirichlet probs2 = new Dirichlet(1.0, 2);
Discrete sample2 = new Discrete(0.4, 0.6);
Dirichlet result2 = DiscreteFromDirichletOp.ProbsAverageConditional(sample2, probs2, Dirichlet.Uniform(2));
Beta beta = new Beta(1.0, 2);
Bernoulli bernoulli = new Bernoulli(0.4);
Beta result1 = BernoulliFromBetaOp.ProbTrueAverageConditional(bernoulli, beta);
Assert.Equal(result2.PseudoCount[0], result1.TrueCount, 1e-10);
Assert.Equal(result2.PseudoCount[1], result1.FalseCount, 1e-10);
// test handling of small alphas
Discrete sample = DiscreteFromDirichletOp.SampleAverageLogarithm(Dirichlet.Symmetric(2, 1e-8), Discrete.Uniform(2));
Assert.True(sample.IsUniform());
}
/// <summary>
/// Perform the quadrature required for the VMP evidence message
/// </summary>
/// <param name="meanQ">Incoming message from m='mean'.</param>
/// <param name="totalCountQ">Incoming message from s='totalCount'.</param>
/// <returns>Vector of E[ LogGamma(s*m_k)].</returns>
/// <remarks><para>
/// The quadrature over 'totalCount' (which is Gamma-distributed) is
/// peformed by a change of variable x=log(s) followed by Gauss-Hermite
/// quadrature. The quadrature over m is performed using Gauss-Legendre.
/// </para></remarks>
public static Vector EvidenceMessageExpectations(
Dirichlet meanQ,
Gamma totalCountQ)
{
// Get shape and scale of the distribution
double at, bt;
totalCountQ.GetShapeAndScale(out at, out bt);
bt = 1 / bt; // want rate not scale
// Mean in the transformed domain
double proposalMean = totalCountQ.GetMeanLog();
// Laplace approximation of variance in transformed domain
double proposalVariance = 1 / at;
// Quadrature coefficient
int nt = 32;
Vector nodes = Vector.Zero(nt);
Vector weights = Vector.Zero(nt);
Vector expx = Vector.Zero(nt);
if (!totalCountQ.IsPointMass) {
Quadrature.GaussianNodesAndWeights(proposalMean, proposalVariance, nodes, weights);
// Precompute weights for each m slice
for (int i = 0; i < nt; i++) {
double x = nodes[i];
expx[i] = Math.Exp(x);
double p = at * x - bt * expx[i] - Gaussian.GetLogProb(x, proposalMean, proposalVariance);
weights[i] *= Math.Exp(p);
}
}
int nm = 20;
Vector mnodes = Vector.Zero(nm);
Vector mweight = Vector.Zero(nm);
Quadrature.UniformNodesAndWeights(0, 1, mnodes, mweight);
int K = meanQ.Dimension;
Vector[] mweights = new Vector[K];
Beta[] mkDist = new Beta[K];
double[] EELogGamma = new double[K];
for (int i = 0; i < K; i++) {
mweights[i] = Vector.Copy(mweight);
mkDist[i] = new Beta(meanQ.PseudoCount[i], meanQ.TotalCount - meanQ.PseudoCount[i]);
EELogGamma[i] = 0;
}
for (int j = 0; j < nm; j++) {
double m = mnodes[j];
double ELogGamma = 0;
if (totalCountQ.IsPointMass)
ELogGamma = MMath.GammaLn(m * totalCountQ.Point);
else {
// Calculate expectations in x=log(s) space using Gauss-Hermite quadrature
for (int i = 0; i < nt; i++) {
double x = nodes[i];
ELogGamma += weights[i] * (MMath.GammaLn(m * expx[i]) + x);
}
// Normalise and add removed components
double normalisation = Math.Pow(bt, at) / MMath.Gamma(at);
ELogGamma = normalisation * ELogGamma - proposalMean;
}
for (int i = 0; i < K; i++) {
mweights[i][j] *= Math.Exp(mkDist[i].GetLogProb(m));
EELogGamma[i] += mweights[i][j] * ELogGamma;
}
}
return Vector.FromArray(EELogGamma);
}
/// <summary>
/// Attachs the data to the workers labels with and sets the workers' confusion matrix priors.
/// </summary>
/// <param name="taskIndices">The matrix of the task indices (columns) of each worker (rows).</param>
/// <param name="workerLabels">The matrix of the labels (columns) of each worker (rows).</param>
/// <param name="confusionMatrixPrior">The workers' confusion matrix priors.</param>
protected virtual void AttachData(int[][] taskIndices, int[][] workerLabels, Dirichlet[][] confusionMatrixPrior)
{
int numClasses = c.SizeAsInt;
WorkerCount.ObservedValue = taskIndices.Length;
WorkerTaskCount.ObservedValue = taskIndices.Select(tasks => tasks.Length).ToArray();
WorkerTaskIndex.ObservedValue = taskIndices;
// Prediction mode is indicated by none of the workers having a label.
// We can just look at the first one
if (workerLabels[0] != null)
{
WorkerLabel.ObservedValue = workerLabels;
}
else
{
WorkerLabel.ClearObservedValue();
}
if (confusionMatrixPrior != null)
{
ConfusionMatrixPrior.ObservedValue = Util.ArrayInit(confusionMatrixPrior.Length, worker => Util.ArrayInit(numClasses, lab => confusionMatrixPrior[worker][lab]));
}
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="DiscreteFromDirichletOp"]/message_doc[@name="SampleAverageConditional(Dirichlet, Discrete)"]/*'/>
public static Discrete SampleAverageConditional([SkipIfUniform] Dirichlet probs, Discrete result)
{
result.SetProbs(probs.GetMean(result.GetWorkspace()));
return(result);
}
/// <summary>
/// Evidence message for VMP
/// </summary>
/// <param name="sample">Incoming message from 'sampleFromPseudoCounts'.</param>
/// <param name="pseudoCounts">Constant value for 'pseudoCount'.</param>
/// <param name="to_sample">Outgoing message to 'sample'.</param>
/// <returns>Average of the factor's log-value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>sum_(sampleFromPseudoCounts) p(sampleFromPseudoCounts) log(factor(sampleFromPseudoCounts,pseudoCount))</c>.
/// Adding up these values across all factors and variables gives the log-evidence estimate for VMP.
/// </para></remarks>
public static double AverageLogFactor(Dirichlet sample, Vector pseudoCounts, [Fresh] Dirichlet to_sample)
{
return to_sample.GetAverageLog(sample);
}
public static Dirichlet ProbAverageConditional([SkipIfUniform] Gamma alpha, Dirichlet result)
{
throw new NotSupportedException(NotSupportedMessage);
}
public void DirichletOpQuadratureTest()
{
var matlabResults
= new double[]
{
0.625872049875551,
0.866057568760984,
-0.266065360660541,
-1.227320719860393,
1.280900246404125
};
var matlabResults2
= new double[]
{
0.843302107208523,
0.610546297106219,
-2.182481855300747,
-0.254011377373013,
-0.217430057568389
};
double am = 2;
double bm = 1;
double at = 3;
double bt = 1;
Dirichlet meanQ = new Dirichlet(new double[] {am, bm});
Gamma totalCountQ = new Gamma(at, 1/bt);
double[] EELogGamma;
double[] EELogMLogGamma;
double[] EELogOneMinusMLogGamma;
double[] EELogSLogGamma;
double[] EEMSDigamma;
DirichletOp.MeanMessageExpectations(
meanQ.PseudoCount,
totalCountQ,
out EELogGamma,
out EELogMLogGamma,
out EELogOneMinusMLogGamma);
Console.WriteLine(System.Math.Abs(EELogGamma[0] - matlabResults[0]));
Console.WriteLine(System.Math.Abs(EELogMLogGamma[0] - matlabResults[2]));
Console.WriteLine(System.Math.Abs(EELogOneMinusMLogGamma[0] - matlabResults[3]));
Console.WriteLine(System.Math.Abs(EELogGamma[1] - matlabResults2[0]));
Console.WriteLine(System.Math.Abs(EELogMLogGamma[1] - matlabResults2[2]));
Console.WriteLine(System.Math.Abs(EELogOneMinusMLogGamma[1] - matlabResults2[3]));
DirichletOp.TotalCountMessageExpectations(
meanQ.PseudoCount,
totalCountQ,
out EELogGamma,
out EELogSLogGamma,
out EEMSDigamma);
Console.WriteLine(System.Math.Abs(EELogGamma[0] - matlabResults[0]));
Console.WriteLine(System.Math.Abs(EELogSLogGamma[0] - matlabResults[1]));
Console.WriteLine(System.Math.Abs(EEMSDigamma[0] - matlabResults[4]));
Console.WriteLine(System.Math.Abs(EELogGamma[1] - matlabResults2[0]));
Console.WriteLine(System.Math.Abs(EELogSLogGamma[1] - matlabResults2[1]));
Console.WriteLine(System.Math.Abs(EEMSDigamma[1] - matlabResults2[4]));
}
/// <summary>
/// Constructs an LDA model
/// </summary>
/// <param name="sizeVocab">Size of vocabulary</param>
/// <param name="numTopics">Number of topics</param>
public LDAShared(int numBatches, int sizeVocab, int numTopics)
{
SizeVocab = sizeVocab;
NumTopics = numTopics;
ThetaSparsity = Sparsity.Dense;
PhiSparsity = Sparsity.ApproximateWithTolerance(0.00000000001); // Allow for round-off error
NumDocuments = Variable.New <int>().Named("NumDocuments");
NumBatches = numBatches;
IterationsPerPass = new int[] { 1, 3, 5, 7, 9 };
//---------------------------------------------
// The model
//---------------------------------------------
Range D = new Range(NumDocuments).Named("D");
Range W = new Range(SizeVocab).Named("W");
Range T = new Range(NumTopics).Named("T");
NumWordsInDoc = Variable.Array <int>(D).Named("NumWordsInDoc");
Range WInD = new Range(NumWordsInDoc[D]).Named("WInD");
Evidence = SharedVariable <bool> .Random(new Bernoulli(0.5)).Named("Evidence");
Evidence.IsEvidenceVariable = true;
Phi = SharedVariable <Vector> .Random(T, CreateUniformDirichletArray(numTopics, sizeVocab, PhiSparsity)).Named("Phi");
// Phi definition sub-model - just one copy
PhiDefModel = new Model(1).Named("PhiDefModel");
IfBlock evidencePhiDefBlock = null;
EvidencePhiDef = Evidence.GetCopyFor(PhiDefModel).Named("EvidencePhiDef");
evidencePhiDefBlock = Variable.If(EvidencePhiDef);
PhiDef = Variable.Array <Vector>(T).Named("PhiDef");
PhiDef.SetSparsity(PhiSparsity);
PhiDef.SetValueRange(W);
PhiPrior = Variable.Array <Dirichlet>(T).Named("PhiPrior");
PhiDef[T] = Variable <Vector> .Random(PhiPrior[T]);
Phi.SetDefinitionTo(PhiDefModel, PhiDef);
evidencePhiDefBlock.CloseBlock();
// Document sub-model - many copies
DocModel = new Model(numBatches).Named("DocModel");
IfBlock evidenceDocBlock = null;
EvidenceDoc = Evidence.GetCopyFor(DocModel).Named("EvidenceDoc");
evidenceDocBlock = Variable.If(EvidenceDoc);
Theta = Variable.Array <Vector>(D).Named("Theta");
Theta.SetSparsity(ThetaSparsity);
Theta.SetValueRange(T);
ThetaPrior = Variable.Array <Dirichlet>(D).Named("ThetaPrior");
Theta[D] = Variable <Vector> .Random(ThetaPrior[D]);
PhiDoc = Phi.GetCopyFor(DocModel);
PhiDoc.AddAttribute(new MarginalPrototype(Dirichlet.Uniform(sizeVocab, PhiSparsity)));
Words = Variable.Array(Variable.Array <int>(WInD), D).Named("Words");
WordCounts = Variable.Array(Variable.Array <double>(WInD), D).Named("WordCounts");
using (Variable.ForEach(D))
{
using (Variable.ForEach(WInD))
{
using (Variable.Repeat(WordCounts[D][WInD]))
{
Variable <int> topic = Variable.Discrete(Theta[D]).Named("topic");
using (Variable.Switch(topic))
{
Words[D][WInD] = Variable.Discrete(PhiDoc[topic]);
}
}
}
}
evidenceDocBlock.CloseBlock();
// Initialization to break symmetry
ThetaInit = Variable.New <IDistribution <Vector[]> >().Named("ThetaInit");
Theta.InitialiseTo(ThetaInit);
EnginePhiDef = new InferenceEngine(new VariationalMessagePassing());
EnginePhiDef.Compiler.ShowWarnings = false;
EnginePhiDef.ModelName = "LDASharedPhiDef";
Engine = new InferenceEngine(new VariationalMessagePassing());
Engine.OptimiseForVariables = new IVariable[] { Theta, PhiDoc, EvidenceDoc };
Engine.Compiler.ShowWarnings = false;
Engine.ModelName = "LDAShared";
Engine.Compiler.ReturnCopies = false;
Engine.Compiler.FreeMemory = true;
}
public static Discrete SampleAverageLogarithmInit([IgnoreDependency] Dirichlet probs)
{
return(Discrete.Uniform(probs.Dimension));
}
public void MixtureOfMultivariateGaussians()
{
// Define a range for the number of mixture components
Range k = new Range(2).Named("k");
// Mixture component means
VariableArray <Vector> means = Variable.Array <Vector>(k).Named("means");
means[k] = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(2), PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// Mixture component precisions
VariableArray <PositiveDefiniteMatrix> precs = Variable.Array <PositiveDefiniteMatrix>(k).Named("precs");
precs[k] = Variable.WishartFromShapeAndScale(100.0, PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// Mixture weights
Variable <Vector> weights = Variable.Dirichlet(k, new double[] { 1, 1 }).Named("weights");
// Create a variable array which will hold the data
Range n = new Range(300).Named("n");
VariableArray <Vector> data = Variable.Array <Vector>(n).Named("x");
// Create latent indicator variable for each data point
VariableArray <int> z = Variable.Array <int>(n).Named("z");
// The mixture of Gaussians model
using (Variable.ForEach(n))
{
z[n] = Variable.Discrete(weights);
using (Variable.Switch(z[n]))
{
data[n] = Variable.VectorGaussianFromMeanAndPrecision(means[z[n]], precs[z[n]]);
}
}
// Attach some generated data
double truePi = 0.6;
data.ObservedValue = GenerateData(n.SizeAsInt, truePi);
// Initialise messages randomly to break symmetry
VariableArray <Discrete> zInit = Variable.Array <Discrete>(n).Named("zInit");
bool useObservedValue = true;
if (useObservedValue)
{
zInit.ObservedValue = Util.ArrayInit(n.SizeAsInt, i => Discrete.PointMass(Rand.Int(k.SizeAsInt), k.SizeAsInt));
}
else
{
// This approach doesn't work, because Infer.NET notices that Rand.Int is stochastic and thinks that it should perform message-passing here.
using (Variable.ForEach(n))
{
var randk = Variable <int> .Factor(new Func <int, int>(Rand.Int), (Variable <int>) k.Size);
randk.SetValueRange(k);
zInit[n] = Variable <Discrete> .Factor(Discrete.PointMass, randk, (Variable <int>) k.Size);
}
}
z[n].InitialiseTo(zInit[n]);
// The inference
InferenceEngine ie = new InferenceEngine();
ie.Algorithm = new VariationalMessagePassing();
//ie.Compiler.GenerateInMemory = false;
//ie.NumberOfIterations = 200;
Dirichlet wDist = (Dirichlet)ie.Infer(weights);
Vector wEstMean = wDist.GetMean();
object meansActual = ie.Infer(means);
Console.WriteLine("means = ");
Console.WriteLine(meansActual);
var precsActual = ie.Infer <IList <Wishart> >(precs);
Console.WriteLine("precs = ");
Console.WriteLine(precsActual);
Console.WriteLine("w = {0} should be {1}", wEstMean, Vector.FromArray(truePi, 1 - truePi));
//Console.WriteLine(StringUtil.JoinColumns("z = ", ie.Infer(z)));
Assert.True(
MMath.AbsDiff(wEstMean[0], truePi) < 0.05 ||
MMath.AbsDiff(wEstMean[1], truePi) < 0.05);
}
public static double LogEvidenceRatio(Discrete sample, Dirichlet probs)
{
return(0.0);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="DiscreteFromDirichletOp"]/message_doc[@name="LogEvidenceRatio(int, Dirichlet)"]/*'/>
public static double LogEvidenceRatio(int sample, Dirichlet probs)
{
return(LogAverageFactor(sample, probs));
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="DiscreteFromDirichletOp"]/message_doc[@name="LogAverageFactor(int, Dirichlet)"]/*'/>
public static double LogAverageFactor(int sample, Dirichlet probs)
{
Discrete to_sample = SampleAverageConditional(probs, Discrete.Uniform(probs.Dimension, probs.Sparsity));
return(to_sample.GetLogProb(sample));
}
/// <summary>
/// Buffer for E[log(prob)]
/// </summary>
/// <param name="prob">Incoming message from 'prob'</param>
/// <param name="result">Will be the returned value. </param>
/// <returns>E[log(prob)]</returns>
public static Vector ProbMeanLog(Dirichlet prob, Vector result)
{
prob.GetMeanLog(result);
return result;
}
/// <summary>
/// Runs inference on the LDA model.
/// <para>
/// Words in documents are observed, topic distributions per document (<see cref="Theta"/>)
/// and word distributions per topic (<see cref="Phi"/>) are inferred.
/// </para>
/// </summary>
/// <param name="wordsInDoc">For each document, the unique word counts in the document</param>
/// <param name="alpha">Hyper-parameter for <see cref="Theta"/></param>
/// <param name="beta">Hyper-parameter for <see cref="Phi"/></param>
/// <param name="postTheta">Posterior marginals for <see cref="Theta"/></param>
/// <param name="postPhi">Posterior marginals for <see cref="Phi"/></param>
/// <returns>Log evidence - can be used for model selection.</returns>
public virtual double Infer(Dictionary <int, int>[] wordsInDoc, double alpha, double beta, out Dirichlet[] postTheta, out Dirichlet[] postPhi)
{
int numDocs = wordsInDoc.Length;
postTheta = new Dirichlet[numDocs];
int numIters = Engine.NumberOfIterations;
bool showProgress = Engine.ShowProgress;
Engine.ShowProgress = false; // temporarily disable Infer.NET progress
// Set up document index boundaries for each batch
double numDocsPerBatch = ((double)numDocs) / NumBatches;
if (numDocsPerBatch == 0)
{
numDocsPerBatch = 1;
}
int[] boundary = new int[NumBatches + 1];
boundary[0] = 0;
double currBoundary = 0.0;
for (int batch = 1; batch <= NumBatches; batch++)
{
currBoundary += numDocsPerBatch;
int bnd = (int)currBoundary;
if (bnd > numDocs)
{
bnd = numDocs;
}
boundary[batch] = bnd;
}
boundary[NumBatches] = numDocs;
PhiPrior.ObservedValue = new Dirichlet[NumTopics];
for (int i = 0; i < NumTopics; i++)
{
PhiPrior.ObservedValue[i] = Dirichlet.Symmetric(SizeVocab, beta);
}
NumDocuments.ObservedValue = -1;
try
{
for (int pass = 0; pass < NumPasses; pass++)
{
Engine.NumberOfIterations = IterationsPerPass[pass];
if (showProgress)
{
Console.Write(String.Format(
"\nPass {0} ({1} iteration{2} per batch)",
pass, IterationsPerPass[pass], IterationsPerPass[pass] == 1 ? "" : "s"));
}
PhiDefModel.InferShared(EnginePhiDef, 0);
for (int batch = 0; batch < NumBatches; batch++)
{
int startDoc = boundary[batch];
int endDoc = boundary[batch + 1];
if (startDoc >= numDocs)
{
break;
}
int numDocsInThisBatch = endDoc - startDoc;
if (pass == 0)
{
ThetaInit.ObservedValue = LDAModel.GetInitialisation(numDocsInThisBatch, NumTopics, ThetaSparsity);
}
else
{
var thetaInit = new DirichletArray(numDocsInThisBatch);
for (int d = 0; d < numDocsInThisBatch; d++)
{
thetaInit[d] = new Dirichlet(postTheta[d + startDoc]);
}
ThetaInit.ObservedValue = thetaInit;
}
// Set up the observed values
if (NumDocuments.ObservedValue != numDocsInThisBatch)
{
NumDocuments.ObservedValue = numDocsInThisBatch;
ThetaPrior.ObservedValue = new Dirichlet[numDocsInThisBatch];
for (int i = 0; i < numDocsInThisBatch; i++)
{
ThetaPrior.ObservedValue[i] = Dirichlet.Symmetric(NumTopics, alpha);
}
}
int[] numWordsInDocBatch = new int[numDocsInThisBatch];
int[][] wordsInDocBatch = new int[numDocsInThisBatch][];
double[][] wordCountsInDocBatch = new double[numDocsInThisBatch][];
for (int i = 0, j = startDoc; j < endDoc; i++, j++)
{
numWordsInDocBatch[i] = wordsInDoc[j].Count;
wordsInDocBatch[i] = wordsInDoc[j].Keys.ToArray();
ICollection <int> cnts = wordsInDoc[j].Values;
wordCountsInDocBatch[i] = new double[cnts.Count];
int k = 0;
foreach (int val in cnts)
{
wordCountsInDocBatch[i][k++] = (double)val;
}
}
NumWordsInDoc.ObservedValue = numWordsInDocBatch;
Words.ObservedValue = wordsInDocBatch;
WordCounts.ObservedValue = wordCountsInDocBatch;
DocModel.InferShared(Engine, batch);
var postThetaBatch = Engine.Infer <Dirichlet[]>(Theta);
for (int i = 0, j = startDoc; j < endDoc; i++, j++)
{
postTheta[j] = postThetaBatch[i];
}
postPhi = Distribution.ToArray <Dirichlet[]>(Phi.Marginal <IDistribution <Vector[]> >());
if (showProgress)
{
if ((batch % 80) == 0)
{
Console.WriteLine("");
}
Console.Write(".");
}
}
}
}
finally
{
Engine.NumberOfIterations = numIters;
Engine.ShowProgress = showProgress;
}
if (showProgress)
{
Console.WriteLine();
}
postPhi = Distribution.ToArray <Dirichlet[]>(Phi.Marginal <IDistribution <Vector[]> >());
return(Model.GetEvidenceForAll(PhiDefModel, DocModel));
}
public static Dirichlet ProbAverageLogarithm([SkipIfUniform] Gamma alpha, Dirichlet result)
{
result.PseudoCount.SetAllElementsTo(alpha.GetMean());
result.TotalCount = result.PseudoCount.Sum();
return result;
}
/// <inheritdoc />
public override void SetDefaultPriors()
{
base.SetDefaultPriors();
this.RandomGuessPrior.ObservedValue = Dirichlet.Uniform(this.Labels.SizeAsInt);
this.AbilityPrior.ObservedValue = Util.ArrayInit(this.WorkerCount, input => new Beta(2, 1));
}
public static double LogEvidenceRatio(Dirichlet prob, double alpha) { return 0.0; }
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="CharFromProbabilitiesOp"]/message_doc[@name="ProbabilitiesAverageConditional(DiscreteChar, Dirichlet, Dirichlet)"]/*'/>
public static Dirichlet ProbabilitiesAverageConditional([SkipIfUniform] DiscreteChar character, Dirichlet probabilities, Dirichlet result)
{
return(DiscreteFromDirichletOp.ProbsAverageConditional(new Discrete(character.GetProbs()), probabilities, result));
}
/// <summary>
/// VMP message to 'mean'
/// </summary>
/// <param name="mean">Incoming message from 'mean'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="totalCount">Constant value for 'totalCount'.</param>
/// <param name="prob">Incoming message from 'prob'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="to_mean">Previous outgoing message to 'mean'.</param>
/// <returns>The outgoing VMP message to the 'mean' argument</returns>
/// <remarks><para>
/// The outgoing message is the exponential of the average log-factor value, where the average is over all arguments except 'mean'.
/// The formula is <c>exp(sum_(prob) p(prob) log(factor(prob,mean,totalCount)))</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="mean"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="prob"/> is not a proper distribution</exception>
public static Dirichlet MeanAverageLogarithm([Proper] Dirichlet mean, double totalCount, [SkipIfUniform] Dirichlet prob, Dirichlet to_mean)
{
return MeanAverageLogarithm(mean, Gamma.PointMass(totalCount), prob, to_mean);
}
public static double LogEvidenceRatio(Dirichlet probabilities, DiscreteChar character)
{
return(0);
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="DiscreteFromDirichletOp"]/message_doc[@name="AverageLogFactor(int, Dirichlet)"]/*'/>
public static double AverageLogFactor(int sample, [Proper] Dirichlet probs)
{
return(probs.GetMeanLogAt(sample));
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="CharFromProbabilitiesOp"]/message_doc[@name="LogEvidenceRatio(Dirichlet, Char)"]/*'/>
public static double LogEvidenceRatio(Dirichlet probabilities, char character)
{
return(DiscreteFromDirichletOp.LogEvidenceRatio(character, probabilities));
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="DiscreteFromDirichletOp"]/message_doc[@name="ProbsAverageLogarithm(int, Dirichlet)"]/*'/>
public static Dirichlet ProbsAverageLogarithm(int sample, Dirichlet result)
{
return(ProbsConditional(sample, result));
}
public void InferPosteriors()
{
CPTTransPosterior = Engine.Infer <Dirichlet[]>(CPTTrans);
ProbInitPosterior = Engine.Infer <Dirichlet>(ProbInit);
ModelEvidencePosterior = Engine.Infer <Bernoulli>(ModelEvidence);
}
public void CanGetDimension()
{
var d = new Dirichlet(0.3, 10);
Assert.AreEqual(10, d.Dimension);
}
public void SetUninformedPriors()
{
ProbInitPrior.ObservedValue = Dirichlet.Uniform(K.SizeAsInt);
CPTTransPrior.ObservedValue = Util.ArrayInit(K.SizeAsInt, k => Dirichlet.Uniform(K.SizeAsInt)).ToArray();
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="prob">Constant value for 'prob'.</param>
/// <param name="mean">Constant value for 'mean'.</param>
/// <param name="totalCount">Constant value for 'totalCount'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log(factor(prob,mean,totalCount))</c>.
/// </para></remarks>
public static double LogAverageFactor(Vector prob, Vector mean, double totalCount)
{
var temp = mean.Clone();
mean.SetToProduct(mean, totalCount);
var d = new Dirichlet(temp);
return d.GetLogProb(prob);
}
public void SetPriors(Dirichlet ProbInitPriorParamObs, Dirichlet[] CPTTransPriorObs)
{
ProbInitPrior.ObservedValue = ProbInitPriorParamObs;
CPTTransPrior.ObservedValue = CPTTransPriorObs;
}
/// <summary>
/// EP message to 'prob'
/// </summary>
/// <param name="prob">Incoming message from 'prob'.</param>
/// <param name="mean">Constant value for 'mean'.</param>
/// <param name="totalCount">Constant value for 'totalCount'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'prob' conditioned on the given values.
/// </para></remarks>
public static Dirichlet ProbAverageConditional(Dirichlet prob, Vector mean, double totalCount, Dirichlet result)
{
result.PseudoCount.SetToProduct(mean, totalCount);
return result;
}
public void MixtureOfMultivariateGaussians()
{
// Define a range for the number of mixture components
Range k = new Range(2).Named("k");
// Mixture component means
VariableArray <Vector> means = Variable.Array <Vector>(k).Named("means");
means[k] = Variable.VectorGaussianFromMeanAndPrecision(Vector.Zero(2), PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// Mixture component precisions
VariableArray <PositiveDefiniteMatrix> precs = Variable.Array <PositiveDefiniteMatrix>(k).Named("precs");
precs[k] = Variable.WishartFromShapeAndScale(100.0, PositiveDefiniteMatrix.IdentityScaledBy(2, 0.01)).ForEach(k);
// Mixture weights
Variable <Vector> weights = Variable.Dirichlet(k, new double[] { 1, 1 }).Named("weights");
// Create a variable array which will hold the data
Range n = new Range(300).Named("n");
VariableArray <Vector> data = Variable.Array <Vector>(n).Named("x");
// Create latent indicator variable for each data point
VariableArray <int> z = Variable.Array <int>(n).Named("z");
// The mixture of Gaussians model
using (Variable.ForEach(n))
{
z[n] = Variable.Discrete(weights);
using (Variable.Switch(z[n]))
{
data[n] = Variable.VectorGaussianFromMeanAndPrecision(means[z[n]], precs[z[n]]);
}
}
// Attach some generated data
double truePi = 0.6;
data.ObservedValue = GenerateData(n.SizeAsInt, truePi);
// Initialise messages randomly so as to break symmetry
Discrete[] zinit = new Discrete[n.SizeAsInt];
for (int i = 0; i < zinit.Length; i++)
{
zinit[i] = Discrete.PointMass(Rand.Int(k.SizeAsInt), k.SizeAsInt);
}
z.InitialiseTo(Distribution <int> .Array(zinit));
// The inference
InferenceEngine ie = new InferenceEngine();
ie.Algorithm = new VariationalMessagePassing();
//ie.Compiler.GenerateInMemory = false;
//ie.NumberOfIterations = 200;
Dirichlet wDist = (Dirichlet)ie.Infer(weights);
Vector wEstMean = wDist.GetMean();
object meansActual = ie.Infer(means);
Console.WriteLine("means = ");
Console.WriteLine(meansActual);
var precsActual = ie.Infer <IList <Wishart> >(precs);
Console.WriteLine("precs = ");
Console.WriteLine(precsActual);
Console.WriteLine("w = {0} should be {1}", wEstMean, Vector.FromArray(truePi, 1 - truePi));
//Console.WriteLine(StringUtil.JoinColumns("z = ", ie.Infer(z)));
Assert.True(
MMath.AbsDiff(wEstMean[0], truePi) < 0.05 ||
MMath.AbsDiff(wEstMean[1], truePi) < 0.05);
}
public static double LogEvidenceRatio(Dirichlet sample, Vector pseudoCounts) { return 0.0; }
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="DiscreteFromDirichletOp"]/message_doc[@name="ProbsAverageConditional(Discrete, Dirichlet, Dirichlet)"]/*'/>
public static Dirichlet ProbsAverageConditional([SkipIfUniform] Discrete sample, [NoInit] Dirichlet probs, Dirichlet result)
{
if (probs.IsPointMass)
{
return(ProbsAverageConditional(sample, probs.Point, result));
}
if (sample.IsPointMass)
{
return(ProbsConditional(sample.Point, result));
}
// Z = sum_x q(x) int_p f(x,p)*q(p) = sum_x q(x) E[p[x]]
Vector sampleProbs = sample.GetProbs();
double Z = sampleProbs.Inner(probs.PseudoCount);
double invZ = 1.0 / Z;
// the posterior is a mixture of Dirichlets having the following form:
// sum_x q(x) (alpha(x)/sum_i alpha(i)) Dirichlet(p; alpha(x)+1, alpha(not x)+0)
// where the mixture weights are w(x) =propto q(x) alpha(x)
// w[i] = sample[i]*probs.PseudoCount[i]/Z
// The posterior mean of probs(x) = (w(x) + alpha(x))/(1 + sum_x alpha(x))
double invTotalCountPlus1 = 1.0 / (probs.TotalCount + 1);
Vector m = Vector.Zero(sample.Dimension, sample.Sparsity);
m.SetToFunction(sampleProbs, probs.PseudoCount, (x, y) => (x * invZ + 1.0) * y * invTotalCountPlus1);
if (!Dirichlet.AllowImproperSum)
{
// To get the correct mean, we need (probs.PseudoCount[i] + delta[i]) to be proportional to m[i].
// If we set delta[argmin] = 0, then we just solve the equation
// (probs.PseudoCount[i] + delta[i])/probs.PseudoCount[argmin] = m[i]/m[argmin]
// for delta[i].
int argmin = sampleProbs.IndexOfMinimum();
Debug.Assert(argmin != -1);
double newTotalCount = probs.PseudoCount[argmin] / m[argmin];
double argMinValue = sampleProbs[argmin];
result.PseudoCount.SetToFunction(m, probs.PseudoCount, (x, y) => 1.0 + (x * newTotalCount) - y);
result.PseudoCount.SetToFunction(result.PseudoCount, sampleProbs, (x, y) => (y == argMinValue) ? 1.0 : x);
result.TotalCount = result.PseudoCount.Sum(); // result.Dimension + newTotalCount - probs.TotalCount;
return(result);
}
else
{
// The posterior meanSquare of probs(x) = (2 w(x) + alpha(x))/(2 + sum_x alpha(x)) * (1 + alpha(x))/(1 + sum_x alpha(x))
double invTotalCountPlus2 = 1.0 / (2 + probs.TotalCount);
Vector m2 = Vector.Zero(sample.Dimension, sample.Sparsity);
m2.SetToFunction(sampleProbs, probs.PseudoCount, (x, y) => (2.0 * x * invZ + 1.0) * y * invTotalCountPlus2 * (1.0 + y) * invTotalCountPlus1);
result.SetMeanAndMeanSquare(m, m2);
result.SetToRatio(result, probs);
return(result);
}
}
public static Dirichlet ProbAverageLogarithm([SkipIfUniform] ConjugateDirichlet alpha, Dirichlet result)
{
if (!alpha.IsPointMass)
{
double mean, variance;
alpha.SmartProposal(out mean, out variance);
double alphaMode = Math.Exp(mean);
if (double.IsNaN(alphaMode) || double.IsInfinity(alphaMode))
throw new ApplicationException("Nan message in ProbAverageLogarithm");
//result.PseudoCount.SetAllElementsTo(alphaMode);
result.PseudoCount.SetAllElementsTo(alpha.GetMean());
}
else
result.PseudoCount.SetAllElementsTo(alpha.Point);
result.TotalCount = result.PseudoCount.Sum();
return result;
}
/// <summary>
/// Run a single test for a single model
/// </summary>
/// <param name="sizeVocab">Size of the vocabulary</param>
/// <param name="numTopics">Number of topics</param>
/// <param name="trainWordsInTrainDoc">Lists of words in training documents used for training</param>
/// <param name="testWordsInTrainDoc">Lists of words in training documents used for testing</param>
/// <param name="alpha">Background pseudo-counts for distributions over topics</param>
/// <param name="beta">Background pseudo-counts for distributions over words</param>
/// <param name="shared">If true, uses shared variable version of the model</param>
/// <param name="trueThetaTest">The true topic distributions for the documents in the test set</param>
/// <param name="wordsInTestDoc">Lists of words in test documents</param>
/// <param name="vocabulary">Vocabulary</param>
static void RunTest(
int sizeVocab,
int numTopics,
Dictionary<int, int>[] trainWordsInTrainDoc,
Dictionary<int, int>[] testWordsInTrainDoc,
double alpha,
double beta,
bool shared,
Dirichlet[] trueThetaTest,
Dictionary<int, int>[] wordsInTestDoc,
Dictionary<int, string> vocabulary = null
)
{
Stopwatch stopWatch = new Stopwatch();
// Square root of number of documents is the optimal for memory
int batchCount = (int)Math.Sqrt((double)trainWordsInTrainDoc.Length);
Rand.Restart(5);
ILDA model;
LDAPredictionModel predictionModel;
LDATopicInferenceModel topicInfModel;
if (shared)
{
model = new LDAShared(batchCount, sizeVocab, numTopics);
((LDAShared)model).IterationsPerPass = Enumerable.Repeat(10, 5).ToArray();
}
else
{
model = new LDAModel(sizeVocab, numTopics);
model.Engine.NumberOfIterations = 50;
}
Console.WriteLine("\n\n************************************");
Console.WriteLine(
String.Format("\nTraining {0}LDA model...\n",
shared ? "batched " : "non-batched "));
// Train the model - we will also get rough estimates of execution time and memory
Dirichlet[] postTheta, postPhi;
GC.Collect();
PerformanceCounter memCounter = new PerformanceCounter("Memory", "Available MBytes");
float preMem = memCounter.NextValue();
stopWatch.Reset();
stopWatch.Start();
double logEvidence = model.Infer(trainWordsInTrainDoc, alpha, beta, out postTheta, out postPhi);
stopWatch.Stop();
float postMem = memCounter.NextValue();
double approxMB = preMem - postMem;
GC.KeepAlive(model); // Keep the model alive to this point (for the memory counter)
Console.WriteLine(String.Format("Approximate memory usage: {0:F2} MB", approxMB));
Console.WriteLine(String.Format("Approximate execution time (including model compilation): {0} seconds", stopWatch.ElapsedMilliseconds/1000));
// Calculate average log evidence over total training words
int totalWords = trainWordsInTrainDoc.Sum(doc => doc.Sum(w => w.Value));
Console.WriteLine("\nTotal number of training words = {0}", totalWords);
Console.WriteLine(String.Format("Average log evidence of model: {0:F2}", logEvidence / (double)totalWords));
if (vocabulary != null)
{
int numWordsToPrint = 20;
// Print out the top n words for each topic
for (int i = 0; i < postPhi.Length; i++)
{
double[] pc = postPhi[i].PseudoCount.ToArray();
int[] wordIndices = new int[pc.Length];
for (int j=0; j < wordIndices.Length; j++)
wordIndices[j] = j;
Array.Sort(pc, wordIndices);
Console.WriteLine("Top {0} words in topic {1}:", numWordsToPrint, i);
int idx = wordIndices.Length;
for (int j = 0; j < numWordsToPrint; j++)
Console.Write("\t{0}", vocabulary[wordIndices[--idx]]);
Console.WriteLine();
}
}
if (testWordsInTrainDoc != null)
{
// Test on unseen words in training documents
Console.WriteLine("\n\nCalculating perplexity on test words in training documents...");
predictionModel = new LDAPredictionModel(sizeVocab, numTopics);
predictionModel.Engine.NumberOfIterations = 5;
var predDist = predictionModel.Predict(postTheta, postPhi);
var perplexity = Utilities.Perplexity(predDist, testWordsInTrainDoc);
Console.WriteLine(String.Format("\nPerplexity = {0:F3}", perplexity));
}
if (wordsInTestDoc != null)
{
// Test on unseen documents. Note that topic ids for the trained model will be a random
// permutation of the topic ids for the ground truth
Console.WriteLine("\n\nInferring topics for test documents...");
topicInfModel = new LDATopicInferenceModel(sizeVocab, numTopics);
topicInfModel.Engine.NumberOfIterations = 10;
var inferredTopicDists = topicInfModel.InferTopic(alpha, postPhi, wordsInTestDoc);
Dictionary<TopicPair,int> topicPairCounts = new Dictionary<TopicPair, int>();
for (int i = 0; i < inferredTopicDists.Length; i++)
{
int infTopic = inferredTopicDists[i].PseudoCount.IndexOfMaximum();
int trueTopic = trueThetaTest[i].PseudoCount.IndexOfMaximum();
TopicPair tp = new TopicPair() { InferredTopic = infTopic, TrueTopic = trueTopic };
if (!topicPairCounts.ContainsKey(tp)) topicPairCounts.Add(tp, 1);
else topicPairCounts[tp] = topicPairCounts[tp]+1;
}
var correctCount = CountCorrectTopicPredictions(topicPairCounts, numTopics);
Console.WriteLine(String.Format("Maximum inferred topic matches maximum true topic {0} times out of {1}",
correctCount, inferredTopicDists.Length));
Console.WriteLine("\nThis uses a greedy algorithm to determine the mapping from inferred topic indices to true topic indices");
Console.WriteLine("\n************************************");
}
}
/// <summary>
/// Evidence message for EP
/// </summary>
/// <param name="prob">Incoming message from 'prob'.</param>
/// <param name="alpha">Constant value for 'alpha'.</param>
/// <returns>Logarithm of the factor's average value across the given argument distributions</returns>
/// <remarks><para>
/// The formula for the result is <c>log int_prob p(prob) factor(prob,alpha,K))</c>.
/// </para></remarks>
public static double LogAverageFactor(Dirichlet prob, double alpha)
{
int dim = prob.Dimension;
double sum = MMath.GammaLn(dim*alpha) - dim*MMath.GammaLn(alpha) - prob.GetLogNormalizer() - MMath.GammaLn(prob.TotalCount+dim*alpha);
for (int i = 0; i < dim; i++) {
sum += MMath.GammaLn(prob.PseudoCount[i]+alpha);
}
return sum;
}
public void DiscreteFromDirichletOpMomentMatchTest()
{
using (TestUtils.TemporarilyAllowDirichletImproperSums)
{
Dirichlet probsDist = new Dirichlet(1, 2, 3, 4);
Discrete sampleDist = Discrete.Uniform(4);
Assert.True(Dirichlet.Uniform(4).MaxDiff(DiscreteFromDirichletOp.ProbsAverageConditional(sampleDist, probsDist, Dirichlet.Uniform(4))) < 1e-4);
sampleDist = Discrete.PointMass(1, 4);
Assert.True(new Dirichlet(1, 2, 1, 1).MaxDiff(DiscreteFromDirichletOp.ProbsAverageConditional(sampleDist, probsDist, Dirichlet.Uniform(4))) < 1e-4);
sampleDist = new Discrete(0, 1, 1, 0);
Assert.True(new Dirichlet(0.9364, 1.247, 1.371, 0.7456).MaxDiff(DiscreteFromDirichletOp.ProbsAverageConditional(sampleDist, probsDist, Dirichlet.Uniform(4))) <
1e-3);
}
}
/// <summary>
/// EP message to 'prob'
/// </summary>
/// <param name="alpha">Constant value for 'alpha'.</param>
/// <param name="result">Modified to contain the outgoing message</param>
/// <returns><paramref name="result"/></returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'prob' conditioned on the given values.
/// </para></remarks>
public static Dirichlet ProbAverageConditional([SkipIfUniform] double alpha, Dirichlet result)
{
result.PseudoCount.SetAllElementsTo(alpha);
result.TotalCount = result.PseudoCount.Sum();
return result;
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="DiscreteFromDirichletOp"]/message_doc[@name="ProbsAverageConditional(Discrete, Vector, Dirichlet)"]/*'/>
public static Dirichlet ProbsAverageConditional([SkipIfUniform] Discrete sample, Vector probs, Dirichlet result)
{
Vector sampleProbs = sample.GetProbs();
double Z = sampleProbs.Inner(probs);
Vector dLogP = Vector.Zero(probs.Count);
Vector ddLogP = Vector.Zero(probs.Count);
double sampleN = sampleProbs[probs.Count - 1];
double probN = probs[probs.Count - 1];
dLogP.SetToFunction(sampleProbs, sampleProb => (sampleProb - sampleN) / Z);
ddLogP.SetToFunction(dLogP, x => - x * x);
result.SetDerivatives(probs, dLogP, ddLogP, !Dirichlet.AllowImproperSum);
return(result);
}
/// <summary>
/// VMP message to 'mean'
/// </summary>
/// <param name="mean">Incoming message from 'mean'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="totalCount">Constant value for 'totalCount'.</param>
/// <param name="prob">Constant value for 'prob'.</param>
/// <param name="to_mean">Previous outgoing message to 'mean'.</param>
/// <returns>The outgoing VMP message to the 'mean' argument</returns>
/// <remarks><para>
/// The outgoing message is the factor viewed as a function of 'mean' conditioned on the given values.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="mean"/> is not a proper distribution</exception>
public static Dirichlet MeanAverageLogarithm([Proper] Dirichlet mean, double totalCount, Vector prob, Dirichlet to_mean)
{
return MeanAverageLogarithm(mean, Gamma.PointMass(totalCount), Dirichlet.PointMass(prob), to_mean);
}
public void ValidateMean()
{
var d = new Dirichlet(0.3, 5);
for (var i = 0; i < 5; i++)
{
AssertHelpers.AlmostEqual(0.3 / 1.5, d.Mean[i], 15);
}
}
/// <summary>
/// VMP message to 'mean'
/// </summary>
/// <param name="mean">Incoming message from 'mean'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="totalCount">Incoming message from 'totalCount'. Must be a proper distribution. If uniform, the result will be uniform.</param>
/// <param name="prob">Incoming message from 'prob'. Must be a proper distribution. If any element is uniform, the result will be uniform.</param>
/// <param name="to_mean">Previous outgoing message to 'mean'.</param>
/// <returns>The outgoing VMP message to the 'mean' argument</returns>
/// <remarks><para>
/// The outgoing message is the exponential of the average log-factor value, where the average is over all arguments except 'mean'.
/// The formula is <c>exp(sum_(totalCount,prob) p(totalCount,prob) log(factor(prob,mean,totalCount)))</c>.
/// </para></remarks>
/// <exception cref="ImproperMessageException"><paramref name="mean"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="totalCount"/> is not a proper distribution</exception>
/// <exception cref="ImproperMessageException"><paramref name="prob"/> is not a proper distribution</exception>
public static Dirichlet MeanAverageLogarithm([Proper] Dirichlet mean, [Proper] Gamma totalCount, [SkipIfUniform] Dirichlet prob, Dirichlet to_mean)
{
Vector gradS = CalculateGradientForMean(mean.PseudoCount, totalCount, prob.GetMeanLog());
// Project onto Dirichlet, efficient matrix inversion (see TM's Dirichlet fitting paper)
int K = mean.Dimension;
Vector q = Vector.Zero(K);
double gOverQ = 0, OneOverQ = 0;
for (int k = 0; k < K; k++) {
q[k] = MMath.Trigamma(mean.PseudoCount[k]);
gOverQ += gradS[k] / q[k];
OneOverQ += 1 / q[k];
}
double z = -MMath.Trigamma(mean.TotalCount);
double b = gOverQ / (1 / z + OneOverQ);
// Create new approximation and damp
if (damping == 0.0) {
to_mean.PseudoCount.SetToFunction(gradS, q, (x, y) => ((x - b) / y) + 1.0);
return to_mean;
} else {
var old_msg = (Dirichlet)to_mean.Clone();
to_mean.PseudoCount.SetToFunction(gradS, q, (x, y) => ((x - b) / y) + 1.0);
return (to_mean ^ (1 - damping)) * (old_msg ^ damping);
}
}
public void ValidateVariance()
{
var alpha = new double[10];
var sum = 0.0;
for (var i = 0; i < 10; i++)
{
alpha[i] = i;
sum += i;
}
var d = new Dirichlet(alpha);
for (var i = 0; i < 10; i++)
{
AssertHelpers.AlmostEqual(i * (sum - i) / (sum * sum * (sum + 1.0)), d.Variance[i], 15);
}
}
/// <summary>
/// Returns the confusion matrix prior of each worker.
/// </summary>
/// <returns>The confusion matrix prior of each worker.</returns>
public Dirichlet[] GetConfusionMatrixPrior()
{
var confusionMatrixPrior = new Dirichlet[LabelCount];
for (int d = 0; d < LabelCount; d++)
{
//confusionMatrixPrior[d] = new Dirichlet(Util.ArrayInit(LabelCount, i => i == d ? (InitialWorkerBelief / (1 - InitialWorkerBelief)) * (LabelCount - 1) : 1.0));
confusionMatrixPrior[d] = new Dirichlet(Util.ArrayInit(LabelCount, i => i == d ? 0.8 : 0.2));
}
return confusionMatrixPrior;
}
/// <include file='FactorDocs.xml' path='factor_docs/message_op_class[@name="DiscreteFromDirichletOp"]/message_doc[@name="AverageLogFactor(Discrete, Vector)"]/*'/>
public static double AverageLogFactor(Discrete sample, Vector probs)
{
return(AverageLogFactor(sample, Dirichlet.PointMass(probs)));
}
/// <summary>
/// Main test program for Infer.NET LDA models
/// </summary>
/// <param name="args"></param>
static void Main(string[] args)
{
Rand.Restart(5);
Dictionary<int, string> vocabulary = null;
#if blei_corpus
Dictionary<int, int>[] trainWordsInTrainDoc = Utilities.LoadWordCounts("C:\\Users\\PrairieRose\\Documents\\GitHub\\HabilisX\\Infer.NET 2.5\\Samples\\C#\\LDA\\TestLDA\\890Histobram.txt");
vocabulary = Utilities.LoadVocabulary(@"C:\\Users\\PrairieRose\\Documents\\GitHub\\HabilisX\\Infer.NET 2.5\\Samples\\C#\\LDA\\TestLDA\\890Vocab.txt");
Dictionary<int, int>[] testWordsInTrainDoc = trainWordsInTrainDoc;
Dictionary<int, int>[] wordsInTestDoc = null;
int sizeVocab = Utilities.GetVocabularySize(trainWordsInTrainDoc);
int numTopics = 4;
int numTrainDocs = trainWordsInTrainDoc.Length;
Dirichlet[] trueThetaTest = null;
Console.WriteLine("************************************");
Console.WriteLine("Vocabulary size = " + sizeVocab);
Console.WriteLine("Number of documents = " + numTrainDocs);
Console.WriteLine("Number of topics = " + numTopics);
Console.WriteLine("************************************");
double alpha = 150 / numTopics;
double beta = 0.1;
#else
int numTopics = 4;
int sizeVocab = 1000;
int numTrainDocs = 500;
int averageDocumentLength = 100;
int averageWordsPerTopic = 10;
int numTestDocs = numTopics * 5;
int numDocs = numTrainDocs + numTestDocs;
// Create the true model
Dirichlet[] trueTheta, truePhi;
Utilities.CreateTrueThetaAndPhi(
sizeVocab, numTopics, numDocs, averageDocumentLength, averageWordsPerTopic,
out trueTheta, out truePhi);
// Split the documents between a train and test set
Dirichlet[] trueThetaTrain = new Dirichlet[numTrainDocs];
Dirichlet[] trueThetaTest = new Dirichlet[numTestDocs];
int docx = 0;
for (int i = 0; i < numTrainDocs; i++) trueThetaTrain[i] = trueTheta[docx++];
for (int i = 0; i < numTestDocs; i++) trueThetaTest[i] = trueTheta[docx++];
// Generate training and test data for the training documents
Dictionary<int, int>[] trainWordsInTrainDoc = Utilities.GenerateLDAData(trueThetaTrain, truePhi, (int)(0.9 * averageDocumentLength));
Dictionary<int, int>[] testWordsInTrainDoc = Utilities.GenerateLDAData(trueThetaTrain, truePhi, (int)(0.1 * averageDocumentLength));
Dictionary<int, int>[] wordsInTestDoc = Utilities.GenerateLDAData(trueThetaTest, truePhi, averageDocumentLength);
Console.WriteLine("************************************");
Console.WriteLine("Vocabulary size = " + sizeVocab);
Console.WriteLine("Number of topics = " + numTopics);
Console.WriteLine("True average words per topic = " + averageWordsPerTopic);
Console.WriteLine("Number of training documents = " + numTrainDocs);
Console.WriteLine("Number of test documents = " + numTestDocs);
Console.WriteLine("************************************");
double alpha = 1.0;
double beta = 0.1;
#endif
for (int i = 0; i < 2; i++)
{
bool shared = i == 0;
// if (!shared) continue; // Comment out this line to see full LDA models
RunTest(
sizeVocab,
numTopics,
trainWordsInTrainDoc,
testWordsInTrainDoc,
alpha,
beta,
shared,
trueThetaTest,
wordsInTestDoc,
vocabulary);
}
Console.ReadLine();
}
// Generate toy data - returns indices into vocab for each doc
private int[][] GenerateToyLDAData(
int numTopics, int numVocab, int numDocs, int expectedDocLength,
out Dirichlet[] trueTheta, out Dirichlet[] truePhi)
{
truePhi = new Dirichlet[numTopics];
for (int i = 0; i < numTopics; i++)
{
truePhi[i] = Dirichlet.Uniform(numVocab);
truePhi[i].PseudoCount.SetAllElementsTo(0.0);
// Draw the number of unique words in the topic
int numUniqueWordsPerTopic = Poisson.Sample((double)numVocab / numTopics);
if (numUniqueWordsPerTopic >= numVocab)
{
numUniqueWordsPerTopic = numVocab;
}
double expectedRepeatOfWordInTopic =
((double)numDocs) * expectedDocLength / numUniqueWordsPerTopic;
int[] shuffledWordIndices = Rand.Perm(numVocab);
for (int j = 0; j < numUniqueWordsPerTopic; j++)
{
int wordIndex = shuffledWordIndices[j];
// Draw the count for that word
int cnt = Poisson.Sample(expectedRepeatOfWordInTopic);
truePhi[i].PseudoCount[wordIndex] = cnt + 1.0;
}
}
trueTheta = new Dirichlet[numDocs];
for (int i = 0; i < numDocs; i++)
{
trueTheta[i] = Dirichlet.Uniform(numTopics);
trueTheta[i].PseudoCount.SetAllElementsTo(0.0);
// Draw the number of unique topics in the doc. We expect this to be
// very sparse
int numUniqueTopicsPerDoc = System.Math.Min(1 + Poisson.Sample(1.0), numTopics);
double expectedRepeatOfTopicInDoc =
expectedDocLength / numUniqueTopicsPerDoc;
int[] shuffledTopicIndices = Rand.Perm(numTopics);
for (int j = 0; j < numUniqueTopicsPerDoc; j++)
{
int topicIndex = shuffledTopicIndices[j];
// Draw the count for that topic
int cnt = Poisson.Sample(expectedRepeatOfTopicInDoc);
trueTheta[i].PseudoCount[topicIndex] = cnt + 1.0;
}
}
// Sample from the model
Vector[] topicDist = new Vector[numDocs];
Vector[] wordDist = new Vector[numTopics];
for (int i = 0; i < numDocs; i++)
{
topicDist[i] = trueTheta[i].Sample();
}
for (int i = 0; i < numTopics; i++)
{
wordDist[i] = truePhi[i].Sample();
}
int[][] wordsInDoc = new int[numDocs][];
for (int i = 0; i < numDocs; i++)
{
int LengthOfDoc = Poisson.Sample((double)expectedDocLength);
wordsInDoc[i] = new int[LengthOfDoc];
for (int j = 0; j < LengthOfDoc; j++)
{
int topic = Discrete.Sample(topicDist[i]);
wordsInDoc[i][j] = Discrete.Sample(wordDist[topic]);
}
}
return(wordsInDoc);
}
