private DistributionArray[,] GetDigramToCharTransitionMatrix()
{
var result = new DistributionArray[alphabet.Length, alphabet.Length];
for (var i = 0; i < alphabet.Length; i++)
{
for (var j = 0; j < alphabet.Length; j++)
{
result[i, j] = new DistributionArray
{
Distribution = new int[alphabet.Length],
Sum = 0
}
}
}
;
var trigrams = words.SelectMany(WordFeatureRow.GetTrigrams).ToArray();
foreach (var trigram in trigrams)
{
var i1 = alphabetIndexes[trigram[0]];
var i2 = alphabetIndexes[trigram[1]];
var i3 = alphabetIndexes[trigram[2]];
var distribution = result[i1, i2];
distribution.Distribution[i3]++;
distribution.Sum++;
}
return(result);
}
// result = prod_i values[i]^cases[i] (messages out of a gate are blurred)
public static T ExitAverageLogarithm <T>(DistributionArray <Bernoulli> cases, [SkipIfUniform] DistributionArray <T> values, T result)
where T : Diffable, SettableTo <T>, ICloneable, SettableToUniform, SettableToProduct <T>,
SettableToPower <T>, SettableToRatio <T>, SettableToWeightedSum <T>, LogInnerProductable <T>, CanGetAverageLog <T>
{
if (cases.Count != values.Count)
{
throw new ArgumentException("cases.Count != values.Count");
}
if (cases.Count == 0)
{
throw new ArgumentException("cases.Count == 0");
}
else
{
result.SetToPower(values[0], cases[0].LogOdds);
if (cases.Count > 1)
{
// TODO: use pre-allocated buffer
T power = (T)result.Clone();
for (int i = 1; i < cases.Count; i++)
{
power.SetToPower(values[i], cases[i].LogOdds);
result.SetToProduct(result, power);
}
}
}
return(result);
}
private DistributionArray[] GetCharToCharTransitionMatrix()
{
var result = new DistributionArray[alphabet.Length];
for (var i = 0; i < alphabet.Length; i++)
{
result[i] = new DistributionArray
{
Distribution = new int[alphabet.Length],
Sum = 0
}
}
;
var digrams = words.SelectMany(WordFeatureRow.GetDigrams).ToArray();
foreach (var digram in digrams)
{
var i1 = alphabetIndexes[digram[0]];
var i2 = alphabetIndexes[digram[1]];
var distribution = result[i1];
distribution.Distribution[i2]++;
distribution.Sum++;
}
return(result);
}
public void MP_GetItemSparseGP()
{
InferenceEngine engine = new InferenceEngine();
ModelDefinitionMethod meth = new ModelDefinitionMethod(GetItemSparseGPModel);
var ca = engine.Compiler.CompileWithoutParams(
declaringType, meth.Method, new AttributeRegistry <object, ICompilerAttribute>(true));
ca.Execute(1);
SparseGP cMarg = ca.Marginal <SparseGP>("c");
DistributionArray <SparseGP> dMarg = ca.Marginal <DistributionArray <SparseGP> >("d");
}
internal void ProbabilisticIndexMapNoGate()
{
//TODO: change path for cross platform using
double[,] pixData = MatlabReader.ReadMatrix(new double[10, 6400], @"c:\temp\pim\chand2.txt", ' ');
Range images = new Range(pixData.GetLength(0));
Range pixels = new Range(pixData.GetLength(1));
VariableArray2D <double> pixelData = Variable.Constant(pixData, images, pixels);
//pixelData.QuoteInMSL = false;
// For each image we have a palette of L multivariate Gaussians
VariableArray <double> means = Variable.Array <double>(images).Named("means");
means[images] = Variable.GaussianFromMeanAndPrecision(0.5, 1).ForEach(images);
VariableArray <double> precs = Variable.Array <double>(images).Named("precs");
precs[images] = Variable.GammaFromShapeAndScale(1.0, 1.0).ForEach(images);
// Across all pixels we have a
VariableArray <Vector> pi = Variable.Array <Vector>(pixels).Named("pi");
Dirichlet[] dinit = new Dirichlet[pixels.SizeAsInt];
for (int i = 0; i < dinit.Length; i++)
{
double d = Rand.Double();
dinit[i] = new Dirichlet(1.0 + d / 10, 1.0 - d / 10);
}
pi[pixels] = Variable.Dirichlet(new double[] { 1.0 }).ForEach(pixels);
// For each pixel of each image we have a discrete indicator
VariableArray2D <int> ind = Variable.Array <int>(images, pixels).Named("ind");
ind[images, pixels] = Variable.Discrete(pi[pixels]).ForEach(images);
using (Variable.ForEach(pixels))
{
using (Variable.ForEach(images))
{
pixelData[images, pixels] = Variable.GaussianFromMeanAndPrecision(means[images], //10);
precs[images]);
Variable.ConstrainEqualRandom(ind[images, pixels], Discrete.Uniform(1));
}
}
InferenceEngine ie = new InferenceEngine(new VariationalMessagePassing());
ie.ModelName = "PIM_NoGate";
ie.NumberOfIterations = 8;
ie.ShowTimings = true;
DistributionArray <Dirichlet> piDist = ie.Infer <DistributionArray <Dirichlet> >(pi);
//Console.WriteLine("Dist over pi: " + ie.Infer(pi));
//TODO: change path for cross platform using
WriteMatrix(piDist.ToArray(), @"C:\temp\pim\results.txt");
}
private DistributionArray GetFirstCharProbabilities()
{
var result = new DistributionArray {
Distribution = new int[alphabet.Length], Sum = words.Length
};
foreach (var word in words)
{
var index = alphabetIndexes[word[0]];
result.Distribution[index]++;
}
return(result);
}
public void SimpleGaussianWithTwoLoopsMsl()
{
InferenceEngine engine = new InferenceEngine(new VariationalMessagePassing());
engine.Compiler.DeclarationProvider = RoslynDeclarationProvider.Instance;
double[] data = { 5, 7 }; //, 9, 11, 17, 41, 32 };
Gaussian meanExpected = Gaussian.FromMeanAndVariance(5.9603207170807826, 0.66132138200164436);
Gamma precisionExpected = Gamma.FromShapeAndRate(2, 2.6628958274937107);
var ca = engine.Compiler.Compile(SimpleGaussianWithTwoLoopsModel, data);
ca.Execute(20);
Console.WriteLine("Marginal for mean = " + ca.Marginal("mean"));
Console.WriteLine("Marginal for precision = " + ca.Marginal("precision"));
DistributionArray <Gaussian> meanActual = ca.Marginal <DistributionArray <Gaussian> >("mean");
DistributionArray <Gamma> precisionActual = ca.Marginal <DistributionArray <Gamma> >("precision");
for (int j = 0; j < meanActual.Count; j++)
{
Assert.True(meanExpected.MaxDiff(meanActual[j]) < 1e-4);
Assert.True(precisionExpected.MaxDiff(precisionActual[j]) < 1e-4);
}
}
public static DistributionArray <Bernoulli> CasesAverageLogarithm(DistributionArray <Bernoulli> result)
{
return(result);
}
static void Run()
{
//likelihood
double dataVar = 1; //variance of the "data component" of the mixture
double clutterMean = 0; //mean of clutter component
double clutterVar = 10; //variance of clutter component
double w = 0.5; //probability of choosing the data component
//prior
double priorVar = 100; //variance of prior over data component mean
double priorMean = 0; //mean of prior
//true mean of data component for generating training data
double trueTheta = 2;
//number of observed data points
int nData = 10;
/**********************************************************************************
*
* construct the model -- we're not generating any data here, just defining the
* model
*
**********************************************************************************/
//mixture weights
double[] mixturePrior = new double[] { w, 1 - w };
//prior over mean
Variable <double> theta = Variable.GaussianFromMeanAndVariance(priorMean, priorVar);
//model will have n observed data points, obseved data is continuous scalar
Range n = new Range(nData);
VariableArray <double> data = Variable.Array <double>(n);
//define how each of these observed data points are being generated (this is the actual model):
VariableArray <int> z = Variable.Array <int>(n); //mixture component indicator for each data point (will be unobserved)
using (Variable.ForEach(n)) //for each datapoint
{
z[n] = Variable.Discrete(mixturePrior); //choose mixture component by drawing z from discrete distribution with 2 values
using (Variable.Case(z[n], 0)) //first mixture component
{
//draw data from data component: Gaussian with mean theta (and variance dataVar)
data[n] = Variable.GaussianFromMeanAndVariance(theta, dataVar);
}
using (Variable.Case(z[n], 1)) //second mixture component
{
//draw data from clutter component
data[n] = Variable.GaussianFromMeanAndVariance(clutterMean, clutterVar);
}
}
/****************************************************************************
*
* create data by sampling from the model and attach to the variables of the
* model defined above
*
* alternatively read data from matlab file
*
****************************************************************************/
double[] observedData = new double[nData];
int[] trueZ = new int[nData];
if (true) //set to false to read from matlab file
{
//sample observed data
Rand.Restart(104); //set seed of random number generator to fixed value
//set up distributions to sample from
Discrete kDistr = new Discrete(mixturePrior);
Gaussian clutterDistr = new Gaussian(clutterMean, clutterVar);
Gaussian dataDistr = new Gaussian(trueTheta, dataVar);
//generate data
for (int jj = 0; jj < nData; jj++)
{
trueZ[jj] = kDistr.Sample();
if (trueZ[jj] == 0)
{
observedData[jj] = dataDistr.Sample();
}
else
{
observedData[jj] = clutterDistr.Sample();
}
}
}
else
{
// read data from Matlab file that contains an (nx1) matrix variable named "observedData"
// (note that the size of the matrix must fit the specified number of observed data
// points from above
Dictionary <String, Object> fullDataFile = MatlabReader.Read("..\\..\\testData.mat");
((Matrix)fullDataFile["observedData"]).CopyTo(observedData, 0);
}
//attach the observed data to model variables defined above
data.ObservedValue = observedData;
/****************************************************************************
*
* run inference
*
****************************************************************************/
// determine whether we allow improper messages (negative precision)
// set to false to allow improper messages
GateEnterOp <double> .ForceProper = true;
// run inference with Expectation Propagation and show the posterior over theta
InferenceEngine ie = new InferenceEngine(new ExpectationPropagation());
Console.WriteLine("Posterior over theta" + ie.Infer(theta));
// also show the posterior over z for each data point
DistributionArray <Discrete> zInferred = (DistributionArray <Discrete>)ie.Infer(z);
Console.WriteLine("\nPosterior over z (true z; observed value):");
for (int jj = 0; jj < nData; jj++)
{
Console.WriteLine(string.Format("{0:0.00} ( {1} ; {2:0.00})", zInferred[jj].GetProbs()[0], trueZ[jj], observedData[jj]));
}
Console.WriteLine("\n");
}
public void BernoulliMixtureTest()
{
int N = 10, D = 2, K = 2;
Range n = new Range(N).Named("n");
Range k = new Range(K).Named("k");
Range d = new Range(D).Named("d");
VariableArray2D <double> p = Variable.Array <double>(k, d).Named("p");
p[k, d] = Variable.Beta(1, 1).ForEach(k, d);
VariableArray2D <bool> x = Variable.Array <bool>(n, d).Named("x");
VariableArray <int> c = Variable.Array <int>(n).Named("c");
using (Variable.ForEach(n))
{
c[n] = Variable.Discrete(k, 0.5, 0.5);
using (Variable.Switch(c[n]))
{
x[n, d] = Variable.Bernoulli(p[c[n], d]);
}
}
InferenceEngine engine = new InferenceEngine();
bool[,] data = new bool[N, D];
int N1 = N / 2;
int i = 0;
for (; i < N1; i++)
{
data[i, 0] = true;
data[i, 1] = false;
}
for (; i < N; i++)
{
data[i, 0] = false;
data[i, 1] = true;
}
x.ObservedValue = data;
Discrete[] cInit = new Discrete[N];
for (int j = 0; j < N; j++)
{
double r = Rand.Double();
cInit[j] = new Discrete(r, 1 - r);
}
c.InitialiseTo(Distribution <int> .Array(cInit));
engine.NumberOfIterations = 1;
var pExpected = engine.Infer(p);
engine.NumberOfIterations = engine.Algorithm.DefaultNumberOfIterations;
DistributionArray <Discrete> cPost = engine.Infer <DistributionArray <Discrete> >(c);
Console.WriteLine(cPost);
DistributionArray2D <Beta> pPost = engine.Infer <DistributionArray2D <Beta> >(p);
Console.WriteLine(pPost);
// test resetting inference
engine.NumberOfIterations = 1;
var pActual = engine.Infer <Diffable>(p);
Assert.True(pActual.MaxDiff(pExpected) < 1e-10);
}
public void BernoulliMixtureGaussianTest()
{
int N = 10, D = 2, K = 2;
Range n = new Range(N).Named("n");
Range k = new Range(K).Named("k");
Range d = new Range(D).Named("d");
VariableArray2D <double> p = Variable.Array <double>(k, d).Named("p");
p[k, d] = Variable.GaussianFromMeanAndVariance(0, 1).ForEach(k, d);
VariableArray2D <bool> x = Variable.Array <bool>(n, d).Named("x");
VariableArray <int> c = Variable.Array <int>(n).Named("c");
using (Variable.ForEach(n))
{
c[n] = Variable.Discrete(k, 0.5, 0.5);
using (Variable.Switch(c[n]))
{
x[n, d] = (Variable.GaussianFromMeanAndVariance(p[c[n], d], 1.0) > 0);
}
}
bool geForceProper = GateEnterOp <double> .ForceProper;
try
{
GateEnterOp <double> .ForceProper = true;
InferenceEngine engine = new InferenceEngine(); //new VariationalMessagePassing());
engine.Compiler.GivePriorityTo(typeof(IsPositiveOp_Proper)); // needed to avoid improper messages in EP
bool[,] data = new bool[N, D];
int N1 = N / 2;
int i = 0;
for (; i < N1; i++)
{
data[i, 0] = true;
data[i, 1] = false;
}
for (; i < N; i++)
{
data[i, 0] = false;
data[i, 1] = true;
}
x.ObservedValue = data;
Discrete[] cInit = new Discrete[N];
for (int j = 0; j < N; j++)
{
double r = Rand.Double();
cInit[j] = new Discrete(r, 1 - r);
}
c.InitialiseTo(Distribution <int> .Array(cInit));
engine.NumberOfIterations = 1;
var pExpected = engine.Infer(p);
engine.NumberOfIterations = engine.Algorithm.DefaultNumberOfIterations;
DistributionArray <Discrete> cPost = engine.Infer <DistributionArray <Discrete> >(c);
Console.WriteLine(cPost);
DistributionArray2D <Gaussian> pPost = engine.Infer <DistributionArray2D <Gaussian> >(p);
Console.WriteLine(pPost);
// test resetting inference
engine.NumberOfIterations = 1;
var pActual = engine.Infer <Diffable>(p);
Assert.True(pActual.MaxDiff(pExpected) < 1e-10);
}
finally
{
GateEnterOp <double> .ForceProper = geForceProper;
}
}
internal void BetaRegression()
{
int P = 8;
double[] b = new double[P];
for (int p = 0; p < P; p++)
{
b[p] = Rand.Beta(1, 1);
}
int N = 100;
double[][] X = new double[N][];
//Gaussian[][] softX = new Gaussian[N][];
double[] Y = new double[N];
for (int n = 0; n < N; n++)
{
X[n] = new double[P];
//softX[n] = new Gaussian[P];
Y[n] = 0;
for (int p = 0; p < P; p++)
{
X[n][p] = Rand.Normal();
//softX[n][p] = new Gaussian(X[n][p], 1e-4);
Y[n] += X[n][p] * b[p];
}
}
Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence");
IfBlock block = Variable.If(evidence);
Range dim = new Range(P).Named("P");
Range item = new Range(N).Named("N");
VariableArray <double> w = Variable.Array <double>(dim).Named("w");
w[dim] = Variable.Beta(1, 1).ForEach(dim);
var x = Variable.Array(Variable.Array <double>(dim), item).Named("x");
var softXvar = Variable.Array(Variable.Array <double>(dim), item).Named("softx");
softXvar.ObservedValue = X;
x[item][dim] = Variable.GaussianFromMeanAndPrecision(softXvar[item][dim], 1e4);
var wx = Variable.Array(Variable.Array <double>(dim), item).Named("wx");
wx[item][dim] = x[item][dim] * w[dim];
var sum = Variable.Array <double>(item).Named("sum");
sum[item] = Variable.Sum(wx[item]);
var prec = Variable.GammaFromShapeAndRate(.1, .1).Named("Noise");
var y = Variable.Array <double>(item).Named("y");
y[item] = Variable.GaussianFromMeanAndPrecision(sum[item], prec);
block.CloseBlock();
y.ObservedValue = Y;
InferenceEngine engine = new InferenceEngine(new VariationalMessagePassing());
var ca = engine.GetCompiledInferenceAlgorithm(evidence, w);
ca.Reset();
double oldLogEvidence = double.NegativeInfinity;
for (int i = 0; i < 1000; i++)
{
ca.Update(1);
double logEvidence1 = ca.Marginal <Bernoulli>(evidence.NameInGeneratedCode).LogOdds;
Console.WriteLine(logEvidence1);
if (i > 20 && System.Math.Abs(logEvidence1 - oldLogEvidence) < 0.01)
{
break;
}
oldLogEvidence = logEvidence1;
}
DistributionArray <Beta> wInferred = ca.Marginal <DistributionArray <Beta> >(w.NameInGeneratedCode);
for (int p = 0; p < P; p++)
{
Console.WriteLine("w[{0}] = {1} +/- {2} should be {3}",
p, wInferred[p].GetMean(), System.Math.Sqrt(wInferred[p].GetVariance()), b[p]);
}
}
private State LoadOrCreateState()
{
var file = Path.Combine(dir, "simple_generator.txt");
if (!File.Exists(file))
{
var firstChar = GetFirstCharProbabilities();
var charToChar = GetCharToCharTransitionMatrix();
var startSeed = 1u;
firstChar.MakeDistributionBinary();
foreach (var distr in charToChar)
{
distr.MakeDistributionBinary();
}
File.WriteAllLines(file, new[]
{
string.Join(",", firstChar.Distribution),
Environment.NewLine,
string.Join(Environment.NewLine, charToChar.Select(d => string.Join(",", d.Distribution))),
Environment.NewLine,
});
Console.WriteLine($"Sums: {firstChar.Sum}; {string.Join(",", charToChar.Select(d => d.Sum))}");
Console.WriteLine("File created");
return(new State
{
FirstChar = firstChar,
CharToChar = charToChar,
Seed = startSeed,
CoolSeeds = new List <uint>()
});
}
else
{
DistributionArray firstChar = null;
DistributionArray[] charToChar = null;
var coolSeeds = new List <uint>();
var content = File.ReadAllLines(file);
var charToCharList = new List <DistributionArray>();
foreach (var line in content)
{
if (string.IsNullOrEmpty(line))
{
continue;
}
if (firstChar == null)
{
firstChar = new DistributionArray {
Distribution = line.Split(',').Select(int.Parse).ToArray()
};
firstChar.Sum = firstChar.Distribution.Sum();
}
else if (line.Contains(","))
{
var arr = new DistributionArray {
Distribution = line.Split(',').Select(int.Parse).ToArray()
};
arr.Sum = arr.Distribution.Sum();
charToCharList.Add(arr);
}
else
{
coolSeeds.Add(uint.Parse(line));
}
}
charToChar = charToCharList.ToArray();
var startSeed = coolSeeds.Max() + 1;
Console.WriteLine($"Sums: {firstChar.Sum}; {string.Join(",", charToChar.Select(d => d.Sum))}");
Console.WriteLine("Loaded seed = " + startSeed);
return(new State
{
FirstChar = firstChar,
CharToChar = charToChar,
Seed = startSeed,
CoolSeeds = coolSeeds
});
}
}
