private DistributionArray[,] GetDigramToCharTransitionMatrix() { var result = new DistributionArray[alphabet.Length, alphabet.Length]; for (var i = 0; i < alphabet.Length; i++) { for (var j = 0; j < alphabet.Length; j++) { result[i, j] = new DistributionArray { Distribution = new int[alphabet.Length], Sum = 0 } } } ; var trigrams = words.SelectMany(WordFeatureRow.GetTrigrams).ToArray(); foreach (var trigram in trigrams) { var i1 = alphabetIndexes[trigram[0]]; var i2 = alphabetIndexes[trigram[1]]; var i3 = alphabetIndexes[trigram[2]]; var distribution = result[i1, i2]; distribution.Distribution[i3]++; distribution.Sum++; } return(result); }
// result = prod_i values[i]^cases[i] (messages out of a gate are blurred) public static T ExitAverageLogarithm <T>(DistributionArray <Bernoulli> cases, [SkipIfUniform] DistributionArray <T> values, T result) where T : Diffable, SettableTo <T>, ICloneable, SettableToUniform, SettableToProduct <T>, SettableToPower <T>, SettableToRatio <T>, SettableToWeightedSum <T>, LogInnerProductable <T>, CanGetAverageLog <T> { if (cases.Count != values.Count) { throw new ArgumentException("cases.Count != values.Count"); } if (cases.Count == 0) { throw new ArgumentException("cases.Count == 0"); } else { result.SetToPower(values[0], cases[0].LogOdds); if (cases.Count > 1) { // TODO: use pre-allocated buffer T power = (T)result.Clone(); for (int i = 1; i < cases.Count; i++) { power.SetToPower(values[i], cases[i].LogOdds); result.SetToProduct(result, power); } } } return(result); }
private DistributionArray[] GetCharToCharTransitionMatrix() { var result = new DistributionArray[alphabet.Length]; for (var i = 0; i < alphabet.Length; i++) { result[i] = new DistributionArray { Distribution = new int[alphabet.Length], Sum = 0 } } ; var digrams = words.SelectMany(WordFeatureRow.GetDigrams).ToArray(); foreach (var digram in digrams) { var i1 = alphabetIndexes[digram[0]]; var i2 = alphabetIndexes[digram[1]]; var distribution = result[i1]; distribution.Distribution[i2]++; distribution.Sum++; } return(result); }
public void MP_GetItemSparseGP() { InferenceEngine engine = new InferenceEngine(); ModelDefinitionMethod meth = new ModelDefinitionMethod(GetItemSparseGPModel); var ca = engine.Compiler.CompileWithoutParams( declaringType, meth.Method, new AttributeRegistry <object, ICompilerAttribute>(true)); ca.Execute(1); SparseGP cMarg = ca.Marginal <SparseGP>("c"); DistributionArray <SparseGP> dMarg = ca.Marginal <DistributionArray <SparseGP> >("d"); }
internal void ProbabilisticIndexMapNoGate() { //TODO: change path for cross platform using double[,] pixData = MatlabReader.ReadMatrix(new double[10, 6400], @"c:\temp\pim\chand2.txt", ' '); Range images = new Range(pixData.GetLength(0)); Range pixels = new Range(pixData.GetLength(1)); VariableArray2D <double> pixelData = Variable.Constant(pixData, images, pixels); //pixelData.QuoteInMSL = false; // For each image we have a palette of L multivariate Gaussians VariableArray <double> means = Variable.Array <double>(images).Named("means"); means[images] = Variable.GaussianFromMeanAndPrecision(0.5, 1).ForEach(images); VariableArray <double> precs = Variable.Array <double>(images).Named("precs"); precs[images] = Variable.GammaFromShapeAndScale(1.0, 1.0).ForEach(images); // Across all pixels we have a VariableArray <Vector> pi = Variable.Array <Vector>(pixels).Named("pi"); Dirichlet[] dinit = new Dirichlet[pixels.SizeAsInt]; for (int i = 0; i < dinit.Length; i++) { double d = Rand.Double(); dinit[i] = new Dirichlet(1.0 + d / 10, 1.0 - d / 10); } pi[pixels] = Variable.Dirichlet(new double[] { 1.0 }).ForEach(pixels); // For each pixel of each image we have a discrete indicator VariableArray2D <int> ind = Variable.Array <int>(images, pixels).Named("ind"); ind[images, pixels] = Variable.Discrete(pi[pixels]).ForEach(images); using (Variable.ForEach(pixels)) { using (Variable.ForEach(images)) { pixelData[images, pixels] = Variable.GaussianFromMeanAndPrecision(means[images], //10); precs[images]); Variable.ConstrainEqualRandom(ind[images, pixels], Discrete.Uniform(1)); } } InferenceEngine ie = new InferenceEngine(new VariationalMessagePassing()); ie.ModelName = "PIM_NoGate"; ie.NumberOfIterations = 8; ie.ShowTimings = true; DistributionArray <Dirichlet> piDist = ie.Infer <DistributionArray <Dirichlet> >(pi); //Console.WriteLine("Dist over pi: " + ie.Infer(pi)); //TODO: change path for cross platform using WriteMatrix(piDist.ToArray(), @"C:\temp\pim\results.txt"); }
private DistributionArray GetFirstCharProbabilities() { var result = new DistributionArray { Distribution = new int[alphabet.Length], Sum = words.Length }; foreach (var word in words) { var index = alphabetIndexes[word[0]]; result.Distribution[index]++; } return(result); }
public void SimpleGaussianWithTwoLoopsMsl() { InferenceEngine engine = new InferenceEngine(new VariationalMessagePassing()); engine.Compiler.DeclarationProvider = RoslynDeclarationProvider.Instance; double[] data = { 5, 7 }; //, 9, 11, 17, 41, 32 }; Gaussian meanExpected = Gaussian.FromMeanAndVariance(5.9603207170807826, 0.66132138200164436); Gamma precisionExpected = Gamma.FromShapeAndRate(2, 2.6628958274937107); var ca = engine.Compiler.Compile(SimpleGaussianWithTwoLoopsModel, data); ca.Execute(20); Console.WriteLine("Marginal for mean = " + ca.Marginal("mean")); Console.WriteLine("Marginal for precision = " + ca.Marginal("precision")); DistributionArray <Gaussian> meanActual = ca.Marginal <DistributionArray <Gaussian> >("mean"); DistributionArray <Gamma> precisionActual = ca.Marginal <DistributionArray <Gamma> >("precision"); for (int j = 0; j < meanActual.Count; j++) { Assert.True(meanExpected.MaxDiff(meanActual[j]) < 1e-4); Assert.True(precisionExpected.MaxDiff(precisionActual[j]) < 1e-4); } }
public static DistributionArray <Bernoulli> CasesAverageLogarithm(DistributionArray <Bernoulli> result) { return(result); }
static void Run() { //likelihood double dataVar = 1; //variance of the "data component" of the mixture double clutterMean = 0; //mean of clutter component double clutterVar = 10; //variance of clutter component double w = 0.5; //probability of choosing the data component //prior double priorVar = 100; //variance of prior over data component mean double priorMean = 0; //mean of prior //true mean of data component for generating training data double trueTheta = 2; //number of observed data points int nData = 10; /********************************************************************************** * * construct the model -- we're not generating any data here, just defining the * model * **********************************************************************************/ //mixture weights double[] mixturePrior = new double[] { w, 1 - w }; //prior over mean Variable <double> theta = Variable.GaussianFromMeanAndVariance(priorMean, priorVar); //model will have n observed data points, obseved data is continuous scalar Range n = new Range(nData); VariableArray <double> data = Variable.Array <double>(n); //define how each of these observed data points are being generated (this is the actual model): VariableArray <int> z = Variable.Array <int>(n); //mixture component indicator for each data point (will be unobserved) using (Variable.ForEach(n)) //for each datapoint { z[n] = Variable.Discrete(mixturePrior); //choose mixture component by drawing z from discrete distribution with 2 values using (Variable.Case(z[n], 0)) //first mixture component { //draw data from data component: Gaussian with mean theta (and variance dataVar) data[n] = Variable.GaussianFromMeanAndVariance(theta, dataVar); } using (Variable.Case(z[n], 1)) //second mixture component { //draw data from clutter component data[n] = Variable.GaussianFromMeanAndVariance(clutterMean, clutterVar); } } /**************************************************************************** * * create data by sampling from the model and attach to the variables of the * model defined above * * alternatively read data from matlab file * ****************************************************************************/ double[] observedData = new double[nData]; int[] trueZ = new int[nData]; if (true) //set to false to read from matlab file { //sample observed data Rand.Restart(104); //set seed of random number generator to fixed value //set up distributions to sample from Discrete kDistr = new Discrete(mixturePrior); Gaussian clutterDistr = new Gaussian(clutterMean, clutterVar); Gaussian dataDistr = new Gaussian(trueTheta, dataVar); //generate data for (int jj = 0; jj < nData; jj++) { trueZ[jj] = kDistr.Sample(); if (trueZ[jj] == 0) { observedData[jj] = dataDistr.Sample(); } else { observedData[jj] = clutterDistr.Sample(); } } } else { // read data from Matlab file that contains an (nx1) matrix variable named "observedData" // (note that the size of the matrix must fit the specified number of observed data // points from above Dictionary <String, Object> fullDataFile = MatlabReader.Read("..\\..\\testData.mat"); ((Matrix)fullDataFile["observedData"]).CopyTo(observedData, 0); } //attach the observed data to model variables defined above data.ObservedValue = observedData; /**************************************************************************** * * run inference * ****************************************************************************/ // determine whether we allow improper messages (negative precision) // set to false to allow improper messages GateEnterOp <double> .ForceProper = true; // run inference with Expectation Propagation and show the posterior over theta InferenceEngine ie = new InferenceEngine(new ExpectationPropagation()); Console.WriteLine("Posterior over theta" + ie.Infer(theta)); // also show the posterior over z for each data point DistributionArray <Discrete> zInferred = (DistributionArray <Discrete>)ie.Infer(z); Console.WriteLine("\nPosterior over z (true z; observed value):"); for (int jj = 0; jj < nData; jj++) { Console.WriteLine(string.Format("{0:0.00} ( {1} ; {2:0.00})", zInferred[jj].GetProbs()[0], trueZ[jj], observedData[jj])); } Console.WriteLine("\n"); }
public void BernoulliMixtureTest() { int N = 10, D = 2, K = 2; Range n = new Range(N).Named("n"); Range k = new Range(K).Named("k"); Range d = new Range(D).Named("d"); VariableArray2D <double> p = Variable.Array <double>(k, d).Named("p"); p[k, d] = Variable.Beta(1, 1).ForEach(k, d); VariableArray2D <bool> x = Variable.Array <bool>(n, d).Named("x"); VariableArray <int> c = Variable.Array <int>(n).Named("c"); using (Variable.ForEach(n)) { c[n] = Variable.Discrete(k, 0.5, 0.5); using (Variable.Switch(c[n])) { x[n, d] = Variable.Bernoulli(p[c[n], d]); } } InferenceEngine engine = new InferenceEngine(); bool[,] data = new bool[N, D]; int N1 = N / 2; int i = 0; for (; i < N1; i++) { data[i, 0] = true; data[i, 1] = false; } for (; i < N; i++) { data[i, 0] = false; data[i, 1] = true; } x.ObservedValue = data; Discrete[] cInit = new Discrete[N]; for (int j = 0; j < N; j++) { double r = Rand.Double(); cInit[j] = new Discrete(r, 1 - r); } c.InitialiseTo(Distribution <int> .Array(cInit)); engine.NumberOfIterations = 1; var pExpected = engine.Infer(p); engine.NumberOfIterations = engine.Algorithm.DefaultNumberOfIterations; DistributionArray <Discrete> cPost = engine.Infer <DistributionArray <Discrete> >(c); Console.WriteLine(cPost); DistributionArray2D <Beta> pPost = engine.Infer <DistributionArray2D <Beta> >(p); Console.WriteLine(pPost); // test resetting inference engine.NumberOfIterations = 1; var pActual = engine.Infer <Diffable>(p); Assert.True(pActual.MaxDiff(pExpected) < 1e-10); }
public void BernoulliMixtureGaussianTest() { int N = 10, D = 2, K = 2; Range n = new Range(N).Named("n"); Range k = new Range(K).Named("k"); Range d = new Range(D).Named("d"); VariableArray2D <double> p = Variable.Array <double>(k, d).Named("p"); p[k, d] = Variable.GaussianFromMeanAndVariance(0, 1).ForEach(k, d); VariableArray2D <bool> x = Variable.Array <bool>(n, d).Named("x"); VariableArray <int> c = Variable.Array <int>(n).Named("c"); using (Variable.ForEach(n)) { c[n] = Variable.Discrete(k, 0.5, 0.5); using (Variable.Switch(c[n])) { x[n, d] = (Variable.GaussianFromMeanAndVariance(p[c[n], d], 1.0) > 0); } } bool geForceProper = GateEnterOp <double> .ForceProper; try { GateEnterOp <double> .ForceProper = true; InferenceEngine engine = new InferenceEngine(); //new VariationalMessagePassing()); engine.Compiler.GivePriorityTo(typeof(IsPositiveOp_Proper)); // needed to avoid improper messages in EP bool[,] data = new bool[N, D]; int N1 = N / 2; int i = 0; for (; i < N1; i++) { data[i, 0] = true; data[i, 1] = false; } for (; i < N; i++) { data[i, 0] = false; data[i, 1] = true; } x.ObservedValue = data; Discrete[] cInit = new Discrete[N]; for (int j = 0; j < N; j++) { double r = Rand.Double(); cInit[j] = new Discrete(r, 1 - r); } c.InitialiseTo(Distribution <int> .Array(cInit)); engine.NumberOfIterations = 1; var pExpected = engine.Infer(p); engine.NumberOfIterations = engine.Algorithm.DefaultNumberOfIterations; DistributionArray <Discrete> cPost = engine.Infer <DistributionArray <Discrete> >(c); Console.WriteLine(cPost); DistributionArray2D <Gaussian> pPost = engine.Infer <DistributionArray2D <Gaussian> >(p); Console.WriteLine(pPost); // test resetting inference engine.NumberOfIterations = 1; var pActual = engine.Infer <Diffable>(p); Assert.True(pActual.MaxDiff(pExpected) < 1e-10); } finally { GateEnterOp <double> .ForceProper = geForceProper; } }
internal void BetaRegression() { int P = 8; double[] b = new double[P]; for (int p = 0; p < P; p++) { b[p] = Rand.Beta(1, 1); } int N = 100; double[][] X = new double[N][]; //Gaussian[][] softX = new Gaussian[N][]; double[] Y = new double[N]; for (int n = 0; n < N; n++) { X[n] = new double[P]; //softX[n] = new Gaussian[P]; Y[n] = 0; for (int p = 0; p < P; p++) { X[n][p] = Rand.Normal(); //softX[n][p] = new Gaussian(X[n][p], 1e-4); Y[n] += X[n][p] * b[p]; } } Variable <bool> evidence = Variable.Bernoulli(0.5).Named("evidence"); IfBlock block = Variable.If(evidence); Range dim = new Range(P).Named("P"); Range item = new Range(N).Named("N"); VariableArray <double> w = Variable.Array <double>(dim).Named("w"); w[dim] = Variable.Beta(1, 1).ForEach(dim); var x = Variable.Array(Variable.Array <double>(dim), item).Named("x"); var softXvar = Variable.Array(Variable.Array <double>(dim), item).Named("softx"); softXvar.ObservedValue = X; x[item][dim] = Variable.GaussianFromMeanAndPrecision(softXvar[item][dim], 1e4); var wx = Variable.Array(Variable.Array <double>(dim), item).Named("wx"); wx[item][dim] = x[item][dim] * w[dim]; var sum = Variable.Array <double>(item).Named("sum"); sum[item] = Variable.Sum(wx[item]); var prec = Variable.GammaFromShapeAndRate(.1, .1).Named("Noise"); var y = Variable.Array <double>(item).Named("y"); y[item] = Variable.GaussianFromMeanAndPrecision(sum[item], prec); block.CloseBlock(); y.ObservedValue = Y; InferenceEngine engine = new InferenceEngine(new VariationalMessagePassing()); var ca = engine.GetCompiledInferenceAlgorithm(evidence, w); ca.Reset(); double oldLogEvidence = double.NegativeInfinity; for (int i = 0; i < 1000; i++) { ca.Update(1); double logEvidence1 = ca.Marginal <Bernoulli>(evidence.NameInGeneratedCode).LogOdds; Console.WriteLine(logEvidence1); if (i > 20 && System.Math.Abs(logEvidence1 - oldLogEvidence) < 0.01) { break; } oldLogEvidence = logEvidence1; } DistributionArray <Beta> wInferred = ca.Marginal <DistributionArray <Beta> >(w.NameInGeneratedCode); for (int p = 0; p < P; p++) { Console.WriteLine("w[{0}] = {1} +/- {2} should be {3}", p, wInferred[p].GetMean(), System.Math.Sqrt(wInferred[p].GetVariance()), b[p]); } }
private State LoadOrCreateState() { var file = Path.Combine(dir, "simple_generator.txt"); if (!File.Exists(file)) { var firstChar = GetFirstCharProbabilities(); var charToChar = GetCharToCharTransitionMatrix(); var startSeed = 1u; firstChar.MakeDistributionBinary(); foreach (var distr in charToChar) { distr.MakeDistributionBinary(); } File.WriteAllLines(file, new[] { string.Join(",", firstChar.Distribution), Environment.NewLine, string.Join(Environment.NewLine, charToChar.Select(d => string.Join(",", d.Distribution))), Environment.NewLine, }); Console.WriteLine($"Sums: {firstChar.Sum}; {string.Join(",", charToChar.Select(d => d.Sum))}"); Console.WriteLine("File created"); return(new State { FirstChar = firstChar, CharToChar = charToChar, Seed = startSeed, CoolSeeds = new List <uint>() }); } else { DistributionArray firstChar = null; DistributionArray[] charToChar = null; var coolSeeds = new List <uint>(); var content = File.ReadAllLines(file); var charToCharList = new List <DistributionArray>(); foreach (var line in content) { if (string.IsNullOrEmpty(line)) { continue; } if (firstChar == null) { firstChar = new DistributionArray { Distribution = line.Split(',').Select(int.Parse).ToArray() }; firstChar.Sum = firstChar.Distribution.Sum(); } else if (line.Contains(",")) { var arr = new DistributionArray { Distribution = line.Split(',').Select(int.Parse).ToArray() }; arr.Sum = arr.Distribution.Sum(); charToCharList.Add(arr); } else { coolSeeds.Add(uint.Parse(line)); } } charToChar = charToCharList.ToArray(); var startSeed = coolSeeds.Max() + 1; Console.WriteLine($"Sums: {firstChar.Sum}; {string.Join(",", charToChar.Select(d => d.Sum))}"); Console.WriteLine("Loaded seed = " + startSeed); return(new State { FirstChar = firstChar, CharToChar = charToChar, Seed = startSeed, CoolSeeds = coolSeeds }); } }