/// <summary> /// Assuming NGram[] is homogenous /// </summary> public static float CalculateProbability( Dictionary <NGram <T>, HashSet <NGram <T> > > blockedGrams, NGramDistribution <T> distrubution, NGram <T>[] array, int x, int y) { if (!blockedGrams.ContainsKey(array[x])) { return(0); } var value = blockedGrams[array[x]]; if (!value.Contains(array[y])) { return(0); } float sumCount = distrubution .AsParallel() .Where(z => value.Contains(z.Key)) .Sum(q => q.Value); return(distrubution.RetrieveCount(array[y]) / sumCount); }
private NGramGraphMarkovChain(bool het) { this.graph = new Dictionary <NGram <T>, List <NonRecursiveNGramProbabilisticEdge <T> > >(); this.IsHeterogenous = het; this.Distrubution = new NGramDistribution <T>(new NGram <T> [0]); this.Grams = HomogenousNGrams <T> .DirectBuiltUnsafe(new NGram <T> [0], 1); }
private void BuildFromHomogenousNGrams(INGrams <T> grams, NGram <T>[] array, NGramDistribution <T> distrubution) { var blockedGrams = NGramHomogenousMatrixMarkovChainHelper <T> .BuildBlockedNGrams(grams.ToArray()); var transitionMatrix = Matrix <float> .Build.Sparse( array.Length, array.Length, new Func <int, int, float>( (x, y) => NGramHomogenousMatrixMarkovChainHelper <T> .CalculateProbability(blockedGrams, distrubution, array, x, y))); int rowIndex = 0; foreach (var row in transitionMatrix.EnumerateRows()) { this.graph.Add(array[rowIndex], new List <NonRecursiveNGramProbabilisticEdge <T> >()); for (int i = 0; i < row.Count; i++) { if (row[i] != 0) { this.graph[array[rowIndex]].Add(new NonRecursiveNGramProbabilisticEdge <T>(row[i], array[i])); } } rowIndex++; } }
public NGramGraphMarkovChain(INGrams <T> grams) { grams.NullCheck(); var array = new HashSet <NGram <T> >(grams).ToArray(); this.Distrubution = new NGramDistribution <T>(grams); var typeOfNGram = (grams as IHeterogenousNGrams <T>); this.IsHeterogenous = typeOfNGram != null; this.graph = new Dictionary <NGram <T>, List <NonRecursiveNGramProbabilisticEdge <T> > >(); this.Grams = grams; if (!this.IsHeterogenous) { this.BuildFromHomogenousNGrams(grams, array, this.Distrubution); } else { this.BuildFromHeterogenousNGrams(typeOfNGram, array, this.Distrubution); } }
public HomogenousNGramMatrixMarkovChain(HomogenousNGrams <T> grams) { grams.NullCheck(); (grams as IHeterogenousNGrams <T> == null).AssertTrue(); var array = new HashSet <NGram <T> >(grams).ToArray(); var blockedGrams = NGramHomogenousMatrixMarkovChainHelper <T> .BuildBlockedNGrams(grams.ToArray()); this.distrubution = new NGramDistribution <T>(grams); this.transitionMatrix = Matrix <float> .Build.Sparse( array.Length, array.Length, new Func <int, int, float>( (x, y) => NGramHomogenousMatrixMarkovChainHelper <T> .CalculateProbability(blockedGrams, distrubution, array, x, y))); this.indices = new Dictionary <int, NGram <T> >(); for (int i = 0; i < array.Length; i++) { this.indices.Add(i, array[i]); } }
public static List <float> CalculateProbability(KeyValuePair <NGram <T>, HashSet <NGram <T> > > set, NGramDistribution <T> distrubtion) { distrubtion.NullCheck(); float sum = set.Value.Select(x => distrubtion.RetrieveCount(x)).Sum(); return(set.Value .Select(x => (float)distrubtion.RetrieveCount(x) / sum) .ToList()); }
private NGramGraphMarkovChain(Dictionary <NGram <T>, List <NonRecursiveNGramProbabilisticEdge <T> > > dictionary, bool p, NGramDistribution <T> orginal) { this.graph = dictionary; this.IsHeterogenous = p; this.Distrubution = orginal; }
private void BuildFromHeterogenousNGrams(IHeterogenousNGrams <T> grams, NGram <T>[] array, NGramDistribution <T> distrubution) { var blocked = NGramHeterogenousMatrixMarkovChainHelper <T> .BuildBlockedNGrams((from item in grams.SubHomogenousNGramConstructs() select item.ToArray()).ToList(), grams.MinN, grams.MaxN); foreach (var item in blocked) { var data = NGramHeterogenousMatrixMarkovChainHelper <T> .CalculateProbability(item, distrubution); var final = new List <NonRecursiveNGramProbabilisticEdge <T> >(data.Zip(item.Value, (x, y) => new NonRecursiveNGramProbabilisticEdge <T>(x, y))); this.graph.Add(item.Key, final); } }