// ReSharper disable once UnusedParameter.Local static void Main(string[] args) { const string kText = "this is a test. this is only a test. if this had been an actual program it would not have been so dumb"; NGrams ngs = new NGrams(3); Console.WriteLine("N-Grams from 'raw' text..."); var ngrams = ngs.GenerateNGramsStrings(kText); foreach (var nGram in ngrams) { Console.WriteLine(nGram.ToString()); } Console.WriteLine("========================================"); Console.WriteLine("N-Grams from 'normalize and stop words removed' text..."); var normalizer = new TextNormalizer(); var normalizedText = normalizer.NormalizeText(kText); var stopWordsRemover = new StopWordsRemover(); var stopWordsRemoved = stopWordsRemover.RemoveStopWords(normalizedText); Console.WriteLine("Normalized and Stop words removed text:"); Console.WriteLine(stopWordsRemoved); Console.WriteLine("\n\n\n"); var normalizedWithoutStopWordsRemoved = normalizer.NormalizeText(kText); Console.WriteLine("Normalized and Stop words NOT removed text:"); Console.WriteLine(normalizedWithoutStopWordsRemoved); Console.WriteLine("\n\n\n"); var ngrams2 = ngs.GenerateNGrams(stopWordsRemoved); Console.WriteLine("N-Grams from 'normalized' text..."); foreach (var nGram in ngrams2) { Console.WriteLine(nGram.ToString()); } PrintEnd(); }
public long GetNGramScore(int index) { return(NGrams.Where(each => each.Index >= index).Sum(each => each.NGram.Frequency)); }
public long GetNGramScore() { return(NGrams.Sum(each => each.NGram.Frequency)); }
public static NGramTree GetNGrams(int length) { InitializeNGrams(); return(NGrams.ContainsKey(length) ? NGrams[length] : null); }