Exemple #1
0
        private NGramItem GetNGrams(string line)
        {
            var segments  = TextCleaner.CleanSplit(" ", line);
            var frequency = long.Parse(segments[0]);

            if (English.IllegalTokens.Any(line.Contains))
            {
                return(null);
            }

            if (frequency < MinimumFrequency)
            {
                return(null);
            }

            var words = segments.Skip(1).ToArray();

            return(new NGramItem {
                Text = string.Join(" ", words),
                Frequency = frequency,
                Words = words
            });
        }