private List <string> GenerateTerms(Dictionary <string, string> docs)
{
List <string> uniques = new List <string>();
foreach (KeyValuePair <string, string> kvp in docs)
{
Tokeniser tokenizer = new Tokeniser();
List <string> words = tokenizer.Partition(kvp.Value, _stopWords, _useWordStemmer);
foreach (string word in words)
{
if (!uniques.Contains(word))
{
uniques.Add(word);
}
}
}
return(uniques);
}
private IDictionary GetWordFrequency(string input)
{
string convertedInput = input.ToLower();
Tokeniser tokenizer = new Tokeniser();
List <string> words = tokenizer.Partition(convertedInput, _stopWords, _useWordStemmer);
words.Sort();
List <string> distinctWords = GetDistinctWords(words);
IDictionary result = new Hashtable();
foreach (string word in distinctWords)
{
object tmp;
tmp = CountWords(word, words);
result[word] = tmp;
}
return(result);
}