private static bool AddNewWordsToWordList()
{
var words = File.ReadAllLines(NewWordsListPath, Encoding.UTF8);
if (Nuve.AreAllWordsValid(words) == false)
{
Console.WriteLine(@"Warning: Not all words in the new word list are correct");
return(false);
}
var unigrams = File.ReadAllLines(WordListPath, Encoding.UTF8).ToList();
unigrams.AddRange(words);
unigrams.Sort();
File.WriteAllLines(WordListPath, unigrams.Distinct(), Encoding.UTF8);
return(true);
}
/// <summary>
/// Tries to stem everyword in given parameter and returns a map contains word-stem pairs.
/// If a word can not be stemmed, it is ignored and does not exist in the map.
/// </summary>
/// <param name="words">words to be stemmed</param>
/// <returns>A map which contains word-stem pairs for every stemmable word in the given parameter.</returns>
public static IDictionary <string, string> Stem(IEnumerable <string> words)
{
var map = new Dictionary <string, string>();
int count = 0;
foreach (var word in words)
{
string stem;
if (!map.ContainsKey(word) && Nuve.TryStem(word, out stem))
{
map.Add(word, stem);
}
else
{
count++;
}
}
Console.WriteLine(count + " of " + words.Count() + " words can not be stemmed");
return(map);
}
