/// <summary>
/// Use NLP method to process the token
/// </summary>
/// <param name="tk"></param>
private void ProcessToken(Token tk)
{
PunctuationMarker.Mark(tk);
Stemmer.Stem(tk); // convert to root form
StopwordMarker.Mark(tk);
IrregularMarker.Mark(tk);
}
/// <summary>
/// Load the stopwords from the txt file
/// </summary>
internal static async Task <string[]> Load()
{
StorageFolder folder = Windows.ApplicationModel.Package.Current.InstalledLocation;
StorageFile file = await folder.GetFileAsync(FilePath.StopWords);
List <string> stopwordlist = new List <string>();
var lines = await FileIO.ReadLinesAsync(file);
foreach (string line in lines)
{
string stemmedStopword = Stemmer.Stem(line.Trim());
stopwordlist.Add(stemmedStopword);
}
stopwords = stopwordlist.ToArray();
return(stopwords);
}