public static void Tokenize(Review review)
{
var words = review.text.Split(' ');
bool negate = false;
foreach (var word in words)
{
if (word != "")
{
review.AddToken(GetToken(word, !negate));
if (NEGATE_WORDS.Contains(word) ||
word.EndsWith("n't"))
{
negate = !negate;
}
if (STOP_SYMBOLS.Contains(word.Last()))
{
negate = false;
review.AddToken(word.Last().ToString());
}
}
}
review.text = null;
}
