public static void Tokenize(Review review) { var words = review.text.Split(' '); bool negate = false; foreach (var word in words) { if (word != "") { review.AddToken(GetToken(word, !negate)); if (NEGATE_WORDS.Contains(word) || word.EndsWith("n't")) { negate = !negate; } if (STOP_SYMBOLS.Contains(word.Last())) { negate = false; review.AddToken(word.Last().ToString()); } } } review.text = null; }