public TokenizedInput Tokenize(string input)
{
var result = new TokenizedInput();
if (input != null)
{
input = input.ToLower();
input = input.Replace(',', ' ');
var tokens = input.Split(' ');
foreach (string token in tokens)
{
if (IgnoredWords.Contains(token))
{
continue;
}
if (ReplacedWords.Contains(token))
{
result.Add(ReplacedWords.ReplacementFor(token));
}
else
{
result.Add(token);
}
}
}
return(result);
}
