private static void Execute(string words)
{
SearchTokens st = new SearchTokens();
Sanitizer sa = new Sanitizer();
Console.WriteLine("");
Console.WriteLine("Simple Tokenizer");
List<string> tokens = st.ForSearch(words);
foreach (string token in tokens)
{
Console.Write("[" + token + "] ");
}
Console.WriteLine("");
Console.WriteLine("-------------------------------------------");
Console.WriteLine("");
Console.WriteLine("Descriminated Tokenizer with escape");
TokenLists tokenList = st.ForSearchDescriminated(words);
Console.WriteLine(" Singular");
Console.Write(" ");
foreach (string token in tokenList.SingularWords)
{
Console.Write("[" + token + "] ");
}
Console.WriteLine("");
Console.WriteLine("");
Console.WriteLine(" Agregated and Escaped words");
Console.Write(" ");
foreach (string token in tokenList.AgregatedWords)
{
Console.Write("[" + sa.EscapeNonAlphanumeric(token) + "] ");
}
}