private static void Execute(string words)
{
SearchTokens st = new SearchTokens();
Sanitizer sa = new Sanitizer();
Console.WriteLine("");
Console.WriteLine("Simple Tokenizer");
List<string> tokens = st.ForSearch(words);
foreach (string token in tokens)
{
Console.Write("[" + token + "] ");
}
Console.WriteLine("");
Console.WriteLine("-------------------------------------------");
Console.WriteLine("");
Console.WriteLine("Descriminated Tokenizer with escape");
TokenLists tokenList = st.ForSearchDescriminated(words);
Console.WriteLine(" Singular");
Console.Write(" ");
foreach (string token in tokenList.SingularWords)
{
Console.Write("[" + token + "] ");
}
Console.WriteLine("");
Console.WriteLine("");
Console.WriteLine(" Agregated and Escaped words");
Console.Write(" ");
foreach (string token in tokenList.AgregatedWords)
{
Console.Write("[" + sa.EscapeNonAlphanumeric(token) + "] ");
}
}
public void ChangeWordGatheringChars()
{
SearchTokensOption options = new SearchTokensOption();
options.WordGatheringChars = "|%";
SearchTokens stKeep = new SearchTokens(options);
List<string> words = stKeep.ForSearch("|foo ||bar% one two %three four ");
Assert.AreEqual(5, words.Count);
Assert.Contains("foo ", words);
Assert.Contains("bar", words);
Assert.Contains("one", words);
Assert.Contains("two", words);
Assert.Contains("three four ", words);
}
public void TokensDescriminatedByType()
{
SearchTokens st = new SearchTokens();
TokenLists result = st.ForSearchDescriminated("this \"is a\" test \"embrace your selfs\" the winter is comming!");
Assert.AreEqual(2, result.AgregatedWords.Count);
Assert.Contains("is a", result.AgregatedWords);
Assert.Contains("embrace your selfs", result.AgregatedWords);
Assert.AreEqual(6, result.SingularWords.Count);
}
public void TrimWhiteSpace()
{
SearchTokensOption options = new SearchTokensOption();
options.TrimWhiteSpace = true;
SearchTokens stTrim = new SearchTokens(options);
List<string> words = stTrim.ForSearch(" ' foo ' 'bar ' ' simon' 'Victor Daniel Martins'");
Assert.AreEqual(4, words.Count);
Assert.AreEqual("foo", words[0]);
Assert.AreEqual("bar", words[1]);
Assert.AreEqual("simon", words[2]);
Assert.AreEqual("Victor Daniel Martins", words[3]);
}
public void Setup()
{
st = new SearchTokens();
}