| public static AssertTokens ( ITokenizer tokenizer ) : void | ||
| tokenizer | ITokenizer | |
| return | void | |
public void TestSimpleRegex()
{
TokenAssertions.AssertTokens("a token, 100 23 1g 144kg other token",
_filter,
new Token("a", 0),
new Token("token", 2),
new Token("other", 25),
new Token("token", 31),
null
);
}
public void TestWordFilterChaining()
{
string[] filteredWords = { "de", "com" };
string text = "Bolo dé Açafrão com Rúcula";
TokenAssertions.AssertTokens(text, new WordFilter(new SpecialCharactersFilter(), filteredWords),
new Token("bolo", 0),
new Token("acafrao", 8),
new Token("rucula", 20),
null
);
}
public void TestFilter()
{
string[] filteredWords = { "de", "com" };
string text = "Bolo de chocolate com calda de morango";
TokenAssertions.AssertTokens(text, new WordFilter(filteredWords),
new Token("Bolo", 0),
new Token("chocolate", 8),
new Token("calda", 22),
new Token("morango", 31),
null);
}
public void TestReplacements()
{
string text = "áéíóú calção aviões aiouao";
TokenAssertions.AssertTokens(text, new SpecialCharactersFilter(),
new Token("aeiou", 0),
new Token("calcao", 6),
new Token("avioes", 13),
new Token("aiouao", 20),
null
);
}
public void TestSimpleStrings()
{
string text = "a foo Bar a�a\n45\n\n\n";
TokenAssertions.AssertTokens(new StringTokenizer(text),
new Token("a", 0),
new Token("foo", 2),
new Token("Bar", 6),
new Token("a�a", 10),
new Token("45", 14),
null
);
TokenAssertions.AssertTokens(new StringTokenizer(""),
(Token)null);
TokenAssertions.AssertTokens(new StringTokenizer("\n\t "),
(Token)null);
TokenAssertions.AssertTokens(new StringTokenizer("\n\t a"),
new Token("a", 4),
null);
}