public void TestPunctuation()
{
string text = "A foo,bar goest! flu? Oh, yes, flu!!! really? yep.\n.\tdidn't think [so..(yep)";
TokenAssertions.AssertTokenValues(new StringTokenizer(text),
"A", "foo", "bar", "goest", "flu",
"Oh", "yes", "flu", "really", "yep",
"didn", "t", "think", "so", "yep"
);
}
public void TestSimpleRegex()
{
TokenAssertions.AssertTokens("a token, 100 23 1g 144kg other token",
_filter,
new Token("a", 0),
new Token("token", 2),
new Token("other", 25),
new Token("token", 31),
null
);
}
public void TestWordFilterChaining()
{
string[] filteredWords = { "de", "com" };
string text = "Bolo dé Açafrão com Rúcula";
TokenAssertions.AssertTokens(text, new WordFilter(new SpecialCharactersFilter(), filteredWords),
new Token("bolo", 0),
new Token("acafrao", 8),
new Token("rucula", 20),
null
);
}
public void TestFilter()
{
string[] filteredWords = { "de", "com" };
string text = "Bolo de chocolate com calda de morango";
TokenAssertions.AssertTokens(text, new WordFilter(filteredWords),
new Token("Bolo", 0),
new Token("chocolate", 8),
new Token("calda", 22),
new Token("morango", 31),
null);
}
public void TestReplacements()
{
string text = "áéíóú calção aviões aiouao";
TokenAssertions.AssertTokens(text, new SpecialCharactersFilter(),
new Token("aeiou", 0),
new Token("calcao", 6),
new Token("avioes", 13),
new Token("aiouao", 20),
null
);
}
public void TestIndexSerialization()
{
_index = TokenAssertions.SerializeDeserialize(_index) as IIndex;
FullTextSearchIndex index = _index as FullTextSearchIndex;
IRecord[] records = index.Records;
AssertEquals(3, records.Length);
_record1 = FindByTitle(records, (string)_record1["Title"]);
_record2 = FindByTitle(records, (string)_record2["Title"]);
_record3 = FindByTitle(records, (string)_record3["Title"]);
TestSimpleSearch();
}
public void TestSimpleStrings()
{
string text = "a foo Bar a�a\n45\n\n\n";
TokenAssertions.AssertTokens(new StringTokenizer(text),
new Token("a", 0),
new Token("foo", 2),
new Token("Bar", 6),
new Token("a�a", 10),
new Token("45", 14),
null
);
TokenAssertions.AssertTokens(new StringTokenizer(""),
(Token)null);
TokenAssertions.AssertTokens(new StringTokenizer("\n\t "),
(Token)null);
TokenAssertions.AssertTokens(new StringTokenizer("\n\t a"),
new Token("a", 4),
null);
}
public void TestFilter()
{
string text = "a bc dado de o";
TokenAssertions.AssertTokenValues(text,
new TokenLengthFilter(1),
"a", "bc", "dado", "de", "o"
);
TokenAssertions.AssertTokenValues(text,
new TokenLengthFilter(2),
"bc", "dado", "de"
);
TokenAssertions.AssertTokenValues(text,
new TokenLengthFilter(3),
"dado"
);
TokenAssertions.AssertTokenValues(text,
new TokenLengthFilter(4),
"dado"
);
}
public void TestSerializable()
{
_filter = (RegexTokenFilter)TokenAssertions.SerializeDeserialize(_filter);
TestSimpleRegex();
}