public virtual void TestSimple()
{
Analyzer a = new SimpleAnalyzer(TEST_VERSION_CURRENT);
AssertAnalyzesTo(a, "foo bar FOO BAR", new string[] { "foo", "bar", "foo", "bar" });
AssertAnalyzesTo(a, "foo bar . FOO <> BAR", new string[] { "foo", "bar", "foo", "bar" });
AssertAnalyzesTo(a, "foo.bar.FOO.BAR", new string[] { "foo", "bar", "foo", "bar" });
AssertAnalyzesTo(a, "U.S.A.", new string[] { "u", "s", "a" });
AssertAnalyzesTo(a, "C++", new string[] { "c" });
AssertAnalyzesTo(a, "B2B", new string[] { "b", "b" });
AssertAnalyzesTo(a, "2B", new string[] { "b" });
AssertAnalyzesTo(a, "\"QUOTED\" word", new string[] { "quoted", "word" });
}
public virtual void TestSimple()
{
Analyzer a = new SimpleAnalyzer(TEST_VERSION_CURRENT);
AssertAnalyzesTo(a, "foo bar FOO BAR", new string[] { "foo", "bar", "foo", "bar" });
AssertAnalyzesTo(a, "foo bar . FOO <> BAR", new string[] { "foo", "bar", "foo", "bar" });
AssertAnalyzesTo(a, "foo.bar.FOO.BAR", new string[] { "foo", "bar", "foo", "bar" });
AssertAnalyzesTo(a, "U.S.A.", new string[] { "u", "s", "a" });
AssertAnalyzesTo(a, "C++", new string[] { "c" });
AssertAnalyzesTo(a, "B2B", new string[] { "b", "b" });
AssertAnalyzesTo(a, "2B", new string[] { "b" });
AssertAnalyzesTo(a, "\"QUOTED\" word", new string[] { "quoted", "word" });
}
public virtual void TestPerField()
{
string text = "Qwerty";
IDictionary<string, Analyzer> analyzerPerField = new Dictionary<string, Analyzer>();
analyzerPerField["special"] = new SimpleAnalyzer(TEST_VERSION_CURRENT);
PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), analyzerPerField);
TokenStream tokenStream = analyzer.TokenStream("field", text);
try
{
ICharTermAttribute termAtt = tokenStream.GetAttribute<ICharTermAttribute>();
tokenStream.Reset();
assertTrue(tokenStream.IncrementToken());
assertEquals("WhitespaceAnalyzer does not lowercase", "Qwerty", termAtt.ToString());
assertFalse(tokenStream.IncrementToken());
tokenStream.End();
}
finally
{
IOUtils.CloseWhileHandlingException(tokenStream);
}
tokenStream = analyzer.TokenStream("special", text);
try
{
ICharTermAttribute termAtt = tokenStream.GetAttribute<ICharTermAttribute>();
tokenStream.Reset();
assertTrue(tokenStream.IncrementToken());
assertEquals("SimpleAnalyzer lowercases", "qwerty", termAtt.ToString());
assertFalse(tokenStream.IncrementToken());
tokenStream.End();
}
finally
{
IOUtils.CloseWhileHandlingException(tokenStream);
}
}
