public virtual void TestStopList()
{
CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false);
StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
TokenStream stream = newStop.TokenStream("test", "This is a good test of the english stop analyzer");
try
{
assertNotNull(stream);
ICharTermAttribute termAtt = stream.GetAttribute<ICharTermAttribute>();
stream.Reset();
while (stream.IncrementToken())
{
string text = termAtt.ToString();
assertFalse(stopWordsSet.contains(text));
}
stream.End();
}
finally
{
IOUtils.CloseWhileHandlingException(stream);
}
}
public virtual void TestStop()
{
Analyzer a = new StopAnalyzer(TEST_VERSION_CURRENT);
AssertAnalyzesTo(a, "foo bar FOO BAR", new string[] { "foo", "bar", "foo", "bar" });
AssertAnalyzesTo(a, "foo a bar such FOO THESE BAR", new string[] { "foo", "bar", "foo", "bar" });
}
public virtual void TestStopListPositions()
{
CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false);
StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
string s = "This is a good test of the english stop analyzer with positions";
int[] expectedIncr = new int[] { 1, 1, 1, 3, 1, 1, 1, 2, 1 };
TokenStream stream = newStop.TokenStream("test", s);
try
{
assertNotNull(stream);
int i = 0;
ICharTermAttribute termAtt = stream.GetAttribute<ICharTermAttribute>();
IPositionIncrementAttribute posIncrAtt = stream.AddAttribute<IPositionIncrementAttribute>();
stream.Reset();
while (stream.IncrementToken())
{
string text = termAtt.ToString();
assertFalse(stopWordsSet.contains(text));
assertEquals(expectedIncr[i++], posIncrAtt.PositionIncrement);
}
stream.End();
}
finally
{
IOUtils.CloseWhileHandlingException(stream);
}
}
