public virtual void TestStopListPositions()
{
CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false);
StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
string s = "This is a good test of the english stop analyzer with positions";
int[] expectedIncr = new int[] { 1, 1, 1, 3, 1, 1, 1, 2, 1 };
TokenStream stream = newStop.TokenStream("test", s);
try
{
assertNotNull(stream);
int i = 0;
ICharTermAttribute termAtt = stream.GetAttribute <ICharTermAttribute>();
IPositionIncrementAttribute posIncrAtt = stream.AddAttribute <IPositionIncrementAttribute>();
stream.Reset();
while (stream.IncrementToken())
{
string text = termAtt.ToString();
assertFalse(stopWordsSet.contains(text));
assertEquals(expectedIncr[i++], posIncrAtt.PositionIncrement);
}
stream.End();
}
finally
{
IOUtils.CloseWhileHandlingException(stream);
}
}
public virtual void TestDefaults()
{
assertTrue(stop != null);
TokenStream stream = stop.TokenStream("test", "This is a test of the english stop analyzer");
try
{
assertTrue(stream != null);
ICharTermAttribute termAtt = stream.GetAttribute <ICharTermAttribute>();
stream.Reset();
while (stream.IncrementToken())
{
assertFalse(inValidTokens.Contains(termAtt.ToString()));
}
stream.End();
}
finally
{
IOUtils.CloseWhileHandlingException(stream);
}
}
public virtual void TestStopList()
{
CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false);
StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
TokenStream stream = newStop.TokenStream("test", "This is a good test of the english stop analyzer");
try
{
assertNotNull(stream);
ICharTermAttribute termAtt = stream.GetAttribute<ICharTermAttribute>();
stream.Reset();
while (stream.IncrementToken())
{
string text = termAtt.ToString();
assertFalse(stopWordsSet.contains(text));
}
stream.End();
}
finally
{
IOUtils.CloseWhileHandlingException(stream);
}
}
public virtual void TestStopList()
{
CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false);
StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
TokenStream stream = newStop.TokenStream("test", "This is a good test of the english stop analyzer");
try
{
assertNotNull(stream);
ICharTermAttribute termAtt = stream.GetAttribute <ICharTermAttribute>();
stream.Reset();
while (stream.IncrementToken())
{
string text = termAtt.ToString();
assertFalse(stopWordsSet.contains(text));
}
stream.End();
}
finally
{
IOUtils.CloseWhileHandlingException(stream);
}
}
public virtual void TestStopListPositions()
{
CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false);
StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
string s = "This is a good test of the english stop analyzer with positions";
int[] expectedIncr = new int[] { 1, 1, 1, 3, 1, 1, 1, 2, 1 };
TokenStream stream = newStop.TokenStream("test", s);
try
{
assertNotNull(stream);
int i = 0;
ICharTermAttribute termAtt = stream.GetAttribute<ICharTermAttribute>();
IPositionIncrementAttribute posIncrAtt = stream.AddAttribute<IPositionIncrementAttribute>();
stream.Reset();
while (stream.IncrementToken())
{
string text = termAtt.ToString();
assertFalse(stopWordsSet.contains(text));
assertEquals(expectedIncr[i++], posIncrAtt.PositionIncrement);
}
stream.End();
}
finally
{
IOUtils.CloseWhileHandlingException(stream);
}
}
