public virtual void TestStopListPositions() { CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false); StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet); string s = "This is a good test of the english stop analyzer with positions"; int[] expectedIncr = new int[] { 1, 1, 1, 3, 1, 1, 1, 2, 1 }; TokenStream stream = newStop.TokenStream("test", s); try { assertNotNull(stream); int i = 0; ICharTermAttribute termAtt = stream.GetAttribute <ICharTermAttribute>(); IPositionIncrementAttribute posIncrAtt = stream.AddAttribute <IPositionIncrementAttribute>(); stream.Reset(); while (stream.IncrementToken()) { string text = termAtt.ToString(); assertFalse(stopWordsSet.contains(text)); assertEquals(expectedIncr[i++], posIncrAtt.PositionIncrement); } stream.End(); } finally { IOUtils.CloseWhileHandlingException(stream); } }
public virtual void TestDefaults() { assertTrue(stop != null); TokenStream stream = stop.TokenStream("test", "This is a test of the english stop analyzer"); try { assertTrue(stream != null); ICharTermAttribute termAtt = stream.GetAttribute <ICharTermAttribute>(); stream.Reset(); while (stream.IncrementToken()) { assertFalse(inValidTokens.Contains(termAtt.ToString())); } stream.End(); } finally { IOUtils.CloseWhileHandlingException(stream); } }
public virtual void TestStopList() { CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false); StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet); TokenStream stream = newStop.TokenStream("test", "This is a good test of the english stop analyzer"); try { assertNotNull(stream); ICharTermAttribute termAtt = stream.GetAttribute<ICharTermAttribute>(); stream.Reset(); while (stream.IncrementToken()) { string text = termAtt.ToString(); assertFalse(stopWordsSet.contains(text)); } stream.End(); } finally { IOUtils.CloseWhileHandlingException(stream); } }
public virtual void TestStopList() { CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false); StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet); TokenStream stream = newStop.TokenStream("test", "This is a good test of the english stop analyzer"); try { assertNotNull(stream); ICharTermAttribute termAtt = stream.GetAttribute <ICharTermAttribute>(); stream.Reset(); while (stream.IncrementToken()) { string text = termAtt.ToString(); assertFalse(stopWordsSet.contains(text)); } stream.End(); } finally { IOUtils.CloseWhileHandlingException(stream); } }
public virtual void TestStopListPositions() { CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false); StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet); string s = "This is a good test of the english stop analyzer with positions"; int[] expectedIncr = new int[] { 1, 1, 1, 3, 1, 1, 1, 2, 1 }; TokenStream stream = newStop.TokenStream("test", s); try { assertNotNull(stream); int i = 0; ICharTermAttribute termAtt = stream.GetAttribute<ICharTermAttribute>(); IPositionIncrementAttribute posIncrAtt = stream.AddAttribute<IPositionIncrementAttribute>(); stream.Reset(); while (stream.IncrementToken()) { string text = termAtt.ToString(); assertFalse(stopWordsSet.contains(text)); assertEquals(expectedIncr[i++], posIncrAtt.PositionIncrement); } stream.End(); } finally { IOUtils.CloseWhileHandlingException(stream); } }