Exemple #1
0
        public virtual void TestStopListPositions()
        {
            CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false);
            StopAnalyzer newStop      = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
            string       s            = "This is a good test of the english stop analyzer with positions";

            int[]       expectedIncr = new int[] { 1, 1, 1, 3, 1, 1, 1, 2, 1 };
            TokenStream stream       = newStop.TokenStream("test", s);

            try
            {
                assertNotNull(stream);
                int i = 0;
                ICharTermAttribute          termAtt    = stream.GetAttribute <ICharTermAttribute>();
                IPositionIncrementAttribute posIncrAtt = stream.AddAttribute <IPositionIncrementAttribute>();

                stream.Reset();
                while (stream.IncrementToken())
                {
                    string text = termAtt.ToString();
                    assertFalse(stopWordsSet.contains(text));
                    assertEquals(expectedIncr[i++], posIncrAtt.PositionIncrement);
                }
                stream.End();
            }
            finally
            {
                IOUtils.CloseWhileHandlingException(stream);
            }
        }
Exemple #2
0
        public virtual void TestDefaults()
        {
            assertTrue(stop != null);
            TokenStream stream = stop.TokenStream("test", "This is a test of the english stop analyzer");

            try
            {
                assertTrue(stream != null);
                ICharTermAttribute termAtt = stream.GetAttribute <ICharTermAttribute>();
                stream.Reset();

                while (stream.IncrementToken())
                {
                    assertFalse(inValidTokens.Contains(termAtt.ToString()));
                }
                stream.End();
            }
            finally
            {
                IOUtils.CloseWhileHandlingException(stream);
            }
        }
        public virtual void TestStopList()
        {
            CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false);
            StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
            TokenStream stream = newStop.TokenStream("test", "This is a good test of the english stop analyzer");
            try
            {
                assertNotNull(stream);
                ICharTermAttribute termAtt = stream.GetAttribute<ICharTermAttribute>();

                stream.Reset();
                while (stream.IncrementToken())
                {
                    string text = termAtt.ToString();
                    assertFalse(stopWordsSet.contains(text));
                }
                stream.End();
            }
            finally
            {
                IOUtils.CloseWhileHandlingException(stream);
            }
        }
Exemple #4
0
        public virtual void TestStopList()
        {
            CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false);
            StopAnalyzer newStop      = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
            TokenStream  stream       = newStop.TokenStream("test", "This is a good test of the english stop analyzer");

            try
            {
                assertNotNull(stream);
                ICharTermAttribute termAtt = stream.GetAttribute <ICharTermAttribute>();

                stream.Reset();
                while (stream.IncrementToken())
                {
                    string text = termAtt.ToString();
                    assertFalse(stopWordsSet.contains(text));
                }
                stream.End();
            }
            finally
            {
                IOUtils.CloseWhileHandlingException(stream);
            }
        }
        public virtual void TestStopListPositions()
        {
            CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, new string[] { "good", "test", "analyzer" }, false);
            StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
            string s = "This is a good test of the english stop analyzer with positions";
            int[] expectedIncr = new int[] { 1, 1, 1, 3, 1, 1, 1, 2, 1 };
            TokenStream stream = newStop.TokenStream("test", s);
            try
            {
                assertNotNull(stream);
                int i = 0;
                ICharTermAttribute termAtt = stream.GetAttribute<ICharTermAttribute>();
                IPositionIncrementAttribute posIncrAtt = stream.AddAttribute<IPositionIncrementAttribute>();

                stream.Reset();
                while (stream.IncrementToken())
                {
                    string text = termAtt.ToString();
                    assertFalse(stopWordsSet.contains(text));
                    assertEquals(expectedIncr[i++], posIncrAtt.PositionIncrement);
                }
                stream.End();
            }
            finally
            {
                IOUtils.CloseWhileHandlingException(stream);
            }
        }