TokenStream output from a tee with optional filtering.
상속: TokenStream
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void test() throws java.io.IOException
        public virtual void test()
        {
            TokenRangeSinkFilter sinkFilter = new TokenRangeSinkFilter(2, 4);
            string             test         = "The quick red fox jumped over the lazy brown dogs";
            TeeSinkTokenFilter tee          = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));

            TeeSinkTokenFilter.SinkTokenStream rangeToks = tee.newSinkTokenStream(sinkFilter);

            int count = 0;

            tee.reset();
            while (tee.incrementToken())
            {
                count++;
            }

            int sinkCount = 0;

            rangeToks.reset();
            while (rangeToks.incrementToken())
            {
                sinkCount++;
            }

            assertTrue(count + " does not equal: " + 10, count == 10);
            assertTrue("rangeToks Size: " + sinkCount + " is not: " + 2, sinkCount == 2);
        }
예제 #2
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testMultipleSources() throws Exception
        public virtual void testMultipleSources()
        {
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final TeeSinkTokenFilter tee1 = new TeeSinkTokenFilter(new MockTokenizer(new java.io.StringReader(buffer1.toString()), MockTokenizer.WHITESPACE, false));
            TeeSinkTokenFilter tee1 = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer1.ToString()), MockTokenizer.WHITESPACE, false));

//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final TeeSinkTokenFilter.SinkTokenStream dogDetector = tee1.newSinkTokenStream(dogFilter);
            TeeSinkTokenFilter.SinkTokenStream dogDetector = tee1.newSinkTokenStream(dogFilter);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final TeeSinkTokenFilter.SinkTokenStream theDetector = tee1.newSinkTokenStream(theFilter);
            TeeSinkTokenFilter.SinkTokenStream theDetector = tee1.newSinkTokenStream(theFilter);
            tee1.reset();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final TokenStream source1 = new CachingTokenFilter(tee1);
            TokenStream source1 = new CachingTokenFilter(tee1);

            tee1.addAttribute(typeof(CheckClearAttributesAttribute));
            dogDetector.addAttribute(typeof(CheckClearAttributesAttribute));
            theDetector.addAttribute(typeof(CheckClearAttributesAttribute));

//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(new MockTokenizer(new java.io.StringReader(buffer2.toString()), MockTokenizer.WHITESPACE, false));
            TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer2.ToString()), MockTokenizer.WHITESPACE, false));

            tee2.addSinkTokenStream(dogDetector);
            tee2.addSinkTokenStream(theDetector);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final TokenStream source2 = tee2;
            TokenStream source2 = tee2;

            assertTokenStreamContents(source1, tokens1);
            assertTokenStreamContents(source2, tokens2);

            assertTokenStreamContents(theDetector, new string[] { "The", "the", "The", "the" });
            assertTokenStreamContents(dogDetector, new string[] { "Dogs", "Dogs" });

            source1.reset();
            TokenStream lowerCasing = new LowerCaseFilter(TEST_VERSION_CURRENT, source1);

            string[] lowerCaseTokens = new string[tokens1.Length];
            for (int i = 0; i < tokens1.Length; i++)
            {
                lowerCaseTokens[i] = tokens1[i].ToLower(Locale.ROOT);
            }
            assertTokenStreamContents(lowerCasing, lowerCaseTokens);
        }
예제 #3
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void test() throws java.io.IOException
        public virtual void test()
        {
            TokenTypeSinkFilter sinkFilter = new TokenTypeSinkFilter("D");
            string test = "The quick red fox jumped over the lazy brown dogs";

            TeeSinkTokenFilter ttf = new TeeSinkTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)));

            TeeSinkTokenFilter.SinkTokenStream sink = ttf.newSinkTokenStream(sinkFilter);

            bool seenDogs = false;

            CharTermAttribute termAtt = ttf.addAttribute(typeof(CharTermAttribute));
            TypeAttribute     typeAtt = ttf.addAttribute(typeof(TypeAttribute));

            ttf.reset();
            while (ttf.incrementToken())
            {
                if (termAtt.ToString().Equals("dogs"))
                {
                    seenDogs = true;
                    assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().Equals("D") == true);
                }
                else
                {
                    assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().Equals("word"));
                }
            }
            assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);

            int sinkCount = 0;

            sink.reset();
            while (sink.incrementToken())
            {
                sinkCount++;
            }

            assertTrue("sink Size: " + sinkCount + " is not: " + 1, sinkCount == 1);
        }