Ejemplo n.º 1
0
        /*
         * How to construct a boolean query with shingles. A query like this will
         * implicitly score those documents higher that contain the words in the query
         * in the right order and adjacent to each other.
         */
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testShingleAnalyzerWrapperBooleanQuery() throws Exception
        public virtual void testShingleAnalyzerWrapperBooleanQuery()
        {
            BooleanQuery q = new BooleanQuery();

            TokenStream ts = analyzer.tokenStream("content", "test sentence");

            try
            {
                CharTermAttribute termAtt = ts.addAttribute(typeof(CharTermAttribute));

                ts.reset();
                while (ts.incrementToken())
                {
                    string termText = termAtt.ToString();
                    q.add(new TermQuery(new Term("content", termText)), BooleanClause.Occur.SHOULD);
                }
                ts.end();
            }
            finally
            {
                IOUtils.closeWhileHandlingException(ts);
            }

            ScoreDoc[] hits  = searcher.search(q, null, 1000).scoreDocs;
            int[]      ranks = new int[] { 1, 2, 0 };
            compareRanks(hits, ranks);
        }
Ejemplo n.º 2
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void test() throws java.io.IOException
        public virtual void test()
        {
            string test = "The quick red fox jumped over the lazy brown dogs";

            NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)), 3, "D");
            bool seenDogs                = false;
            CharTermAttribute termAtt    = nptf.getAttribute(typeof(CharTermAttribute));
            TypeAttribute     typeAtt    = nptf.getAttribute(typeof(TypeAttribute));
            PayloadAttribute  payloadAtt = nptf.getAttribute(typeof(PayloadAttribute));

            nptf.reset();
            while (nptf.incrementToken())
            {
                if (termAtt.ToString().Equals("dogs"))
                {
                    seenDogs = true;
                    assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().Equals("D") == true);
                    assertTrue("payloadAtt.getPayload() is null and it shouldn't be", payloadAtt.Payload != null);
                    sbyte[] bytes = payloadAtt.Payload.bytes;     //safe here to just use the bytes, otherwise we should use offset, length
                    assertTrue(bytes.Length + " does not equal: " + payloadAtt.Payload.length, bytes.Length == payloadAtt.Payload.length);
                    assertTrue(payloadAtt.Payload.offset + " does not equal: " + 0, payloadAtt.Payload.offset == 0);
                    float pay = PayloadHelper.decodeFloat(bytes);
                    assertTrue(pay + " does not equal: " + 3, pay == 3);
                }
                else
                {
                    assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().Equals("word"));
                }
            }
            assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);
        }
Ejemplo n.º 3
0
        /*
         * This shows how to construct a phrase query containing shingles.
         */
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testShingleAnalyzerWrapperPhraseQuery() throws Exception
        public virtual void testShingleAnalyzerWrapperPhraseQuery()
        {
            PhraseQuery q = new PhraseQuery();

            TokenStream ts = analyzer.tokenStream("content", "this sentence");

            try
            {
                int j = -1;

                PositionIncrementAttribute posIncrAtt = ts.addAttribute(typeof(PositionIncrementAttribute));
                CharTermAttribute          termAtt    = ts.addAttribute(typeof(CharTermAttribute));

                ts.reset();
                while (ts.incrementToken())
                {
                    j += posIncrAtt.PositionIncrement;
                    string termText = termAtt.ToString();
                    q.add(new Term("content", termText), j);
                }
                ts.end();
            }
            finally
            {
                IOUtils.closeWhileHandlingException(ts);
            }

            ScoreDoc[] hits  = searcher.search(q, null, 1000).scoreDocs;
            int[]      ranks = new int[] { 0 };
            compareRanks(hits, ranks);
        }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testReset() throws Exception
        public virtual void testReset()
        {
            CharArraySet dict = makeDictionary("Rind", "Fleisch", "Draht", "Schere", "Gesetz", "Aufgabe", "Überwachung");

            Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("Rindfleischüberwachungsgesetz"));
            DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT, wsTokenizer, dict, CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE, CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE, CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, false);

            CharTermAttribute termAtt = tf.getAttribute(typeof(CharTermAttribute));

            tf.reset();
            assertTrue(tf.incrementToken());
            assertEquals("Rindfleischüberwachungsgesetz", termAtt.ToString());
            assertTrue(tf.incrementToken());
            assertEquals("Rind", termAtt.ToString());
            tf.end();
            tf.close();
            wsTokenizer.Reader = new StringReader("Rindfleischüberwachungsgesetz");
            tf.reset();
            assertTrue(tf.incrementToken());
            assertEquals("Rindfleischüberwachungsgesetz", termAtt.ToString());
        }
Ejemplo n.º 5
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testQueryReset() throws Exception
        public virtual void testQueryReset()
        {
            const string           input = "How the s a brown s cow d like A B thing?";
            WhitespaceTokenizer    wt    = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
            CommonGramsFilter      cgf   = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);
            CommonGramsQueryFilter nsf   = new CommonGramsQueryFilter(cgf);

            CharTermAttribute term = wt.addAttribute(typeof(CharTermAttribute));

            nsf.reset();
            assertTrue(nsf.incrementToken());
            assertEquals("How_the", term.ToString());
            assertTrue(nsf.incrementToken());
            assertEquals("the_s", term.ToString());
            nsf.close();

            wt.Reader = new StringReader(input);
            nsf.reset();
            assertTrue(nsf.incrementToken());
            assertEquals("How_the", term.ToString());
        }
Ejemplo n.º 6
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public final boolean incrementToken() throws java.io.IOException
            public override bool incrementToken()
            {
                if (!input.incrementToken())
                {
                    return(false);
                }

                if (termAtt.ToString().Equals("dogs"))
                {
                    typeAtt.Type = "D";
                }
                return(true);
            }
Ejemplo n.º 7
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
            public override bool incrementToken()
            {
                if (input.incrementToken())
                {
                    if (!keywordAttr.Keyword)
                    {
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final String term = termAtt.toString().toLowerCase(java.util.Locale.ROOT);
                        string term = termAtt.ToString().ToLower(Locale.ROOT);
                        termAtt.setEmpty().append(term);
                    }
                    return(true);
                }
                return(false);
            }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: void assertTermEquals(String expected, org.apache.lucene.analysis.TokenStream stream, org.apache.lucene.analysis.tokenattributes.CharTermAttribute termAtt, org.apache.lucene.analysis.tokenattributes.PayloadAttribute payAtt, byte[] expectPay) throws Exception
        internal virtual void assertTermEquals(string expected, TokenStream stream, CharTermAttribute termAtt, PayloadAttribute payAtt, sbyte[] expectPay)
        {
            assertTrue(stream.incrementToken());
            assertEquals(expected, termAtt.ToString());
            BytesRef payload = payAtt.Payload;

            if (payload != null)
            {
                assertTrue(payload.length + " does not equal: " + expectPay.Length, payload.length == expectPay.Length);
                for (int i = 0; i < expectPay.Length; i++)
                {
                    assertTrue(expectPay[i] + " does not equal: " + payload.bytes[i + payload.offset], expectPay[i] == payload.bytes[i + payload.offset]);
                }
            }
            else
            {
                assertTrue("expectPay is not null and it should be", expectPay == null);
            }
        }
Ejemplo n.º 9
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void test() throws java.io.IOException
        public virtual void test()
        {
            TokenTypeSinkFilter sinkFilter = new TokenTypeSinkFilter("D");
            string test = "The quick red fox jumped over the lazy brown dogs";

            TeeSinkTokenFilter ttf = new TeeSinkTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)));

            TeeSinkTokenFilter.SinkTokenStream sink = ttf.newSinkTokenStream(sinkFilter);

            bool seenDogs = false;

            CharTermAttribute termAtt = ttf.addAttribute(typeof(CharTermAttribute));
            TypeAttribute     typeAtt = ttf.addAttribute(typeof(TypeAttribute));

            ttf.reset();
            while (ttf.incrementToken())
            {
                if (termAtt.ToString().Equals("dogs"))
                {
                    seenDogs = true;
                    assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().Equals("D") == true);
                }
                else
                {
                    assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().Equals("word"));
                }
            }
            assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);

            int sinkCount = 0;

            sink.reset();
            while (sink.incrementToken())
            {
                sinkCount++;
            }

            assertTrue("sink Size: " + sinkCount + " is not: " + 1, sinkCount == 1);
        }
Ejemplo n.º 10
0
        /// <returns>  Returns true for next token in the stream, or false at EOS </returns>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (input.incrementToken())
            {
                string term = termAtt.ToString();

                if (!keywordAttr.Keyword)
                {
                    string s = stemmer.stem(term);
                    // If not stemmed, don't waste the time adjusting the token.
                    if ((s != null) && !s.Equals(term))
                    {
                        termAtt.setEmpty().append(s);
                    }
                }
                return(true);
            }
            else
            {
                return(false);
            }
        }
Ejemplo n.º 11
0
 /// <summary>
 /// Pop one input token's worth of tokens off the filter and verify that they are as expected.
 /// </summary>
 //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
 //ORIGINAL LINE: void assertNextTerms(String expectedUnfolded, String expectedFolded, ASCIIFoldingFilter filter, org.apache.lucene.analysis.tokenattributes.CharTermAttribute termAtt) throws Exception
 internal virtual void assertNextTerms(string expectedUnfolded, string expectedFolded, ASCIIFoldingFilter filter, CharTermAttribute termAtt)
 {
     assertTrue(filter.incrementToken());
     assertEquals(expectedFolded, termAtt.ToString());
     if (filter.PreserveOriginal && !expectedUnfolded.Equals(expectedFolded))
     {
       assertTrue(filter.incrementToken());
       assertEquals(expectedUnfolded, termAtt.ToString());
     }
 }
        //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
        //ORIGINAL LINE: void assertTermEquals(String expected, org.apache.lucene.analysis.TokenStream stream, org.apache.lucene.analysis.tokenattributes.CharTermAttribute termAtt, org.apache.lucene.analysis.tokenattributes.PayloadAttribute payAtt, byte[] expectPay) throws Exception
        internal virtual void assertTermEquals(string expected, TokenStream stream, CharTermAttribute termAtt, PayloadAttribute payAtt, sbyte[] expectPay)
        {
            assertTrue(stream.incrementToken());
            assertEquals(expected, termAtt.ToString());
            BytesRef payload = payAtt.Payload;
            if (payload != null)
            {
              assertTrue(payload.length + " does not equal: " + expectPay.Length, payload.length == expectPay.Length);
              for (int i = 0; i < expectPay.Length; i++)
              {
            assertTrue(expectPay[i] + " does not equal: " + payload.bytes[i + payload.offset], expectPay[i] == payload.bytes[i + payload.offset]);

              }
            }
            else
            {
              assertTrue("expectPay is not null and it should be", expectPay == null);
            }
        }