//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (input.incrementToken())
            {
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final char[] buffer = termAtt.buffer();
                char[] buffer = termAtt.buffer();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int length = termAtt.length();
                int length = termAtt.length();
                for (int i = 0; i < length; i++)
                {
                    if (buffer[i] == delimiter)
                    {
                        payAtt.Payload = encoder.encode(buffer, i + 1, (length - (i + 1)));
                        termAtt.Length = i;   // simply set a new length
                        return(true);
                    }
                }
                // we have not seen the delimiter
                payAtt.Payload = null;
                return(true);
            }
            else
            {
                return(false);
            }
        }
Ejemplo n.º 2
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (input.incrementToken())
            {
                char[] chArray = termAtt.buffer();
                int    chLen   = termAtt.length();
                int    idx     = 0;

                if (chLen > 1 && (chArray[0] == 'n' || chArray[0] == 't') && isUpperVowel(chArray[1]))
                {
                    chArray = termAtt.resizeBuffer(chLen + 1);
                    for (int i = chLen; i > 1; i--)
                    {
                        chArray[i] = chArray[i - 1];
                    }
                    chArray[1]     = '-';
                    termAtt.Length = chLen + 1;
                    idx            = 2;
                    chLen          = chLen + 1;
                }

                for (int i = idx; i < chLen;)
                {
                    i += char.toChars(char.ToLower(chArray[i]), chArray, i);
                }
                return(true);
            }
            else
            {
                return(false);
            }
        }
Ejemplo n.º 3
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (input.incrementToken())
            {
                int newlen = normalizer.normalize(termAtt.buffer(), termAtt.length());
                termAtt.Length = newlen;
                return(true);
            }
            return(false);
        }
Ejemplo n.º 4
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (input.incrementToken())
            {
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int newlen = normalizer.normalize(termAtt.buffer(), termAtt.length());
                int newlen = normalizer.normalize(termAtt.buffer(), termAtt.length());
                termAtt.Length = newlen;
                return(true);
            }
            return(false);
        }
Ejemplo n.º 5
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (input.incrementToken())
            {
                if (!keywordAtt.Keyword)
                {
                    termAtt.Length = normalizer.normalize(termAtt.buffer(), termAtt.length());
                }
                return(true);
            }
            return(false);
        }
Ejemplo n.º 6
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
            public override bool incrementToken()
            {
                if (input.incrementToken())
                {
                    typeAtt.Type = char.ToUpper(termAtt.buffer()[0]).ToString();
                    return(true);
                }
                else
                {
                    return(false);
                }
            }
Ejemplo n.º 7
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public final boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (!input.incrementToken())
            {
                return(false);
            }

            if ((!keywordAttr.Keyword) && stemmer.stem(termAtt.buffer(), 0, termAtt.length()))
            {
                termAtt.copyBuffer(stemmer.ResultBuffer, 0, stemmer.ResultLength);
            }
            return(true);
        }
Ejemplo n.º 8
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private org.apache.lucene.analysis.Token getNextPrefixInputToken(org.apache.lucene.analysis.Token token) throws java.io.IOException
        private Token getNextPrefixInputToken(Token token)
        {
            if (!prefix.incrementToken())
            {
                return(null);
            }
            token.copyBuffer(p_termAtt.buffer(), 0, p_termAtt.length());
            token.PositionIncrement = p_posIncrAtt.PositionIncrement;
            token.Flags             = p_flagsAtt.Flags;
            token.setOffset(p_offsetAtt.startOffset(), p_offsetAtt.endOffset());
            token.Type    = p_typeAtt.type();
            token.Payload = p_payloadAtt.Payload;
            return(token);
        }
Ejemplo n.º 9
0
        /// <summary>
        /// refills buffers with new data from the current token.
        /// </summary>
        private void refill()
        {
            // compact buffers to keep them smallish if they become large
            // just a safety check, but technically we only need the last codepoint
            if (bufferLen > 64)
            {
                int last = bufferLen - 1;
                buffer[0]      = buffer[last];
                startOffset[0] = startOffset[last];
                endOffset[0]   = endOffset[last];
                bufferLen      = 1;
                index         -= last;
            }

            char[] termBuffer = termAtt.buffer();
            int    len        = termAtt.length();
            int    start      = offsetAtt.startOffset();
            int    end        = offsetAtt.endOffset();

            int newSize = bufferLen + len;

            buffer        = ArrayUtil.grow(buffer, newSize);
            startOffset   = ArrayUtil.grow(startOffset, newSize);
            endOffset     = ArrayUtil.grow(endOffset, newSize);
            lastEndOffset = end;

            if (end - start != len)
            {
                // crazy offsets (modified by synonym or charfilter): just preserve
                for (int i = 0, cp = 0; i < len; i += char.charCount(cp))
                {
                    cp = buffer[bufferLen] = char.codePointAt(termBuffer, i, len);
                    startOffset[bufferLen] = start;
                    endOffset[bufferLen]   = end;
                    bufferLen++;
                }
            }
            else
            {
                // normal offsets
                for (int i = 0, cp = 0, cpLen = 0; i < len; i += cpLen)
                {
                    cp    = buffer[bufferLen] = char.codePointAt(termBuffer, i, len);
                    cpLen = char.charCount(cp);
                    startOffset[bufferLen] = start;
                    start = endOffset[bufferLen] = start + cpLen;
                    bufferLen++;
                }
            }
        }
Ejemplo n.º 10
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private org.apache.lucene.analysis.Token getNextSuffixInputToken(org.apache.lucene.analysis.Token token) throws java.io.IOException
        private Token getNextSuffixInputToken(Token token)
        {
            if (!suffix.incrementToken())
            {
                return(null);
            }
            token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
            token.PositionIncrement = posIncrAtt.PositionIncrement;
            token.Flags             = flagsAtt.Flags;
            token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
            token.Type    = typeAtt.type();
            token.Payload = payloadAtt.Payload;
            return(token);
        }
Ejemplo n.º 11
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (input.incrementToken())
            {
                if (!keywordAtt.Keyword)
                {
                    termAtt.Length = stemmer.stem(termAtt.buffer(), termAtt.length());
                }
                return(true);
            }
            else
            {
                return(false);
            }
        }
Ejemplo n.º 12
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (input.incrementToken())
            {
                char[] chArray = termAtt.buffer();
                int    chLen   = termAtt.length();
                for (int i = 0; i < chLen;)
                {
                    i += char.toChars(lowerCase(charUtils.codePointAt(chArray, i, chLen)), chArray, i);
                }
                return(true);
            }
            else
            {
                return(false);
            }
        }
Ejemplo n.º 13
0
        /// <summary>
        /// Returns the next, stemmed, input Token. </summary>
        ///  <returns> The stemmed form of a token. </returns>
        ///  <exception cref="IOException"> If there is a low-level I/O error. </exception>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (!input.incrementToken())
            {
                return(false);
            }

            char[] term = termAttribute.buffer();
            int    len  = termAttribute.length();

            if ((!keywordAtt.Keyword) && stemmer.stem(term, len))
            {
                termAttribute.setEmpty().append(stemmer.asCharSequence());
            }

            return(true);
        }
Ejemplo n.º 14
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (input.incrementToken())
            {
                if (!keywordAttr.Keyword)
                {
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());
                    int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());
                    termAtt.Length = newlen;
                }
                return(true);
            }
            else
            {
                return(false);
            }
        }
Ejemplo n.º 15
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public final boolean incrementTokenClassic() throws java.io.IOException
        public bool incrementTokenClassic()
        {
            if (!input.incrementToken())
            {
                return(false);
            }

//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final char[] buffer = termAtt.buffer();
            char[] buffer = termAtt.buffer();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int bufferLength = termAtt.length();
            int bufferLength = termAtt.length();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final String type = typeAtt.type();
            string type = typeAtt.type();

            if (type == APOSTROPHE_TYPE && bufferLength >= 2 && buffer[bufferLength - 2] == '\'' && (buffer[bufferLength - 1] == 's' || buffer[bufferLength - 1] == 'S'))     // remove 's
            {
                // Strip last 2 characters off
                termAtt.Length = bufferLength - 2;
            }     // remove dots
            else if (type == ACRONYM_TYPE)
            {
                int upto = 0;
                for (int i = 0; i < bufferLength; i++)
                {
                    char c = buffer[i];
                    if (c != '.')
                    {
                        buffer[upto++] = c;
                    }
                }
                termAtt.Length = upto;
            }

            return(true);
        }
Ejemplo n.º 16
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (input.incrementToken())
            {
                char[] text   = termAtt.buffer();
                int    length = termAtt.length();
                for (int i = 0; i < length; i++)
                {
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final char ch = text[i];
                    char ch = text[i];
                    if (ch >= 0xFF01 && ch <= 0xFF5E)
                    {
                        // Fullwidth ASCII variants
                        text[i] -= 0xFEE0;
                    }
                    else if (ch >= 0xFF65 && ch <= 0xFF9F)
                    {
                        // Halfwidth Katakana variants
                        if ((ch == 0xFF9E || ch == 0xFF9F) && i > 0 && combine(text, i, ch))
                        {
                            length = StemmerUtil.delete(text, i--, length);
                        }
                        else
                        {
                            text[i] = KANA_NORM[ch - 0xFF65];
                        }
                    }
                }
                termAtt.Length = length;
                return(true);
            }
            else
            {
                return(false);
            }
        }
Ejemplo n.º 17
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public final boolean incrementToken() throws java.io.IOException
        public override bool incrementToken()
        {
            if (!input.incrementToken())
            {
                return(false);
            }

//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final char[] buffer = termAtt.buffer();
            char[] buffer = termAtt.buffer();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int length = termAtt.length();
            int length = termAtt.length();

            for (int i = 0; i < length; i++)
            {
                if (buffer[i] == '\'' || buffer[i] == '\u2019')
                {
                    termAtt.Length = i;
                    return(true);
                }
            }
            return(true);
        }
Ejemplo n.º 18
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void test() throws java.io.IOException
        public virtual void test()
        {
            string test = "The quick red fox jumped over the lazy brown dogs";

            TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(this, new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)));
            int count = 0;
            CharTermAttribute termAtt    = nptf.getAttribute(typeof(CharTermAttribute));
            TypeAttribute     typeAtt    = nptf.getAttribute(typeof(TypeAttribute));
            PayloadAttribute  payloadAtt = nptf.getAttribute(typeof(PayloadAttribute));

            nptf.reset();
            while (nptf.incrementToken())
            {
                assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().Equals(char.ToUpper(termAtt.buffer()[0]).ToString()));
                assertTrue("nextToken.getPayload() is null and it shouldn't be", payloadAtt.Payload != null);
                string type = payloadAtt.Payload.utf8ToString();
                assertTrue(type + " is not equal to " + typeAtt.type(), type.Equals(typeAtt.type()));
                count++;
            }

            assertTrue(count + " does not equal: " + 10, count == 10);
        }