//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public final boolean incrementToken() throws java.io.IOException
            public override bool incrementToken()
            {
                bool isTokenAvailable = false;

                while (input.incrementToken())
                {
                    if (typeAtt.type() == UAX29URLEmailTokenizer.TOKEN_TYPES[UAX29URLEmailTokenizer.URL])
                    {
                        isTokenAvailable = true;
                        break;
                    }
                }
                return(isTokenAvailable);
            }
Exemplo n.º 2
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private void testPositons(TypeTokenFilter stpf) throws java.io.IOException
        private void testPositons(TypeTokenFilter stpf)
        {
            TypeAttribute              typeAtt       = stpf.getAttribute(typeof(TypeAttribute));
            CharTermAttribute          termAttribute = stpf.getAttribute(typeof(CharTermAttribute));
            PositionIncrementAttribute posIncrAtt    = stpf.getAttribute(typeof(PositionIncrementAttribute));

            stpf.reset();
            bool enablePositionIncrements = stpf.EnablePositionIncrements;

            while (stpf.incrementToken())
            {
                log("Token: " + termAttribute.ToString() + ": " + typeAtt.type() + " - " + posIncrAtt.PositionIncrement);
                assertEquals("if position increment is enabled the positionIncrementAttribute value should be 3, otherwise 1", posIncrAtt.PositionIncrement, enablePositionIncrements ? 3 : 1);
            }
            stpf.end();
            stpf.close();
        }
Exemplo n.º 3
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testFilterTokens() throws Exception
        public virtual void testFilterTokens()
        {
            SnowballFilter             filter     = new SnowballFilter(new TestTokenStream(this), "English");
            CharTermAttribute          termAtt    = filter.getAttribute(typeof(CharTermAttribute));
            OffsetAttribute            offsetAtt  = filter.getAttribute(typeof(OffsetAttribute));
            TypeAttribute              typeAtt    = filter.getAttribute(typeof(TypeAttribute));
            PayloadAttribute           payloadAtt = filter.getAttribute(typeof(PayloadAttribute));
            PositionIncrementAttribute posIncAtt  = filter.getAttribute(typeof(PositionIncrementAttribute));
            FlagsAttribute             flagsAtt   = filter.getAttribute(typeof(FlagsAttribute));

            filter.incrementToken();

            assertEquals("accent", termAtt.ToString());
            assertEquals(2, offsetAtt.startOffset());
            assertEquals(7, offsetAtt.endOffset());
            assertEquals("wrd", typeAtt.type());
            assertEquals(3, posIncAtt.PositionIncrement);
            assertEquals(77, flagsAtt.Flags);
            assertEquals(new BytesRef(new sbyte[] { 0, 1, 2, 3 }), payloadAtt.Payload);
        }
        /// <summary>
        /// Output bigrams whenever possible to optimize queries. Only output unigrams
        /// when they are not a member of a bigram. Example:
        /// <ul>
        /// <li>input: "the rain in spain falls mainly"
        /// <li>output:"the-rain", "rain-in" ,"in-spain", "falls", "mainly"
        /// </ul>
        /// </summary>
        public override bool IncrementToken()
        {
            while (!exhausted && input.incrementToken())
            {
                State current = CaptureState();

                if (previous != null && !GramType)
                {
                    RestoreState(previous);
                    previous     = current;
                    previousType = typeAttribute.type();

                    if (GramType)
                    {
                        posIncAttribute.PositionIncrement = 1;
                    }
                    return(true);
                }

                previous = current;
            }

            exhausted = true;

            if (previous == null || GRAM_TYPE.Equals(previousType))
            {
                return(false);
            }

            RestoreState(previous);
            previous = null;

            if (GramType)
            {
                posIncAttribute.PositionIncrement = 1;
            }
            return(true);
        }
Exemplo n.º 5
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public final boolean incrementTokenClassic() throws java.io.IOException
	  public bool incrementTokenClassic()
	  {
		if (!input.incrementToken())
		{
		  return false;
		}

//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final char[] buffer = termAtt.buffer();
		char[] buffer = termAtt.buffer();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int bufferLength = termAtt.length();
		int bufferLength = termAtt.length();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final String type = typeAtt.type();
		string type = typeAtt.type();

		if (type == APOSTROPHE_TYPE && bufferLength >= 2 && buffer[bufferLength - 2] == '\'' && (buffer[bufferLength - 1] == 's' || buffer[bufferLength - 1] == 'S')) // remove 's
		{
		  // Strip last 2 characters off
		  termAtt.Length = bufferLength - 2;
		} // remove dots
		else if (type == ACRONYM_TYPE)
		{
		  int upto = 0;
		  for (int i = 0;i < bufferLength;i++)
		  {
			char c = buffer[i];
			if (c != '.')
			{
			  buffer[upto++] = c;
			}
		  }
		  termAtt.Length = upto;
		}

		return true;
	  }