コード例 #1
0
 /// <returns>  Returns the next token in the stream, or null at EOS
 /// </returns>
 public override Token Next()
 {
     if ((token = input.Next()) == null)
     {
         return(null);
     }
     else
     {
         System.String s = stemmer.Stem(token.TermText());
         if (!s.Equals(token.TermText()))
         {
             return(new Token(s, token.StartOffset(), token.EndOffset(), token.Type()));
         }
         return(token);
     }
 }
コード例 #2
0
		/// <returns>  Returns the next token in the stream, or null at EOS
		/// </returns>
		public override Token Next()
		{
			if ((token = input.Next()) == null)
			{
				return null;
			}
			else
			{
				System.String s = stemmer.Stem(token.TermText());
				if (!s.Equals(token.TermText()))
				{
					return new Token(s, token.StartOffset(), token.EndOffset(), token.Type());
				}
				return token;
			}
		}
コード例 #3
0
 /// <returns>  Returns the next token in the stream, or null at EOS
 /// </returns>
 public override Token Next()
 {
     if ((token = input.Next()) == null)
     {
         return(null);
     }
     // Check the exclusiontable
     else if (exclusionSet != null && exclusionSet.Contains(token.TermText()))
     {
         return(token);
     }
     else
     {
         System.String s = stemmer.Stem(token.TermText());
         // If not stemmed, dont waste the time creating a new token
         if (!s.Equals(token.TermText()))
         {
             return(new Token(s, token.StartOffset(), token.EndOffset(), token.Type()));
         }
         return(token);
     }
 }
コード例 #4
0
ファイル: GermanStemFilter.cs プロジェクト: emtees/old-code
		/// <returns>  Returns the next token in the stream, or null at EOS
		/// </returns>
		public override Token Next()
		{
			if ((token = input.Next()) == null)
			{
				return null;
			}
			// Check the exclusiontable
			else if (exclusionSet != null && exclusionSet.Contains(token.TermText()))
			{
				return token;
			}
			else
			{
				System.String s = stemmer.Stem(token.TermText());
				// If not stemmed, dont waste the time creating a new token
				if (!s.Equals(token.TermText()))
				{
					return new Token(s, token.StartOffset(), token.EndOffset(), token.Type());
				}
				return token;
			}
		}
コード例 #5
0
        public override Token Next()
        {
            Token t = input.Next();

            if (t == null)
            {
                return(null);
            }

            System.String txt = t.TermText();

            char[] chArray = txt.ToCharArray();
            for (int i = 0; i < chArray.Length; i++)
            {
                chArray[i] = RussianCharsets.ToLowerCase(chArray[i], charset);
            }

            System.String newTxt = new System.String(chArray);
            // create new token
            Token newToken = new Token(newTxt, t.StartOffset(), t.EndOffset());

            return(newToken);
        }