/* * Returns the next token in the stream, or null at EOS */ public override bool IncrementToken() { if (input.IncrementToken()) { String term = termAtt.Term; // Check the exclusion table. if (exclusions == null || !exclusions.Contains(term)) { String s = stemmer.Stem(term); // If not stemmed, don't waste the time adjusting the token. if ((s != null) && !s.Equals(term)) { termAtt.SetTermBuffer(s); } } return(true); } else { return(false); } }
/// <summary> /// Returns the next token in the stream, or null at EOS /// </summary> public override bool IncrementToken() { if (m_input.IncrementToken()) { string term = termAtt.ToString(); // Check the exclusion table. if (!keywordAttr.IsKeyword) { string s = stemmer.Stem(term); // If not stemmed, don't waste the time adjusting the token. if ((s != null) && !s.Equals(term)) { termAtt.SetEmpty().Append(s); } } return(true); } else { return(false); } }
/// <summary> /// </summary> /// <returns>Returns the next token in the stream, or null at EOS</returns> public override Token Next() { if ((token = input.Next()) == null) { return(null); } // Check the exclusiontable else if (exclusions != null && exclusions.Contains(token.TermText())) { return(token); } else { String s = stemmer.Stem(token.TermText()); // If not stemmed, dont waste the time creating a new token if (!s.Equals(token.TermText())) { return(new Token(s, token.StartOffset(), token.EndOffset(), token.Type())); } return(token); } }