/** * @return Returns the next token in the stream, or null at EOS. */ public override Token Next(Token reusableToken) { System.Diagnostics.Trace.Assert(reusableToken != null); Token nextToken = input.Next(reusableToken); if (nextToken == null) { return(null); } string term = nextToken.TermText(); // Check the exclusion table. if (exclusions == null || !exclusions.Contains(term)) { string s = stemmer.Stem(term); // If not stemmed, don't waste the time adjusting the token. if ((s != null) && !s.Equals(term)) { nextToken.SetTermBuffer(s.ToCharArray(), 0, s.Length);//was SetTermBuffer(s) } } return(nextToken); }
public override bool IncrementToken() { if (m_input.IncrementToken()) { string term = termAtt.ToString(); // Check the exclusion table. if (!keywordAttr.IsKeyword /*&& (exclusions == null || !exclusions.Contains(term))*/) // LUCENENET specific - removed unused variable "exclusions" { string s = stemmer.Stem(term); // If not stemmed, don't waste the time adjusting the token. if ((s != null) && !s.Equals(term, StringComparison.Ordinal)) { termAtt.SetEmpty().Append(s); } } return(true); } else { return(false); } }
public override bool IncrementToken() { if (input.IncrementToken()) { string term = termAtt.ToString(); // Check the exclusion table. if (!keywordAttr.Keyword && (exclusions == null || !exclusions.Contains(term))) { string s = stemmer.Stem(term); // If not stemmed, don't waste the time adjusting the token. if ((s != null) && !s.Equals(term)) { termAtt.SetEmpty().Append(s); } } return(true); } else { return(false); } }