/*
* Returns the next token in the stream, or null at EOS
*/
public override bool IncrementToken()
{
if (input.IncrementToken())
{
String term = termAtt.Term;
// Check the exclusion table.
if (exclusions == null || !exclusions.Contains(term))
{
String s = stemmer.Stem(term);
// If not stemmed, don't waste the time adjusting the token.
if ((s != null) && !s.Equals(term))
{
termAtt.SetTermBuffer(s);
}
}
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// Returns the next token in the stream, or null at EOS
/// </summary>
public override bool IncrementToken()
{
if (m_input.IncrementToken())
{
string term = termAtt.ToString();
// Check the exclusion table.
if (!keywordAttr.IsKeyword)
{
string s = stemmer.Stem(term);
// If not stemmed, don't waste the time adjusting the token.
if ((s != null) && !s.Equals(term))
{
termAtt.SetEmpty().Append(s);
}
}
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// </summary>
/// <returns>Returns the next token in the stream, or null at EOS</returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return(null);
}
// Check the exclusiontable
else if (exclusions != null && exclusions.Contains(token.TermText()))
{
return(token);
}
else
{
String s = stemmer.Stem(token.TermText());
// If not stemmed, dont waste the time creating a new token
if (!s.Equals(token.TermText()))
{
return(new Token(s, token.StartOffset(),
token.EndOffset(), token.Type()));
}
return(token);
}
}
