| private Stem ( String term ) : String | ||
| term | String | The term that should be stemmed. |
| return | String | |
/// <returns> Returns true for next token in the stream, or false at EOS </returns>
public override bool IncrementToken()
{
if (m_input.IncrementToken())
{
string term = termAtt.ToString();
if (!keywordAttr.IsKeyword)
{
string s = stemmer.Stem(term);
// If not stemmed, don't waste the time adjusting the token.
if ((s != null) && !s.Equals(term, StringComparison.Ordinal))
{
termAtt.SetEmpty().Append(s);
}
}
return(true);
}
else
{
return(false);
}
}
/// <returns>
/// Returns true for next token in the stream, or false at EOS
/// </returns>
public override bool IncrementToken()
{
if (input.IncrementToken())
{
String term = termAtt.Term;
// Check the exclusion table.
if (exclusionSet == null || !exclusionSet.Contains(term))
{
String s = stemmer.Stem(term);
// If not stemmed, don't waste the time adjusting the token.
if ((s != null) && !s.Equals(term))
{
termAtt.SetTermBuffer(s);
}
}
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// </summary>
/// <returns>Returns the next token in the stream, or null at EOS</returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return(null);
}
// Check the exclusiontable
else if (exclusions != null && exclusions.Contains(token.TermText()))
{
return(token);
}
else
{
String s = stemmer.Stem(token.TermText());
// If not stemmed, dont waste the time creating a new token
if (!s.Equals(token.TermText()))
{
return(new Token(s, token.StartOffset(),
token.EndOffset(), token.Type()));
}
return(token);
}
}