| protected Stem ( string term ) : string | ||
| term | string | java.langString The term that should be stemmed |
| return | string | |
/// <summary>
/// Returns the next token in the stream, or null at EOS
/// </summary>
/// <returns>
/// Returns the next token in the stream, or null at EOS
/// </returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return(null);
}
// Check the exclusiontable
else if (exclusions != null && exclusions.Contains(token.TermText()))
{
return(token);
}
else
{
String s = stemmer.Stem(token.TermText());
// If not stemmed, dont waste the time creating a new token
if (!s.Equals(token.TermText()))
{
return(new Token(s, 0, s.Length, token.Type()));
}
return(token);
}
}
/*
* @return Returns true for the next token in the stream, or false at EOS
*/
public override bool IncrementToken()
{
if (input.IncrementToken())
{
String term = termAtt.Term;
// Check the exclusion table
if (exclusions == null || !exclusions.Contains(term))
{
String s = stemmer.Stem(term);
// If not stemmed, don't waste the time adjusting the token.
if ((s != null) && !s.Equals(term))
{
termAtt.SetTermBuffer(s);
}
}
return(true);
}
else
{
return(false);
}
}
/// <returns> Returns true for the next token in the stream, or false at EOS </returns>
public override bool IncrementToken()
{
if (m_input.IncrementToken())
{
string term = termAtt.ToString();
// Check the exclusion table
if (!keywordAttr.IsKeyword)
{
string s = stemmer.Stem(term);
// If not stemmed, don't waste the time adjusting the token.
if ((s != null) && !s.Equals(term, StringComparison.Ordinal))
{
termAtt.SetEmpty().Append(s);
}
}
return(true);
}
else
{
return(false);
}
}