/// <returns> Returns the next token in the stream, or null at EOS
/// </returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return null;
}
else
{
System.String s = stemmer.Stem(token.TermText());
if (!s.Equals(token.TermText()))
{
return new Token(s, token.StartOffset(), token.EndOffset(), token.Type());
}
return token;
}
}
/// <returns> Returns the next token in the stream, or null at EOS
/// </returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return(null);
}
else
{
System.String s = stemmer.Stem(token.TermText());
if (!s.Equals(token.TermText()))
{
return(new Token(s, token.StartOffset(), token.EndOffset(), token.Type()));
}
return(token);
}
}
/// <returns> Returns the next token in the stream, or null at EOS
/// </returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return(null);
}
// Check the exclusiontable
else if (exclusionSet != null && exclusionSet.Contains(token.TermText()))
{
return(token);
}
else
{
System.String s = stemmer.Stem(token.TermText());
// If not stemmed, dont waste the time creating a new token
if (!s.Equals(token.TermText()))
{
return(new Token(s, token.StartOffset(), token.EndOffset(), token.Type()));
}
return(token);
}
}
/// <returns> Returns the next token in the stream, or null at EOS
/// </returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return null;
}
// Check the exclusiontable
else if (exclusionSet != null && exclusionSet.Contains(token.TermText()))
{
return token;
}
else
{
System.String s = stemmer.Stem(token.TermText());
// If not stemmed, dont waste the time creating a new token
if (!s.Equals(token.TermText()))
{
return new Token(s, token.StartOffset(), token.EndOffset(), token.Type());
}
return token;
}
}
