/// <returns> Returns the next token in the stream, or null at EOS
/// </returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return(null);
}
else
{
System.String s = stemmer.Stem(token.TermText());
if (!s.Equals(token.TermText()))
{
return(new Token(s, token.StartOffset(), token.EndOffset(), token.Type()));
}
return(token);
}
}
/// <returns> Returns the next token in the stream, or null at EOS
/// </returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return null;
}
else
{
System.String s = stemmer.Stem(token.TermText());
if (!s.Equals(token.TermText()))
{
return new Token(s, token.StartOffset(), token.EndOffset(), token.Type());
}
return token;
}
}
/// <returns> Returns the next token in the stream, or null at EOS
/// </returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return(null);
}
// Check the exclusiontable
else if (exclusionSet != null && exclusionSet.Contains(token.TermText()))
{
return(token);
}
else
{
System.String s = stemmer.Stem(token.TermText());
// If not stemmed, dont waste the time creating a new token
if (!s.Equals(token.TermText()))
{
return(new Token(s, token.StartOffset(), token.EndOffset(), token.Type()));
}
return(token);
}
}
/// <returns> Returns the next token in the stream, or null at EOS
/// </returns>
public override Token Next()
{
if ((token = input.Next()) == null)
{
return null;
}
// Check the exclusiontable
else if (exclusionSet != null && exclusionSet.Contains(token.TermText()))
{
return token;
}
else
{
System.String s = stemmer.Stem(token.TermText());
// If not stemmed, dont waste the time creating a new token
if (!s.Equals(token.TermText()))
{
return new Token(s, token.StartOffset(), token.EndOffset(), token.Type());
}
return token;
}
}
public override Token Next()
{
Token t = input.Next();
if (t == null)
{
return(null);
}
System.String txt = t.TermText();
char[] chArray = txt.ToCharArray();
for (int i = 0; i < chArray.Length; i++)
{
chArray[i] = RussianCharsets.ToLowerCase(chArray[i], charset);
}
System.String newTxt = new System.String(chArray);
// create new token
Token newToken = new Token(newTxt, t.StartOffset(), t.EndOffset());
return(newToken);
}