public override bool IncrementToken()
{
if (mTokenEnumerator == null && base.input == null)
{
return(false);
}
if (mTokenEnumerator == null)
{
mTokenEnumerator = mTokenizer.EnumerateTokens(base.input).GetEnumerator();
}
// Fetch tokens from CodeXCavator tokenizer
bool next = mTokenEnumerator.MoveNext();
if (next)
{
ClearAttributes();
var token = mTokenEnumerator.Current;
// Set term attribute to token text.
mTermAttribute.SetTermBuffer(mCaseMode == Case.Sensitive ? token.Text : token.Text.ToLowerInvariant());
// Set offset attribute to token position
mOffsetAttribute.SetOffset(token.Position, token.Position + token.Length);
}
return(next);
}
