public override bool IncrementToken() { if (mTokenEnumerator == null && base.input == null) { return(false); } if (mTokenEnumerator == null) { mTokenEnumerator = mTokenizer.EnumerateTokens(base.input).GetEnumerator(); } // Fetch tokens from CodeXCavator tokenizer bool next = mTokenEnumerator.MoveNext(); if (next) { ClearAttributes(); var token = mTokenEnumerator.Current; // Set term attribute to token text. mTermAttribute.SetTermBuffer(mCaseMode == Case.Sensitive ? token.Text : token.Text.ToLowerInvariant()); // Set offset attribute to token position mOffsetAttribute.SetOffset(token.Position, token.Position + token.Length); } return(next); }