public static IEnumerable <Token <LexerTokenType> > LexTokens(Position start, ITextProvider textProvider) { System.Diagnostics.Debug.Assert(textProvider != null); Position pos = start; string text; Range range; // So that the caller's implementation of 'tryGetMoreText' doesn't have to buffer and be // smart about token boundaries, we detect when we're about to yield the last token // in the current buffer, and instead try to get more text and re-lex. Token <LexerTokenType> lastToken = null; while (textProvider.TryGetText(pos, out text, out range)) { if (lastToken != null) { text = string.Concat(lastToken.Value, text); range = new Range(lastToken.Range.Start, range.End); } foreach (Token <LexerTokenType> token in RtypeLexer.LexTokens(text, range.Start)) { if (token.Range.End.Offset != range.End.Offset) { lastToken = null; yield return(token); } else { lastToken = token; } } pos = range.End; } // If we fell off the end with a token, make sure the caller gets it! if (lastToken != null) { yield return(lastToken); } }