public static IEnumerable <Token <LexerTokenType> > LexTokens(Position start, ITextProvider textProvider)
{
System.Diagnostics.Debug.Assert(textProvider != null);
Position pos = start;
string text;
Range range;
// So that the caller's implementation of 'tryGetMoreText' doesn't have to buffer and be
// smart about token boundaries, we detect when we're about to yield the last token
// in the current buffer, and instead try to get more text and re-lex.
Token <LexerTokenType> lastToken = null;
while (textProvider.TryGetText(pos, out text, out range))
{
if (lastToken != null)
{
text = string.Concat(lastToken.Value, text);
range = new Range(lastToken.Range.Start, range.End);
}
foreach (Token <LexerTokenType> token in RtypeLexer.LexTokens(text, range.Start))
{
if (token.Range.End.Offset != range.End.Offset)
{
lastToken = null;
yield return(token);
}
else
{
lastToken = token;
}
}
pos = range.End;
}
// If we fell off the end with a token, make sure the caller gets it!
if (lastToken != null)
{
yield return(lastToken);
}
}
