public static IEnumerable <Token <LexerTokenType> > LexTokens(Position start, ITextProvider textProvider)
{
System.Diagnostics.Debug.Assert(textProvider != null);
Position pos = start;
string text;
Range range;
// So that the caller's implementation of 'tryGetMoreText' doesn't have to buffer and be
// smart about token boundaries, we detect when we're about to yield the last token
// in the current buffer, and instead try to get more text and re-lex.
Token <LexerTokenType> lastToken = null;
while (textProvider.TryGetText(pos, out text, out range))
{
if (lastToken != null)
{
text = string.Concat(lastToken.Value, text);
range = new Range(lastToken.Range.Start, range.End);
}
foreach (Token <LexerTokenType> token in RtypeLexer.LexTokens(text, range.Start))
{
if (token.Range.End.Offset != range.End.Offset)
{
lastToken = null;
yield return(token);
}
else
{
lastToken = token;
}
}
pos = range.End;
}
// If we fell off the end with a token, make sure the caller gets it!
if (lastToken != null)
{
yield return(lastToken);
}
}
// Assumes that 'start' is a reasonable place from which to start parsing... that is,
// it's not the second physical line in a logical continuation line.
public static IEnumerable <Statement <StatementType, ParserTokenType> > ParseStatements(Position start, ITextProvider textProvider)
{
TokenBuffer <LexerTokenType> tokenBuffer = new TokenBuffer <LexerTokenType>(RtypeLexer.LexTokens(start, textProvider));
RTypeTokenInfo parserTokenInfo = new RTypeTokenInfo();
while (tokenBuffer.CurrentToken != null)
{
var statement = new StatementBuilder <StatementType, ParserTokenType, LexerTokenType>(tokenBuffer, parserTokenInfo);
// Many statements can start with leading whitespace, we save that off first so that
// the switch below is easier...
Token <LexerTokenType> leadingWhitespace;
tokenBuffer.Accept(LexerTokenType.Whitespace, out leadingWhitespace);
switch (tokenBuffer.CurrentTokenType)
{
case LexerTokenType.Newline:
case LexerTokenType.Hash: // comment
statement.As(StatementType.Ignorable, ParserTokenType.Whitespace, leadingWhitespace)
.ReadToEndOfLine();
break;
case LexerTokenType.UseKeyword:
// Should we disallow leading whitespace for the use statement?
statement.As(StatementType.Use, ParserTokenType.Whitespace, leadingWhitespace)
.Expect(LexerTokenType.UseKeyword, ParserTokenType.UseKeyword)
.Accept(LexerTokenType.Whitespace, ParserTokenType.Whitespace)
.Expect(LexerTokenType.Identifier, ParserTokenType.NamespacePrefixDeclaration)
.ReadToEndOfLine();
break;
case LexerTokenType.Identifier:
// A leading identifier indicates a new message
statement.As(StatementType.Object, ParserTokenType.Whitespace, leadingWhitespace)
.ExpectRtypeObjectName();
// Now, we allow "name=value" and comments and newlines... we have to be smart about
// understanding when it's a continuation line and when it's a new object. We do this by
// looking for an "equals" after the identifier.
bool keepConsuming = true;
bool beginningOfLine = false;
bool canHaveDefaultPropertyValue = true;
while (keepConsuming && tokenBuffer.CurrentToken != null)
{
if (beginningOfLine)
{
// Temporarily eat any whitespace so we can check the text...
tokenBuffer.Accept(LexerTokenType.Whitespace, out leadingWhitespace);
var looksLikeProperty = tokenBuffer.LooksLikePropertyAssignment();
if (leadingWhitespace != null)
{
tokenBuffer.PushBack(leadingWhitespace);
}
if (!looksLikeProperty)
{
keepConsuming = false;
break;
}
}
beginningOfLine = false;
// Because whitespace, comments, and newlines can happen often, it's easier
// to go ahead and account for them first...
statement.AcceptTrailingComment();
if (tokenBuffer.Is(LexerTokenType.Newline))
{
statement.Expect(LexerTokenType.Newline, ParserTokenType.Whitespace);
beginningOfLine = true;
canHaveDefaultPropertyValue = false;
continue;
}
// If it's allowable to have the default property value, we need to check to see if the token after this
// one is an equals or not... if it's not, we assume this is an (optionally) quoted value.
if (tokenBuffer.CurrentToken != null && canHaveDefaultPropertyValue && !tokenBuffer.LooksLikePropertyAssignment())
{
statement.ExpectRtypePropertyValue(ParserTokenType.PropertyValue);
canHaveDefaultPropertyValue = false;
continue;
}
while (!statement.HasError && tokenBuffer.CurrentToken != null && !tokenBuffer.Is(LexerTokenType.Newline))
{
// Read a "Name=Value" pair...
statement.ExpectRtypePropertyName()
.Expect(LexerTokenType.Equals, ParserTokenType.Equals)
.ExpectRtypePropertyValue(ParserTokenType.PropertyValue)
.AcceptTrailingComment();
}
if (statement.HasError)
{
statement.ReadToEndOfLine();
keepConsuming = false;
}
}
break;
default:
// Unknown, unexpected token!
statement.As(StatementType.Unknown, ParserTokenType.Whitespace, leadingWhitespace)
.ReadToEndOfLine();
break;
}
yield return(statement.ToStatement());
}
}
