public IEnumerable <Token> AllTokens()
{
var thisLexer = this;
var nextToken = default(Token);
while (!thisLexer.Eof)
{
(nextToken, thisLexer) = LexerEngine.ReadToken(thisLexer);
yield return(nextToken);
}
}
public static (Token Token, Lexer NextLexer) ReadToken(Lexer lexer)
{
var reader = lexer.Reader;
var peek = reader.Peek();
switch (peek.Value)
{
case '$':
return(LexerEngine.ReadAliasIdentifierToken(reader));
case ']':
return(LexerEngine.ReadAttributeCloseToken(reader));
case '[':
return(LexerEngine.ReadAttributeOpenToken(reader));
case '}':
return(LexerEngine.ReadBlockCloseToken(reader));
case '{':
return(LexerEngine.ReadBlockOpenToken(reader));
case ':':
return(LexerEngine.ReadColonToken(reader));
case ',':
return(LexerEngine.ReadCommaToken(reader));
case '/':
return(LexerEngine.ReadCommentToken(reader));
case '=':
return(LexerEngine.ReadEqualsOperatorToken(reader));
case ')':
return(LexerEngine.ReadParenthesisCloseToken(reader));
case '(':
return(LexerEngine.ReadParenthesisOpenToken(reader));
case '#':
return(LexerEngine.ReadPragmaToken(reader));
case ';':
return(LexerEngine.ReadStatementEndToken(reader));
case '"':
return(LexerEngine.ReadStringLiteralToken(reader));
case '+':
case '-':
return(LexerEngine.ReadNumericLiteralToken(reader));
case '.':
// if the next character is a decimalDigit then we're reading a RealLiteralToken with
// no leading digits before the decimal point (e.g. ".45"), otherwise we're reading
// a DotOperatorToken (e.g. the "." in "MyPropertyValue = MyEnum.Value;")
var readAhead = reader.Read().NextReader;
if (!readAhead.Eof() && StringValidator.IsDecimalDigit(readAhead.Peek().Value))
{
return(LexerEngine.ReadNumericLiteralToken(reader));
}
else
{
return(LexerEngine.ReadDotOperatorToken(reader));
}
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
case 'G':
case 'H':
case 'I':
case 'J':
case 'K':
case 'L':
case 'M':
case 'N':
case 'O':
case 'P':
case 'Q':
case 'R':
case 'S':
case 'T':
case 'U':
case 'V':
case 'W':
case 'X':
case 'Y':
case 'Z':
case '_':
// firstIdentifierChar
var(identifierToken, nextLexer) = LexerEngine.ReadIdentifierToken(reader);
var normalized = identifierToken.Name.ToLowerInvariant();
switch (normalized)
{
case "false":
var falseToken = new BooleanLiteralToken(identifierToken.Extent, false);
return(falseToken, nextLexer);
case "true":
var trueToken = new BooleanLiteralToken(identifierToken.Extent, true);
return(trueToken, nextLexer);
case "null":
var nullLiteralToken = new NullLiteralToken(identifierToken.Extent);
return(nullLiteralToken, nextLexer);
default:
return(identifierToken, nextLexer);
}
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
// decimalDigit
return(LexerEngine.ReadNumericLiteralToken(reader));
case '\u0020': // space
case '\u0009': // horizontal tab
case '\u000D': // carriage return
case '\u000A': // line feed
// WS
return(LexerEngine.ReadWhitespaceToken(reader));
default:
throw new UnexpectedCharacterException(peek);
}
}
