public IEnumerable <Token> AllTokens() { var thisLexer = this; var nextToken = default(Token); while (!thisLexer.Eof) { (nextToken, thisLexer) = LexerEngine.ReadToken(thisLexer); yield return(nextToken); } }
public static (Token Token, Lexer NextLexer) ReadToken(Lexer lexer) { var reader = lexer.Reader; var peek = reader.Peek(); switch (peek.Value) { case '$': return(LexerEngine.ReadAliasIdentifierToken(reader)); case ']': return(LexerEngine.ReadAttributeCloseToken(reader)); case '[': return(LexerEngine.ReadAttributeOpenToken(reader)); case '}': return(LexerEngine.ReadBlockCloseToken(reader)); case '{': return(LexerEngine.ReadBlockOpenToken(reader)); case ':': return(LexerEngine.ReadColonToken(reader)); case ',': return(LexerEngine.ReadCommaToken(reader)); case '/': return(LexerEngine.ReadCommentToken(reader)); case '=': return(LexerEngine.ReadEqualsOperatorToken(reader)); case ')': return(LexerEngine.ReadParenthesisCloseToken(reader)); case '(': return(LexerEngine.ReadParenthesisOpenToken(reader)); case '#': return(LexerEngine.ReadPragmaToken(reader)); case ';': return(LexerEngine.ReadStatementEndToken(reader)); case '"': return(LexerEngine.ReadStringLiteralToken(reader)); case '+': case '-': return(LexerEngine.ReadNumericLiteralToken(reader)); case '.': // if the next character is a decimalDigit then we're reading a RealLiteralToken with // no leading digits before the decimal point (e.g. ".45"), otherwise we're reading // a DotOperatorToken (e.g. the "." in "MyPropertyValue = MyEnum.Value;") var readAhead = reader.Read().NextReader; if (!readAhead.Eof() && StringValidator.IsDecimalDigit(readAhead.Peek().Value)) { return(LexerEngine.ReadNumericLiteralToken(reader)); } else { return(LexerEngine.ReadDotOperatorToken(reader)); } case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': // firstIdentifierChar var(identifierToken, nextLexer) = LexerEngine.ReadIdentifierToken(reader); var normalized = identifierToken.Name.ToLowerInvariant(); switch (normalized) { case "false": var falseToken = new BooleanLiteralToken(identifierToken.Extent, false); return(falseToken, nextLexer); case "true": var trueToken = new BooleanLiteralToken(identifierToken.Extent, true); return(trueToken, nextLexer); case "null": var nullLiteralToken = new NullLiteralToken(identifierToken.Extent); return(nullLiteralToken, nextLexer); default: return(identifierToken, nextLexer); } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': // decimalDigit return(LexerEngine.ReadNumericLiteralToken(reader)); case '\u0020': // space case '\u0009': // horizontal tab case '\u000D': // carriage return case '\u000A': // line feed // WS return(LexerEngine.ReadWhitespaceToken(reader)); default: throw new UnexpectedCharacterException(peek); } }