/// <summary> /// Peek at the next available token without consuming it. /// </summary> /// <returns>the next available token, or the empty token if all tokens have been read</returns> /// <see cref="Token.Empty"/> public Token PeekToken() { if (_needToken) { _nextToken = ReadTokenFromReader(); _needToken = false; } return _nextToken; }
private bool Equals(Token other) { return type == other.type && value == other.value; }
/// <summary> /// Test the token to see if its an identifier /// </summary> /// <param name="tok">the token to test</param> /// <returns>true if its an identifier</returns> private static bool IsIdentifier(Token tok) { return tok.type == TokenType.Identifier; }
/// <summary> /// Test the token to see if its a keyword /// </summary> /// <param name="tok">the token to test</param> /// <returns>true if its a keyword</returns> private static bool IsKeyword(Token tok) { // include null? return tok.type == TokenType.Identifier && tok.value == "new"; }
/// <summary> /// Asserts that the token read is the one expected /// </summary> /// <param name="expected">the expected token</param> /// <param name="actual">the actual token</param> /// <param name="message">message to use in the exception if expected != actual</param> private static void RequireToken(Token expected, Token actual, string message, params object[] args) { if (actual != expected) { message = string.Format(message, args); throw new ParseException(message + " Expected: " + expected + " got: " + actual); } }
/// <summary> /// Test the token to see if its a quoted string /// </summary> /// <param name="tok">the token to test</param> /// <returns>true if its a quoted string</returns> private static bool IsQuotedString(Token tok) { return tok.type == TokenType.DoubleQuotedString || tok.type == TokenType.SingleQuotedString; }
/// <summary> /// Handler for 1 or more construct /// </summary> /// <param name="separator">the separator token between items</param> /// <param name="terminal">the ending token</param> /// <param name="meth">the method to call to parse an item</param> /// <param name="result">the parsed expression</param> /// <param name="first">flag indicating whether this is the first item</param> /// <returns>true if match parsed, false otherwise</returns> private bool ReadAhead(Token separator, Token terminal, ExpressionMethod meth, out Expression result, ref bool first) { Token tok = PeekToken(); result = null; if (tok == terminal) { ReadToken(); return false; } if (!first) { RequireToken(separator, tok, "Items should be separated by {0}", separator); ReadToken(); } else if (first && tok == separator) throw new ParseException("Error unexpected token " + tok); result = meth(); first = false; return true; }
public void SimpleObjectTest() { string toParse = "{\"key\":\"value\"}"; Token[] expected = new Token[] { new Token(TokenType.Symbol,"{"), new Token(TokenType.DoubleQuotedString, "key"), new Token(TokenType.Symbol, ":"), new Token(TokenType.DoubleQuotedString, "value"), new Token(TokenType.Symbol,"}") }; TestTokens(toParse, expected); }
public void SimpleListTest() { string toParse = "[\"item1\",\"item2\"]"; Token[] expected = new Token[] { new Token(TokenType.Symbol,"["), new Token(TokenType.DoubleQuotedString, "item1"), new Token(TokenType.Symbol, ","), new Token(TokenType.DoubleQuotedString, "item2"), new Token(TokenType.Symbol,"]") }; TestTokens(toParse, expected); }
/// <summary> /// Handler for 1 or more construct /// </summary> /// <param name="separator">the separator token between items</param> /// <param name="terminal">the ending token</param> /// <param name="meth">the method to call to parse an item</param> /// <param name="result">the parsed expression</param> /// <returns>true if match parsed, false otherwise</returns> private bool ReadAhead(Token separator, Token terminal, ExpressionMethod meth, out Expression result) { Token tok = PeekToken(); result = null; if (tok == terminal) { ReadToken(); return false; } else if (tok == separator) { ReadToken(); } result = meth(); return true; }
/// <summary> /// Asserts that the token read is the one expected /// </summary> /// <param name="expected">the expected token</param> /// <param name="actual">the actual token</param> /// <param name="message">message to use in the exception if expected != actual</param> private static void RequireToken(Token expected, Token actual, string message) { if (actual != expected) { throw new ParseException(message + " Expected: " + expected + " got: " + actual); } }