private bool lastTokenWasOperatorOrLeftBrace = true; //beginning of input is like a left brace /// <summary> /// Parses the given tokens to an AST. /// </summary> public ExpressionNode Parse(IEnumerable <Token> tokens) { bool sequenceWasEmpty = true; foreach (var token in tokens) { sequenceWasEmpty = false; if (token is LiteralToken) { if (token is NumberLiteralToken) { working.Push(new NumberLiteralNode(((NumberLiteralToken)token).Number)); } else if (token is StringLiteralToken) { working.Push(new StringLiteralNode(((StringLiteralToken)token).String)); } lastTokenWasOperatorOrLeftBrace = false; } else if (token is OperatorToken) { ExpressionOperationType op; if (((OperatorToken)token).OperatorType == OperatorType.SubstractNegate) //need to check if binary or unary { op = lastTokenWasOperatorOrLeftBrace ? ExpressionOperationType.Negate : ExpressionOperationType.Subtract; } else //normal operator { op = operatorToOperation[((OperatorToken)token).OperatorType]; } //TODO: Do we need to check for assosiativity? Only unary operators and assignments are rtl-assosiativ ExpressionOperationType currentOperator; while (operators.Count != 0 && operationPrecedence[currentOperator = operators.Peek()] > operationPrecedence[op]) //stack empty or only low precendence operators on stack { PopOperator(); } operators.Push(op); lastTokenWasOperatorOrLeftBrace = true; } else if (token is OpenBraceToken) { var openBraceToken = token as OpenBraceToken; if (openBraceToken.BraceType == BraceType.Round) { if (working.Count > 0 && working.Peek() is VariableReferenceExpressionNode) { //we have a "variable" sitting on top of the op stack, this must be the name of a function to be called. //Create a function call instead var variable = (VariableReferenceExpressionNode)working.Pop(); working.Push(new FunctionCallExpressionNode(variable.VariableName)); operators.Push(ExpressionOperationType.FunctionCall); arity.Push(0); } operators.Push(ExpressionOperationType.OpenRoundBrace); } else if (openBraceToken.BraceType == BraceType.Square) { //Something to do with lists. if (working.Count > 0 && working.Peek() is VariableReferenceExpressionNode) { //There's an identifier right before, it must be an accessor myList[0] guessType = ExpressionParserGuessType.ListAccessor; } else { //Looks like a list literal: [1, 2, 3] arity.Push(0); //This will be the list size guessType = ExpressionParserGuessType.ListInitializer; } } lastTokenWasOperatorOrLeftBrace = true; } else if (token is CloseBraceToken) { var closeBraceToken = token as CloseBraceToken; if (closeBraceToken.BraceType == BraceType.Round) { while (operators.Peek() != ExpressionOperationType.OpenRoundBrace) { PopOperator(); } operators.Pop(); //pop the opening brace from the stack if (operators.Count > 0 && operators.Peek() == ExpressionOperationType.FunctionCall) //function call sitting on top of the stack { PopOperator(); } } else if (closeBraceToken.BraceType == BraceType.Square) { //End of a list initializer or accessor if (guessType == ExpressionParserGuessType.ListInitializer) { //Try building a list and return it var listElements = new List <ExpressionNode>(); while (working.Count > 0) //Pop all the working expressions onto the list { var listElement = working.Pop(); listElements.Add(listElement); } listElements.Reverse(); //Compensate for the order of the stack working.Push(new ListInitializerExpressionNode(listElements)); } else if (guessType == ExpressionParserGuessType.ListAccessor) { var accessorIndex = working.Pop(); //This is the index of the list we are accessing; for example, in myList[2], the index is 2 var listVariableReference = working.Pop() as VariableReferenceExpressionNode; //This must be an identifier, currently parsed as variable reference working.Push(new ListAccessorExpressionNode(listVariableReference.VariableName, accessorIndex)); } } lastTokenWasOperatorOrLeftBrace = false; } else if (token is IdentifierToken) { if (token is TableIdentifierToken) //These tokens need special treatment { var tokenList = tokens.ToList(); var tokensToSkip = tokenList.IndexOf(token) + 1; var upcomingTokens = new Stack <Token>(tokenList.Skip(tokensToSkip).Take(tokenList.Count - tokensToSkip).Reverse()); Token currentTestToken; List <MemberAccessToken> memberAccessTokens = new List <MemberAccessToken>(); while ((currentTestToken = upcomingTokens.Pop()) is MemberAccessToken) //They can be stacked { memberAccessTokens.Add(currentTestToken as MemberAccessToken); if (upcomingTokens.Count == 0) { break; //We're done. } } working.Push(new TableReferenceExpressionNode(TableQualifier.Create(token as IdentifierToken, memberAccessTokens))); } else { //this could be a function call, but we would need a lookahead to check for an opening brace. //just handle this as a variable reference and change it to a function when a open brace is found working.Push(new VariableReferenceExpressionNode(((IdentifierToken)token).Content)); } } else if (token is MemberAccessToken) { //Basically ignore these for now, they were handled by the tableidentifiertoken conditional } else if (token is ArgSeperatorToken) { /* * if (arity.Peek() == 0) //If it was previously assumed that there were no args * { * arity.Pop(); //There are at least * arity.Push(1); //One argument (at least) * } */ arity.Push(arity.Pop() + 1); //increase arity on top of the stack if (operators.Count > 0) { //It must be a function call, because there's a function name while (operators.Peek() != ExpressionOperationType.OpenRoundBrace) //pop till the open brace of this function call { PopOperator(); } } else { //it's basically commas with no function call, AKA elements of a list } } else { throw new UnexpectedTokenException("Found unknown token while parsing expression!", token); } } if (sequenceWasEmpty) { return(null); } //end of tokens, apply all the remaining operators while (operators.Count != 0) { PopOperator(); } if (working.Count != 1) { throw new ParsingException("Expression seems to be incomplete/invalid."); } return(working.Pop()); }
public ProgramNode ParseToAst() { _scopes.Push(new ProgramNode()); while (!EndOfProgram) { var upcomingToken = PeekNextToken(); if (upcomingToken is KeywordToken) { var keyword = (KeywordToken)NextToken(); var globalScope = _scopes.Count == 1; // There are constraints to what is available on the global scope if (keyword.IsTypeKeyword) { var varType = keyword.ToVariableType(); //it must be followed by a identifier: var name = ReadNextToken <IdentifierToken>(); //so see what it is (function or variable): Token lookahead = PeekNextToken(); if (lookahead is OperatorToken && (((OperatorToken)lookahead).OperatorType == OperatorType.Assignment) || lookahead is StatementSeparatorToken) //variable declaration { if (lookahead is OperatorToken) { NextToken(); //skip the "=" } _currentScope.AddStatement(new VariableDeclarationNode(varType, name.Content, ExpressionNode.CreateFromTokens(ReadUntilStatementSeparator()))); } else if (lookahead is OpenBraceToken) { var lookaheadOpenBrace = lookahead as OpenBraceToken; if (lookaheadOpenBrace.BraceType == BraceType.Square) //It is a list type declaration { //Ensure that the next token is the close square brace NextToken(); //Go past the opening token var secondLookahead = PeekNextToken(); if (secondLookahead is CloseBraceToken && ((CloseBraceToken)secondLookahead).BraceType == BraceType.Square) { NextToken(); //Good, it matched, now skip that too //Add a list declaration node _currentScope.AddStatement(new ListDeclarationNode(varType, name.Content, ExpressionNode.CreateFromTokens(ReadUntilStatementSeparator()))); } else { throw new UnexpectedTokenException("Missing close square bracket in array declaration"); } } //If in the global scope, it can also be a function declaration else if (globalScope && lookaheadOpenBrace.BraceType == BraceType.Round) //function definition { var func = new FunctionDeclarationNode(name.Content); _currentScope.AddStatement(func); //add the function to the old (root) scope... _scopes.Push(func); //...and set it a the new scope! //Read the argument list NextToken(); //skip the opening brace while (!(PeekNextToken() is CloseBraceToken && ((CloseBraceToken)PeekNextToken()).BraceType == BraceType.Round)) //TODO: Refactor using readUntilClosingBrace? { var argType = ReadNextToken <KeywordToken>(); if (!argType.IsTypeKeyword) { throw new ParsingException("Expected type keyword!"); } var argName = ReadNextToken <IdentifierToken>(); func.AddParameter(new ParameterDeclarationNode(argType.ToVariableType(), argName.Content)); if (PeekNextToken() is ArgSeperatorToken) //TODO: Does this allow (int a int b)-style functions? (No arg-seperator!) { NextToken(); //skip the sperator } } NextToken(); //skip the closing brace var curlyBrace = ReadNextToken <OpenBraceToken>(); if (curlyBrace.BraceType != BraceType.Curly) { throw new ParsingException("Wrong brace type found!"); } } } else { throw new Exception("The parser encountered an unexpected token."); } //We can't have anything other than what's listed above on the global scope } //Not a type keyword. If on a local scope, it may be other keywords else if (!globalScope) { switch (keyword.KeywordType) { case KeywordType.Return: _currentScope.AddStatement(new ReturnStatementNode(ExpressionNode.CreateFromTokens(ReadUntilStatementSeparator()))); break; case KeywordType.If: var @if = new IfStatementNode(ExpressionNode.CreateFromTokens(ReadUntilClosingBrace(true))); _currentScope.AddStatement(@if); //Add if statement to previous scope _scopes.Push(@if); //...and set it a the new scope! NextToken(); //skip the opening curly brace break; case KeywordType.While: var @while = new WhileLoopNode(ExpressionNode.CreateFromTokens(ReadUntilClosingBrace(true))); _currentScope.AddStatement(@while); _scopes.Push(@while); NextToken(); //skip the opening curly brace break; default: throw new ParsingException("Unexpected keyword type."); } } else //It was not a keyword, and it was a global scope { throw new ParsingException("Found non-type keyword on global scope."); } } else if (upcomingToken is IdentifierToken && _scopes.Count > 1) //in a nested scope { var identifierToken = upcomingToken as IdentifierToken; NextToken(); //Step past the identifier token var nextToken = PeekNextToken(); //Read the next token if (nextToken is OperatorToken && ((OperatorToken)nextToken).OperatorType == OperatorType.Assignment) //variable assignment { NextToken(); //skip the "=" _currentScope.AddStatement(new VariableAssignmentNode(identifierToken.Content, ExpressionNode.CreateFromTokens(ReadUntilStatementSeparator()))); } else if (nextToken is MemberAccessToken) { List <MemberAccessToken> memberAccessTokens = new List <MemberAccessToken>(); Token currentTestToken; while ((currentTestToken = PeekNextToken()) is MemberAccessToken) //They can be stacked { NextToken(); //Advance memberAccessTokens.Add(currentTestToken as MemberAccessToken); } if (currentTestToken is OperatorToken && ((OperatorToken)currentTestToken).OperatorType == OperatorType.Assignment) //table member assignment { NextToken(); //skip the "=" //Tokens until statement end have to be preloaded as a 'temporary workaround' to allow looking forward var expressionTokens = ReadUntilStatementSeparator().ToList(); _currentScope.AddStatement(new TableAssignmentNode(TableQualifier.Create(identifierToken, memberAccessTokens), ExpressionNode.CreateFromTokens(expressionTokens))); } else if (currentTestToken is OpenBraceToken && ((OpenBraceToken)currentTestToken).BraceType == BraceType.Round) { //Member invocation var fakeIdentifierToken = new IdentifierToken(identifierToken.Content); //So it will be parsed as a function call var memberInvocationInternalFunctionCall = ExpressionNode.CreateFromTokens(new[] { fakeIdentifierToken }.Concat(ReadUntilStatementSeparator())) as FunctionCallExpressionNode; _currentScope.AddStatement(new TableMemberInvocationNode(TableQualifier.Create(identifierToken, memberAccessTokens), memberInvocationInternalFunctionCall.Arguments)); } } else //lone expression (incl. function calls!) { _currentScope.AddStatement(ExpressionNode.CreateFromTokens(new[] { identifierToken }.Concat(ReadUntilStatementSeparator()))); //don't forget the name here! } } else if (upcomingToken is CloseBraceToken) { //Closing a scope var brace = ReadNextToken <CloseBraceToken>(); if (brace.BraceType != BraceType.Curly) { throw new ParsingException("Wrong brace type found!"); } _scopes.Pop(); //Scope has been closed! } else { throw new UnexpectedTokenException("The parser ran into an unexpected token", upcomingToken); } } if (_scopes.Count != 1) { throw new ParsingException("The scopes are not correctly nested."); } return((ProgramNode)_scopes.Pop()); }
public TableReferenceExpressionNode(TableQualifier tableQualifier) { TableQualifier = tableQualifier; }