private bool lastTokenWasOperatorOrLeftBrace = true; //beginning of input is like a left brace
/// <summary>
/// Parses the given tokens to an AST.
/// </summary>
public ExpressionNode Parse(IEnumerable <Token> tokens)
{
bool sequenceWasEmpty = true;
foreach (var token in tokens)
{
sequenceWasEmpty = false;
if (token is LiteralToken)
{
if (token is NumberLiteralToken)
{
working.Push(new NumberLiteralNode(((NumberLiteralToken)token).Number));
}
else if (token is StringLiteralToken)
{
working.Push(new StringLiteralNode(((StringLiteralToken)token).String));
}
lastTokenWasOperatorOrLeftBrace = false;
}
else if (token is OperatorToken)
{
ExpressionOperationType op;
if (((OperatorToken)token).OperatorType == OperatorType.SubstractNegate) //need to check if binary or unary
{
op = lastTokenWasOperatorOrLeftBrace ? ExpressionOperationType.Negate : ExpressionOperationType.Subtract;
}
else //normal operator
{
op = operatorToOperation[((OperatorToken)token).OperatorType];
}
//TODO: Do we need to check for assosiativity? Only unary operators and assignments are rtl-assosiativ
ExpressionOperationType currentOperator;
while (operators.Count != 0 && operationPrecedence[currentOperator = operators.Peek()] > operationPrecedence[op]) //stack empty or only low precendence operators on stack
{
PopOperator();
}
operators.Push(op);
lastTokenWasOperatorOrLeftBrace = true;
}
else if (token is OpenBraceToken)
{
var openBraceToken = token as OpenBraceToken;
if (openBraceToken.BraceType == BraceType.Round)
{
if (working.Count > 0 && working.Peek() is VariableReferenceExpressionNode)
{
//we have a "variable" sitting on top of the op stack, this must be the name of a function to be called.
//Create a function call instead
var variable = (VariableReferenceExpressionNode)working.Pop();
working.Push(new FunctionCallExpressionNode(variable.VariableName));
operators.Push(ExpressionOperationType.FunctionCall);
arity.Push(0);
}
operators.Push(ExpressionOperationType.OpenRoundBrace);
}
else if (openBraceToken.BraceType == BraceType.Square)
{
//Something to do with lists.
if (working.Count > 0 && working.Peek() is VariableReferenceExpressionNode)
{
//There's an identifier right before, it must be an accessor myList[0]
guessType = ExpressionParserGuessType.ListAccessor;
}
else
{
//Looks like a list literal: [1, 2, 3]
arity.Push(0); //This will be the list size
guessType = ExpressionParserGuessType.ListInitializer;
}
}
lastTokenWasOperatorOrLeftBrace = true;
}
else if (token is CloseBraceToken)
{
var closeBraceToken = token as CloseBraceToken;
if (closeBraceToken.BraceType == BraceType.Round)
{
while (operators.Peek() != ExpressionOperationType.OpenRoundBrace)
{
PopOperator();
}
operators.Pop(); //pop the opening brace from the stack
if (operators.Count > 0 && operators.Peek() == ExpressionOperationType.FunctionCall) //function call sitting on top of the stack
{
PopOperator();
}
}
else if (closeBraceToken.BraceType == BraceType.Square)
{
//End of a list initializer or accessor
if (guessType == ExpressionParserGuessType.ListInitializer)
{
//Try building a list and return it
var listElements = new List <ExpressionNode>();
while (working.Count > 0) //Pop all the working expressions onto the list
{
var listElement = working.Pop();
listElements.Add(listElement);
}
listElements.Reverse(); //Compensate for the order of the stack
working.Push(new ListInitializerExpressionNode(listElements));
}
else if (guessType == ExpressionParserGuessType.ListAccessor)
{
var accessorIndex = working.Pop(); //This is the index of the list we are accessing; for example, in myList[2], the index is 2
var listVariableReference = working.Pop() as VariableReferenceExpressionNode; //This must be an identifier, currently parsed as variable reference
working.Push(new ListAccessorExpressionNode(listVariableReference.VariableName, accessorIndex));
}
}
lastTokenWasOperatorOrLeftBrace = false;
}
else if (token is IdentifierToken)
{
if (token is TableIdentifierToken) //These tokens need special treatment
{
var tokenList = tokens.ToList();
var tokensToSkip = tokenList.IndexOf(token) + 1;
var upcomingTokens = new Stack <Token>(tokenList.Skip(tokensToSkip).Take(tokenList.Count - tokensToSkip).Reverse());
Token currentTestToken;
List <MemberAccessToken> memberAccessTokens = new List <MemberAccessToken>();
while ((currentTestToken = upcomingTokens.Pop()) is MemberAccessToken) //They can be stacked
{
memberAccessTokens.Add(currentTestToken as MemberAccessToken);
if (upcomingTokens.Count == 0)
{
break; //We're done.
}
}
working.Push(new TableReferenceExpressionNode(TableQualifier.Create(token as IdentifierToken, memberAccessTokens)));
}
else
{
//this could be a function call, but we would need a lookahead to check for an opening brace.
//just handle this as a variable reference and change it to a function when a open brace is found
working.Push(new VariableReferenceExpressionNode(((IdentifierToken)token).Content));
}
}
else if (token is MemberAccessToken)
{
//Basically ignore these for now, they were handled by the tableidentifiertoken conditional
}
else if (token is ArgSeperatorToken)
{
/*
* if (arity.Peek() == 0) //If it was previously assumed that there were no args
* {
* arity.Pop(); //There are at least
* arity.Push(1); //One argument (at least)
* }
*/
arity.Push(arity.Pop() + 1); //increase arity on top of the stack
if (operators.Count > 0)
{
//It must be a function call, because there's a function name
while (operators.Peek() != ExpressionOperationType.OpenRoundBrace) //pop till the open brace of this function call
{
PopOperator();
}
}
else
{
//it's basically commas with no function call, AKA elements of a list
}
}
else
{
throw new UnexpectedTokenException("Found unknown token while parsing expression!", token);
}
}
if (sequenceWasEmpty)
{
return(null);
}
//end of tokens, apply all the remaining operators
while (operators.Count != 0)
{
PopOperator();
}
if (working.Count != 1)
{
throw new ParsingException("Expression seems to be incomplete/invalid.");
}
return(working.Pop());
}
public ProgramNode ParseToAst()
{
_scopes.Push(new ProgramNode());
while (!EndOfProgram)
{
var upcomingToken = PeekNextToken();
if (upcomingToken is KeywordToken)
{
var keyword = (KeywordToken)NextToken();
var globalScope = _scopes.Count == 1; // There are constraints to what is available on the global scope
if (keyword.IsTypeKeyword)
{
var varType = keyword.ToVariableType();
//it must be followed by a identifier:
var name = ReadNextToken <IdentifierToken>();
//so see what it is (function or variable):
Token lookahead = PeekNextToken();
if (lookahead is OperatorToken && (((OperatorToken)lookahead).OperatorType == OperatorType.Assignment) || lookahead is StatementSeparatorToken) //variable declaration
{
if (lookahead is OperatorToken)
{
NextToken(); //skip the "="
}
_currentScope.AddStatement(new VariableDeclarationNode(varType, name.Content, ExpressionNode.CreateFromTokens(ReadUntilStatementSeparator())));
}
else if (lookahead is OpenBraceToken)
{
var lookaheadOpenBrace = lookahead as OpenBraceToken;
if (lookaheadOpenBrace.BraceType == BraceType.Square) //It is a list type declaration
{
//Ensure that the next token is the close square brace
NextToken(); //Go past the opening token
var secondLookahead = PeekNextToken();
if (secondLookahead is CloseBraceToken && ((CloseBraceToken)secondLookahead).BraceType == BraceType.Square)
{
NextToken(); //Good, it matched, now skip that too
//Add a list declaration node
_currentScope.AddStatement(new ListDeclarationNode(varType, name.Content, ExpressionNode.CreateFromTokens(ReadUntilStatementSeparator())));
}
else
{
throw new UnexpectedTokenException("Missing close square bracket in array declaration");
}
}
//If in the global scope, it can also be a function declaration
else if (globalScope && lookaheadOpenBrace.BraceType == BraceType.Round) //function definition
{
var func = new FunctionDeclarationNode(name.Content);
_currentScope.AddStatement(func); //add the function to the old (root) scope...
_scopes.Push(func); //...and set it a the new scope!
//Read the argument list
NextToken(); //skip the opening brace
while (!(PeekNextToken() is CloseBraceToken && ((CloseBraceToken)PeekNextToken()).BraceType == BraceType.Round)) //TODO: Refactor using readUntilClosingBrace?
{
var argType = ReadNextToken <KeywordToken>();
if (!argType.IsTypeKeyword)
{
throw new ParsingException("Expected type keyword!");
}
var argName = ReadNextToken <IdentifierToken>();
func.AddParameter(new ParameterDeclarationNode(argType.ToVariableType(), argName.Content));
if (PeekNextToken() is ArgSeperatorToken) //TODO: Does this allow (int a int b)-style functions? (No arg-seperator!)
{
NextToken(); //skip the sperator
}
}
NextToken(); //skip the closing brace
var curlyBrace = ReadNextToken <OpenBraceToken>();
if (curlyBrace.BraceType != BraceType.Curly)
{
throw new ParsingException("Wrong brace type found!");
}
}
}
else
{
throw new Exception("The parser encountered an unexpected token.");
}
//We can't have anything other than what's listed above on the global scope
}
//Not a type keyword. If on a local scope, it may be other keywords
else if (!globalScope)
{
switch (keyword.KeywordType)
{
case KeywordType.Return:
_currentScope.AddStatement(new ReturnStatementNode(ExpressionNode.CreateFromTokens(ReadUntilStatementSeparator())));
break;
case KeywordType.If:
var @if = new IfStatementNode(ExpressionNode.CreateFromTokens(ReadUntilClosingBrace(true)));
_currentScope.AddStatement(@if); //Add if statement to previous scope
_scopes.Push(@if); //...and set it a the new scope!
NextToken(); //skip the opening curly brace
break;
case KeywordType.While:
var @while = new WhileLoopNode(ExpressionNode.CreateFromTokens(ReadUntilClosingBrace(true)));
_currentScope.AddStatement(@while);
_scopes.Push(@while);
NextToken(); //skip the opening curly brace
break;
default:
throw new ParsingException("Unexpected keyword type.");
}
}
else //It was not a keyword, and it was a global scope
{
throw new ParsingException("Found non-type keyword on global scope.");
}
}
else if (upcomingToken is IdentifierToken && _scopes.Count > 1) //in a nested scope
{
var identifierToken = upcomingToken as IdentifierToken;
NextToken(); //Step past the identifier token
var nextToken = PeekNextToken(); //Read the next token
if (nextToken is OperatorToken && ((OperatorToken)nextToken).OperatorType == OperatorType.Assignment) //variable assignment
{
NextToken(); //skip the "="
_currentScope.AddStatement(new VariableAssignmentNode(identifierToken.Content, ExpressionNode.CreateFromTokens(ReadUntilStatementSeparator())));
}
else if (nextToken is MemberAccessToken)
{
List <MemberAccessToken> memberAccessTokens = new List <MemberAccessToken>();
Token currentTestToken;
while ((currentTestToken = PeekNextToken()) is MemberAccessToken) //They can be stacked
{
NextToken(); //Advance
memberAccessTokens.Add(currentTestToken as MemberAccessToken);
}
if (currentTestToken is OperatorToken && ((OperatorToken)currentTestToken).OperatorType == OperatorType.Assignment) //table member assignment
{
NextToken(); //skip the "="
//Tokens until statement end have to be preloaded as a 'temporary workaround' to allow looking forward
var expressionTokens = ReadUntilStatementSeparator().ToList();
_currentScope.AddStatement(new TableAssignmentNode(TableQualifier.Create(identifierToken, memberAccessTokens), ExpressionNode.CreateFromTokens(expressionTokens)));
}
else if (currentTestToken is OpenBraceToken && ((OpenBraceToken)currentTestToken).BraceType == BraceType.Round)
{
//Member invocation
var fakeIdentifierToken = new IdentifierToken(identifierToken.Content); //So it will be parsed as a function call
var memberInvocationInternalFunctionCall = ExpressionNode.CreateFromTokens(new[] { fakeIdentifierToken }.Concat(ReadUntilStatementSeparator())) as FunctionCallExpressionNode;
_currentScope.AddStatement(new TableMemberInvocationNode(TableQualifier.Create(identifierToken, memberAccessTokens), memberInvocationInternalFunctionCall.Arguments));
}
}
else //lone expression (incl. function calls!)
{
_currentScope.AddStatement(ExpressionNode.CreateFromTokens(new[] { identifierToken }.Concat(ReadUntilStatementSeparator()))); //don't forget the name here!
}
}
else if (upcomingToken is CloseBraceToken)
{
//Closing a scope
var brace = ReadNextToken <CloseBraceToken>();
if (brace.BraceType != BraceType.Curly)
{
throw new ParsingException("Wrong brace type found!");
}
_scopes.Pop(); //Scope has been closed!
}
else
{
throw new UnexpectedTokenException("The parser ran into an unexpected token", upcomingToken);
}
}
if (_scopes.Count != 1)
{
throw new ParsingException("The scopes are not correctly nested.");
}
return((ProgramNode)_scopes.Pop());
}
public TableReferenceExpressionNode(TableQualifier tableQualifier)
{
TableQualifier = tableQualifier;
}
