internal IfStatement(ParseNode parent, Parser p)
: base(parent, p)
{
// All if statements begin with "<<if EXPRESSION>>", so parse that
Clause primaryClause = new Clause();
p.ExpectSymbol(TokenType.BeginCommand);
p.ExpectSymbol(TokenType.If);
primaryClause.expression = Expression.Parse(this, p);
p.ExpectSymbol(TokenType.EndCommand);
// Read the statements for this clause until we hit an <<endif or <<else
// (which could be an "<<else>>" or an "<<else if"
var statements = new List<Statement>();
while (p.NextSymbolsAre(TokenType.BeginCommand, TokenType.EndIf) == false &&
p.NextSymbolsAre(TokenType.BeginCommand, TokenType.Else) == false &&
p.NextSymbolsAre(TokenType.BeginCommand, TokenType.ElseIf) == false) {
statements.Add(new Statement(this, p));
// Ignore any dedents
while (p.NextSymbolIs(TokenType.Dedent)) {
p.ExpectSymbol(TokenType.Dedent);
}
}
primaryClause.statements = statements;
clauses.Add(primaryClause);
// Handle as many <<elseif clauses as we find
while (p.NextSymbolsAre(TokenType.BeginCommand, TokenType.ElseIf)) {
var elseIfClause = new Clause();
// Parse the syntax for this clause's condition
p.ExpectSymbol(TokenType.BeginCommand);
p.ExpectSymbol(TokenType.ElseIf);
elseIfClause.expression = Expression.Parse(this, p);
p.ExpectSymbol(TokenType.EndCommand);
// Read statements until we hit an <<endif, <<else or another <<elseif
var clauseStatements = new List<Statement>();
while (p.NextSymbolsAre(TokenType.BeginCommand, TokenType.EndIf) == false &&
p.NextSymbolsAre(TokenType.BeginCommand, TokenType.Else) == false &&
p.NextSymbolsAre(TokenType.BeginCommand, TokenType.ElseIf) == false) {
clauseStatements.Add(new Statement(this, p));
// Ignore any dedents
while (p.NextSymbolIs(TokenType.Dedent)) {
p.ExpectSymbol(TokenType.Dedent);
}
}
elseIfClause.statements = clauseStatements;
clauses.Add(elseIfClause);
}
// Handle <<else>> if we have it
if (p.NextSymbolsAre(TokenType.BeginCommand, TokenType.Else, TokenType.EndCommand)) {
// parse the syntax (no expression this time, just "<<else>>"
p.ExpectSymbol(TokenType.BeginCommand);
p.ExpectSymbol(TokenType.Else);
p.ExpectSymbol(TokenType.EndCommand);
// and parse statements until we hit "<<endif"
var elseClause = new Clause();
var clauseStatements = new List<Statement>();
while (p.NextSymbolsAre(TokenType.BeginCommand, TokenType.EndIf) == false) {
clauseStatements.Add(new Statement(this, p));
}
elseClause.statements = clauseStatements;
this.clauses.Add(elseClause);
// Ignore any dedents
while (p.NextSymbolIs(TokenType.Dedent)) {
p.ExpectSymbol(TokenType.Dedent);
}
}
// Finish up by reading the <<endif>>
p.ExpectSymbol(TokenType.BeginCommand);
p.ExpectSymbol(TokenType.EndIf);
p.ExpectSymbol(TokenType.EndCommand);
}
internal CustomCommand(ParseNode parent, Parser p)
: base(parent, p)
{
p.ExpectSymbol(TokenType.BeginCommand);
// Custom commands can have ANY token in them. Read them all until we hit the
// end command token.
var commandTokens = new List<Token>();
do {
commandTokens.Add(p.ExpectSymbol());
} while (p.NextSymbolIs(TokenType.EndCommand) == false);
p.ExpectSymbol(TokenType.EndCommand);
// If the first token is an identifier and the second is
// a left paren, it may be a function call expression;
// evaluate it as such
if (commandTokens.Count > 1 &&
commandTokens[0].type == TokenType.Identifier &&
commandTokens[1].type == TokenType.LeftParen) {
var parser = new Parser(commandTokens, p.library);
var expression = Expression.Parse(this, parser);
type = Type.Expression;
this.expression = expression;
} else {
// Otherwise, evaluate it as a command
type = Type.ClientCommand;
this.clientCommand = commandTokens[0].value;
}
}
internal static Expression Parse(ParseNode parent, Parser p)
{
// Applies Djikstra's "shunting-yard" algorithm to convert the
// stream of infix expressions into postfix notation; we then
// build a tree of expressions from the result
// https://en.wikipedia.org/wiki/Shunting-yard_algorithm
Queue<Token> _expressionRPN = new Queue<Token> ();
var operatorStack = new Stack<Token>();
// used for keeping count of parameters for each function
var functionStack = new Stack<Token> ();
var allValidTokenTypes = new List<TokenType>(Operator.operatorTypes);
allValidTokenTypes.Add(TokenType.Number);
allValidTokenTypes.Add(TokenType.Variable);
allValidTokenTypes.Add(TokenType.String);
allValidTokenTypes.Add(TokenType.LeftParen);
allValidTokenTypes.Add(TokenType.RightParen);
allValidTokenTypes.Add(TokenType.Identifier);
allValidTokenTypes.Add(TokenType.Comma);
allValidTokenTypes.Add(TokenType.True);
allValidTokenTypes.Add(TokenType.False);
allValidTokenTypes.Add(TokenType.Null);
Token lastToken = null;
// Read all the contents of the expression
while (p.tokens.Count > 0 && p.NextSymbolIs(allValidTokenTypes.ToArray())) {
Token nextToken = p.ExpectSymbol(allValidTokenTypes.ToArray());
if (nextToken.type == TokenType.Number ||
nextToken.type == TokenType.Variable ||
nextToken.type == TokenType.String ||
nextToken.type == TokenType.True ||
nextToken.type == TokenType.False ||
nextToken.type == TokenType.Null) {
// Primitive values go straight onto the output
_expressionRPN.Enqueue (nextToken);
} else if (nextToken.type == TokenType.Identifier) {
operatorStack.Push (nextToken);
functionStack.Push (nextToken);
// next token must be a left paren, so process that immediately
nextToken = p.ExpectSymbol (TokenType.LeftParen);
// enter that sub-expression
operatorStack.Push (nextToken);
} else if (nextToken.type == TokenType.Comma) {
// Resolve this sub-expression before moving on to the
// next parameter
try {
// pop operators until we reach a left paren
while (operatorStack.Peek().type != TokenType.LeftParen) {
_expressionRPN.Enqueue(operatorStack.Pop());
}
} catch (InvalidOperationException) {
// we reached the end of the stack prematurely
// this means unbalanced parens!
throw ParseException.Make(nextToken, "Error parsing expression: " +
"unbalanced parentheses");
}
// We expect the top of the stack to now contain the left paren that
// began the list of parameters
if (operatorStack.Peek().type != TokenType.LeftParen) {
throw ParseException.Make (operatorStack.Peek (), "Expression parser got " +
"confused dealing with a function");
}
// The next token is not allowed to be a right-paren or a comma
// (that is, you can't say "foo(2,,)")
if (p.NextSymbolIs(TokenType.RightParen, TokenType.Comma)) {
throw ParseException.Make (p.tokens.Peek(), "Expected expression");
}
// Find the closest function on the stack
// and increment the number of parameters
functionStack.Peek().parameterCount++;
} else if (Operator.IsOperator(nextToken.type)) {
// This is an operator
// If this is a Minus, we need to determine if it's a
// unary minus or a binary minus.
// Unary minus looks like this: "-1"
// Binary minus looks like this: "2 - 3"
// Things get complex when we say stuff like "1 + -1".
// But it's easier when we realise that a minus
// is ONLY unary when the last token was a left paren,
// an operator, or it's the first token.
if (nextToken.type == TokenType.Minus) {
if (lastToken == null ||
lastToken.type == TokenType.LeftParen ||
Operator.IsOperator(lastToken.type)) {
// This is actually a unary minus.
nextToken.type = TokenType.UnaryMinus;
}
}
// We cannot assign values inside an expression. That is,
// saying "$foo = 2" in an express does not assign $foo to 2
// and then evaluate to 2. Instead, Yarn defines this
// to mean "$foo == 2"
if (nextToken.type == TokenType.EqualToOrAssign) {
nextToken.type = TokenType.EqualTo;
}
// O1 = this operator
// O2 = the token at the top of the stack
// While O2 is an operator, and EITHER: 1. O1 is left-associative and
// has precedence <= O2, or 2. O1 is right-associative and
// has precedence > O2:
while (ShouldApplyPrecedence(nextToken.type, operatorStack)) {
var o = operatorStack.Pop();
_expressionRPN.Enqueue(o);
}
operatorStack.Push(nextToken);
} else if (nextToken.type == TokenType.LeftParen) {
// Record that we have entered a paren-delimited
// subexpression
operatorStack.Push(nextToken);
} else if (nextToken.type == TokenType.RightParen) {
// We're leaving a subexpression; time to resolve the
// order of operations that we saw in between the parens.
try {
// pop operators until we reach a left paren
while (operatorStack.Peek().type != TokenType.LeftParen) {
_expressionRPN.Enqueue(operatorStack.Pop());
}
// pop the left paren
operatorStack.Pop();
} catch (InvalidOperationException) {
// we reached the end of the stack prematurely
// this means unbalanced parens!
throw ParseException.Make(nextToken, "Error parsing expression: unbalanced parentheses");
}
if (operatorStack.Peek().type == TokenType.Identifier) {
// This whole paren-delimited subexpression is actually
// a function call
// If the last token was a left-paren, then this
// was a function with no parameters; otherwise, we
// have an additional parameter (on top of the ones we counted
// while encountering commas)
if (lastToken.type != TokenType.LeftParen) {
functionStack.Peek ().parameterCount++;
}
_expressionRPN.Enqueue(operatorStack.Pop());
functionStack.Pop ();
}
}
// Record this as the last token we saw; we'll use
// this to figure out if minuses are unary or not
lastToken = nextToken;
}
// No more tokens; pop all operators onto the output queue
while (operatorStack.Count > 0) {
_expressionRPN.Enqueue(operatorStack.Pop());
}
// If the output queue is empty, then this is not an expression
if (_expressionRPN.Count == 0) {
throw new ParseException ("Error parsing expression: no expression found!");
}
// We've now got this in more easily parsed RPN form;
// time to build the expression tree.
Token firstToken = _expressionRPN.Peek();
var evaluationStack = new Stack<Expression>();
while (_expressionRPN.Count > 0) {
var next = _expressionRPN.Dequeue();
if (Operator.IsOperator(next.type)) {
// This is an operation
var info = Operator.InfoForOperator(next.type);
if (evaluationStack.Count < info.arguments) {
throw ParseException.Make(next, "Error parsing expression: not enough " +
"arguments for operator "+next.type.ToString());
}
var parameters = new List<Expression> ();
for (int i = 0; i < info.arguments; i++) {
parameters.Add (evaluationStack.Pop ());
}
parameters.Reverse ();
var operatorFunc = p.library.GetFunction (next.type.ToString());
var expr = new Expression (parent, operatorFunc, parameters);
evaluationStack.Push(expr);
} else if (next.type == TokenType.Identifier) {
// This is a function call
var info = p.library.GetFunction(next.value as String);
// Ensure that this call has the right number of params
if (info.IsParameterCountCorrect(next.parameterCount) == false) {
string error = string.Format("Error parsing expression: " +
"Unsupported number of parameters for function {0} (expected {1}, got {2})",
next.value as String,
info.paramCount,
next.parameterCount
);
throw ParseException.Make(next, error);
}
var parameterList = new List<Expression> ();
for (int i = 0; i < next.parameterCount; i++) {
parameterList.Add (evaluationStack.Pop());
}
parameterList.Reverse ();
var expr = new Expression (parent, info, parameterList);
evaluationStack.Push (expr);
} else {
// This is a raw value
var v = new ValueNode(parent, next);
Expression expr = new Expression(parent, v);
evaluationStack.Push(expr);
}
}
// We should now have a single expression in this stack, which is the root
// of the expression's tree. If we have more than one, then we have a problem.
if (evaluationStack.Count != 1) {
throw ParseException.Make(firstToken, "Error parsing expression " +
"(stack did not reduce correctly)");
}
// Return it
return evaluationStack.Pop ();
}
internal AssignmentStatement(ParseNode parent, Parser p)
: base(parent, p)
{
p.ExpectSymbol(TokenType.BeginCommand);
p.ExpectSymbol(TokenType.Set);
destinationVariableName = p.ExpectSymbol(TokenType.Variable).value as string;
operation = p.ExpectSymbol(validOperators).type;
valueExpression = Expression.Parse(this, p);
p.ExpectSymbol(TokenType.EndCommand);
}
internal Block(ParseNode parent, Parser p)
: base(parent, p)
{
// Read the indent token
p.ExpectSymbol(TokenType.Indent);
// Keep reading statements until we hit a dedent
while (p.NextSymbolIs(TokenType.Dedent) == false) {
// fun fact! because Blocks are a type of Statement,
// we get nested block parsing for free! \:D/
_statements.Add(new Statement(this, p));
}
// Tidy up by reading the dedent
p.ExpectSymbol(TokenType.Dedent);
}
// Read a number or a variable name from the parser
internal ValueNode(ParseNode parent, Parser p)
: base(parent, p)
{
Token t = p.ExpectSymbol(TokenType.Number, TokenType.Variable, TokenType.String);
UseToken(t);
}
internal Statement(ParseNode parent, Parser p)
: base(parent, p)
{
if (Block.CanParse(p)) {
type = Type.Block;
block = new Block(this, p);
return;
} else if (IfStatement.CanParse(p)) {
type = Type.IfStatement;
ifStatement = new IfStatement(this, p);
return;
} else if (OptionStatement.CanParse(p)) {
type = Type.OptionStatement;
optionStatement = new OptionStatement(this, p);
return;
} else if (AssignmentStatement.CanParse(p)) {
type = Type.AssignmentStatement;
assignmentStatement = new AssignmentStatement(this, p);
return;
} else if (ShortcutOptionGroup.CanParse(p)) {
type = Type.ShortcutOptionGroup;
shortcutOptionGroup = new ShortcutOptionGroup(this, p);
return;
} else if (CustomCommand.CanParse(p)) {
type = Type.CustomCommand;
customCommand = new CustomCommand(this, p);
return;
} else if (p.NextSymbolIs(TokenType.Text)) {
line = p.ExpectSymbol(TokenType.Text).value as string;
type = Type.Line;
} else {
throw ParseException.Make(p.tokens.Peek(), "Expected a statement here but got " + p.tokens.Peek().ToString() +" instead (was there an unbalanced if statement earlier?)");
}
}
internal ShortcutOption(int optionIndex, ParseNode parent, Parser p)
: base(parent, p)
{
p.ExpectSymbol(TokenType.ShortcutOption);
label = p.ExpectSymbol(TokenType.Text).value as string;
// Parse the conditional ("<<if $foo>>") if it's there
if (p.NextSymbolsAre(TokenType.BeginCommand, TokenType.If)) {
p.ExpectSymbol(TokenType.BeginCommand);
p.ExpectSymbol(TokenType.If);
condition = Expression.Parse(this, p);
p.ExpectSymbol(TokenType.EndCommand);
}
// Parse the statements belonging to this option if it has any
if (p.NextSymbolIs(TokenType.Indent)) {
p.ExpectSymbol(TokenType.Indent);
optionNode = new Node(NodeParent().name + "." + optionIndex, this, p);
p.ExpectSymbol(TokenType.Dedent);
}
}
internal OptionStatement(ParseNode parent, Parser p)
: base(parent, p)
{
// The meaning of the string(s) we have changes
// depending on whether we have one or two, so
// keep them both and decide their meaning once
// we know more
string firstString;
string secondString;
// Parse "[[LABEL"
p.ExpectSymbol(TokenType.OptionStart);
firstString = p.ExpectSymbol(TokenType.Text).value as String;
// If there's a | in there, get the string that comes after it
if (p.NextSymbolIs(TokenType.OptionDelimit)) {
p.ExpectSymbol(TokenType.OptionDelimit);
secondString = p.ExpectSymbol(TokenType.Text, TokenType.Identifier).value as String;
// Two strings mean that the first is the label, and the second
// is the name of the node that we should head to if this option
// is selected
label = firstString;
destination = secondString;
} else {
// One string means we don't have a label
label = null;
destination = firstString;
}
// Parse the closing ]]
p.ExpectSymbol(TokenType.OptionEnd);
}
internal Operator(ParseNode parent, Parser p)
: base(parent, p)
{
operatorType = p.ExpectSymbol(Operator.operatorTypes).type;
}
