public override bool Parse(ParseContext context, IAstNode parent)
{
TokenStream <RToken> tokens = context.Tokens;
Debug.Assert(context.Tokens.CurrentToken.TokenType == RTokenType.Identifier ||
context.Tokens.CurrentToken.TokenType == RTokenType.String);
this.Identifier = RParser.ParseToken(context, this);
this.EqualsSign = RParser.ParseToken(context, this);
if (context.Tokens.CurrentToken.TokenType != RTokenType.Comma && context.Tokens.CurrentToken.TokenType != RTokenType.CloseBrace)
{
Expression exp = new Expression(inGroup: true);
if (exp.Parse(context, this))
{
this.DefaultValue = exp;
}
}
else
{
this.DefaultValue = new NullExpression();
if (context.Tokens.IsEndOfStream())
{
context.AddError(new ParseError(ParseErrorType.ExpressionExpected, ErrorLocation.Token, context.Tokens.CurrentToken));
}
else
{
context.AddError(new ParseError(ParseErrorType.ExpressionExpected, ErrorLocation.Token, EqualsSign));
}
}
return(base.Parse(context, parent));
}
public override bool Parse(ParseContext context, IAstNode parent)
{
TokenStream <RToken> tokens = context.Tokens;
if (tokens.CurrentToken.IsVariableKind())
{
this.VariableName = new TokenNode();
this.VariableName.Parse(context, this);
if (tokens.CurrentToken.IsKeywordText(context.TextProvider, "in"))
{
this.InOperator = new TokenNode();
this.InOperator.Parse(context, this);
this.Expression = new Expression(inGroup: true);
if (this.Expression.Parse(context, this))
{
return(base.Parse(context, parent));
}
}
else
{
context.AddError(new MissingItemParseError(ParseErrorType.InKeywordExpected, tokens.CurrentToken));
}
}
else
{
context.AddError(new MissingItemParseError(ParseErrorType.IndentifierExpected, tokens.PreviousToken));
}
return(false);
}
private bool IsConsistentOperationSequence(ParseContext context, OperationType currentOperationType)
{
// Binary operator followed by another binary like 'a +/ b' is an error.
// 'func()()' or 'func() +' is allowed but 'func() operand' is not.
// Binary operator without anything behind it is an error;
if ((_previousOperationType == OperationType.Function && currentOperationType == OperationType.Operand) ||
(_previousOperationType == currentOperationType && currentOperationType != OperationType.Function))
{
switch (currentOperationType)
{
case OperationType.Operand:
// 'operand operand' sequence is an error
context.AddError(new ParseError(ParseErrorType.OperatorExpected, ErrorLocation.Token, GetOperandErrorRange(context)));
break;
case OperationType.UnaryOperator:
// 'unary unary' and 'binary unary' are OK
return(true);
default:
// 'operator operator' sequence is an error
context.AddError(new ParseError(ParseErrorType.RightOperandExpected, ErrorLocation.Token, GetOperatorErrorRange(context)));
break;
}
return(false);
}
if (currentOperationType == OperationType.BinaryOperator && context.Tokens.IsEndOfStream())
{
// 'operator <EOF>' sequence is an error
context.AddError(new ParseError(ParseErrorType.RightOperandExpected, ErrorLocation.Token, GetOperatorErrorRange(context)));
return(false);
}
if (_previousOperationType == OperationType.UnaryOperator && currentOperationType == OperationType.BinaryOperator)
{
// unary followed by binary doesn't make sense
context.AddError(new ParseError(ParseErrorType.IndentifierExpected, ErrorLocation.Token, GetOperatorErrorRange(context)));
return(false);
}
if (_previousOperationType == OperationType.BinaryOperator && currentOperationType == OperationType.EndOfExpression)
{
// missing list selector: z$ }
context.AddError(new ParseError(ParseErrorType.RightOperandExpected, ErrorLocation.Token, GetErrorRange(context)));
return(false);
}
return(true);
}
public override bool Parse(ParseContext context, IAstNode parent)
{
TokenStream <RToken> tokens = context.Tokens;
Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.OpenBrace);
this.OpenBrace = RParser.ParseToken(context, this);
this.Arguments = new ArgumentList(RTokenType.CloseBrace);
bool argumentsParsed = this.Arguments.Parse(context, this);
if (argumentsParsed)
{
if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace)
{
this.CloseBrace = RParser.ParseToken(context, this);
}
}
if (!argumentsParsed || this.CloseBrace == null)
{
CalculateVirtualEnd(context);
}
if (this.CloseBrace == null)
{
context.AddError(new MissingItemParseError(ParseErrorType.CloseBraceExpected, tokens.PreviousToken));
}
return(base.Parse(context, parent));
}
public override bool Parse(ParseContext context, IAstNode parent)
{
// First parse base which should pick up keyword, braces, inner
// expression and either full or simple (single statement) scope
if (!base.Parse(context, parent))
{
return(false);
}
// At this point we should be either at 'else' token or
// at the next statement. In the latter case we are done.
TokenStream <RToken> tokens = context.Tokens;
if (tokens.CurrentToken.IsKeywordText(context.TextProvider, "else"))
{
bool allowLineBreak = AllowLineBreakBeforeElse(context);
if (!allowLineBreak)
{
// Verify that there is no line break before the 'else'
if (context.Tokens.IsLineBreakAfter(context.TextProvider, tokens.Position - 1))
{
context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, tokens.CurrentToken));
return(true);
}
}
this.Else = new KeywordScopeStatement(allowsSimpleScope: true);
return(this.Else.Parse(context, this));
}
// Not at 'else' so we are done here
return(true);
}
public override bool Parse(ParseContext context, IAstNode parent)
{
RToken currentToken = context.Tokens.CurrentToken;
string text = context.TextProvider.GetText(currentToken);
double result;
Debug.Assert(currentToken.TokenType == RTokenType.Number ||
currentToken.TokenType == RTokenType.Infinity ||
currentToken.TokenType == RTokenType.NaN);
if (currentToken.TokenType == RTokenType.Infinity)
{
Value = new RNumber(Double.PositiveInfinity);
}
else if (currentToken.TokenType == RTokenType.NaN)
{
Value = new RNumber(Double.NaN);
}
else
{
// If parsing fails we still need to create node
// since we need a range to squiggle
result = Double.NaN;
if (!Number.TryParse(text, out result))
{
// Something unparsable
context.AddError(new ParseError(ParseErrorType.NumberExpected, ErrorLocation.Token, currentToken));
}
Value = new RNumber(result);
}
return(base.Parse(context, parent));
}
public override bool Parse(ParseContext context, IAstNode parent)
{
TokenStream <RToken> tokens = context.Tokens;
Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.OpenSquareBracket ||
tokens.CurrentToken.TokenType == RTokenType.OpenDoubleSquareBracket);
this.LeftBrackets = RParser.ParseToken(context, this);
RTokenType terminatingTokenType = RParser.GetTerminatingTokenType(this.LeftBrackets.Token.TokenType);
this.Arguments = new ArgumentList(terminatingTokenType);
this.Arguments.Parse(context, this);
if (tokens.CurrentToken.TokenType == terminatingTokenType)
{
this.RightBrackets = RParser.ParseToken(context, this);
return(base.Parse(context, parent));
}
else
{
context.AddError(new MissingItemParseError(ParseErrorType.CloseSquareBracketExpected, tokens.PreviousToken));
}
return(true);
}
public override bool Parse(ParseContext context, IAstNode parent)
{
RToken currentToken = context.Tokens.CurrentToken;
string text = context.TextProvider.GetText(currentToken);
bool? result = null;
Debug.Assert(currentToken.TokenType == RTokenType.Logical);
if (text.Equals("TRUE", StringComparison.Ordinal) || text.Equals("T", StringComparison.Ordinal))
{
result = true;
}
else if (text.Equals("FALSE", StringComparison.Ordinal) || text.Equals("F", StringComparison.Ordinal))
{
result = false;
}
if (result.HasValue)
{
Value = new RLogical(result.Value);
return(base.Parse(context, parent));
}
context.AddError(new ParseError(ParseErrorType.LogicalExpected, ErrorLocation.Token, currentToken));
return(false);
}
public override bool Parse(ParseContext context, IAstNode parent)
{
RToken currentToken = context.Tokens.CurrentToken;
string text = context.TextProvider.GetText(currentToken);
double realPart = 0;
double imaginaryPart = 0;
Debug.Assert(currentToken.TokenType == RTokenType.Complex);
// Split into real and imaginary parts. Imaginary part
// should always be there since otherwise tokenizer would
// not have idenfified the number as complex. Note that
// real part may be missing as in '+0i'. Operator may also
// be missing: 1i is a legal complex number.
Debug.Assert(text[text.Length - 1] == 'i');
// Drop trailing i and retokenize as two numbers
RTokenizer tokenizer = new RTokenizer(separateComments: false);
IReadOnlyTextRangeCollection <RToken> tokens = tokenizer.Tokenize(text.Substring(0, text.Length - 1));
if (tokens.Count == 1)
{
// Only imaginary part is present
Debug.Assert(tokens[0].TokenType == RTokenType.Number);
// TODO: handle complex numbers in Hex
if (!Double.TryParse(text.Substring(tokens[0].Start, tokens[0].Length), out imaginaryPart))
{
imaginaryPart = 0;
}
}
else if (tokens.Count == 3)
{
// Real and imaginary parts present
Debug.Assert(tokens[0].TokenType == RTokenType.Number);
Debug.Assert(tokens[1].TokenType == RTokenType.Operator);
Debug.Assert(tokens[2].TokenType == RTokenType.Number);
// TODO: handle complex numbers in Hex
if (!Double.TryParse(text.Substring(tokens[0].Start, tokens[0].Length), out realPart))
{
realPart = 0;
}
if (!Double.TryParse(text.Substring(tokens[2].Start, tokens[2].Length), out imaginaryPart))
{
imaginaryPart = 0;
}
}
else
{
context.AddError(new MissingItemParseError(ParseErrorType.NumberExpected, context.Tokens.PreviousToken));
return(false);
}
Complex complex = new Complex(realPart, imaginaryPart);
NodeValue = new RComplex(complex);
return(base.Parse(context, parent));
}
public override bool Parse(ParseContext context, IAstNode parent)
{
if (context.Tokens.CurrentToken.TokenType == RTokenType.OpenBrace)
{
context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, context.Tokens.CurrentToken));
}
return(false);
}
public override bool Parse(ParseContext context, IAstNode parent)
{
TokenStream <RToken> tokens = context.Tokens;
Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.Keyword);
this.Keyword = RParser.ParseKeyword(context, this);
this.Text = context.TextProvider.GetText(this.Keyword);
if (tokens.CurrentToken.TokenType == RTokenType.OpenBrace)
{
this.OpenBrace = RParser.ParseToken(context, this);
this.Arguments = new ArgumentList(RTokenType.CloseBrace);
this.Arguments.Parse(context, this);
if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace)
{
this.CloseBrace = RParser.ParseToken(context, this);
this.Scope = RParser.ParseScope(context, this, allowsSimpleScope: true, terminatingKeyword: null);
if (this.Scope != null)
{
return(base.Parse(context, parent));
}
else
{
context.AddError(new ParseError(ParseErrorType.FunctionBodyExpected, ErrorLocation.Token, tokens.PreviousToken));
}
}
else
{
context.AddError(new ParseError(ParseErrorType.CloseBraceExpected, ErrorLocation.Token, tokens.CurrentToken));
}
}
else
{
context.AddError(new ParseError(ParseErrorType.OpenBraceExpected, ErrorLocation.Token, tokens.CurrentToken));
}
return(false);
}
public override bool Parse(ParseContext context, IAstNode parent)
{
TokenStream <RToken> tokens = context.Tokens;
if (tokens.CurrentToken.IsVariableKind())
{
var v = new Variable();
v.Parse(context, this);
// Variables don't set parent since during complex
// exression parsing parent is determined by the
// expression parser based on precedence and grouping.
v.Parent = this;
this.Variable = v;
if (tokens.CurrentToken.IsKeywordText(context.TextProvider, "in"))
{
this.InOperator = new TokenNode();
this.InOperator.Parse(context, this);
this.Expression = new Expression(inGroup: true);
if (this.Expression.Parse(context, this))
{
return(base.Parse(context, parent));
}
}
else
{
context.AddError(new MissingItemParseError(ParseErrorType.InKeywordExpected, tokens.CurrentToken));
}
}
else
{
context.AddError(new MissingItemParseError(ParseErrorType.IndentifierExpected, tokens.PreviousToken));
}
return(false);
}
/// <summary>
/// Constructs AST node from operator (root) and one or two operands.
/// In order for the node to be successfully created stacks must contain
/// an operator and, depending on the operator, one or two operands.
/// </summary>
/// <example>
/// The newly created subtree (operator and root and operands are children)
/// is then pushed into the operands stack. Example: in a*b+c before '+'
/// can be processed, a*b is turned into an subtree and pushed as an operand
/// to the operands stack. Then new subtree can be created with + at the root
/// and 'c' and 'a*b' as its child nodes.
/// </example>
/// <param name="context">Parsing context</param>
/// <returns>Parsing error of any</returns>
private ParseErrorType MakeNode(ParseContext context)
{
var operatorNode = _operators.Pop();
var rightOperand = SafeGetOperand(operatorNode);
if (rightOperand == null)
{
// Oddly, no operands
return(ParseErrorType.RightOperandExpected);
}
if (operatorNode.IsUnary)
{
operatorNode.AppendChild(rightOperand);
operatorNode.RightOperand = rightOperand;
}
else
{
var leftOperand = SafeGetOperand(operatorNode);
if (leftOperand == null)
{
// Operand is missing in expression like x <- [].
// Operator on top of the stack is <- since [] was not
// successfully parsed. So we need to mark right operand
// as error token.
context.AddError(new ParseError(ParseErrorType.LeftOperandExpected, ErrorLocation.Token, GetIndexerOrFunctionErrorRange(context, operatorNode)));
return(ParseErrorType.LeftOperandExpected);
}
if (leftOperand.End <= operatorNode.Start && rightOperand.Start >= operatorNode.End)
{
operatorNode.LeftOperand = leftOperand;
operatorNode.RightOperand = rightOperand;
operatorNode.AppendChild(leftOperand);
operatorNode.AppendChild(rightOperand);
}
else
{
return(ParseErrorType.UnexpectedToken);
}
}
_operands.Push(operatorNode);
return(ParseErrorType.None);
}
protected bool ParseSemicolon(ParseContext context, IAstNode parent)
{
if (!context.Tokens.IsEndOfStream())
{
if (!context.Tokens.IsLineBreakAfter(context.TextProvider, context.Tokens.Position - 1))
{
if (context.Tokens.CurrentToken.TokenType == RTokenType.Semicolon)
{
this.Semicolon = RParser.ParseToken(context, this);
}
else
{
context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, context.Tokens.CurrentToken));
return(false);
}
}
}
return(true);
}
public override bool Parse(ParseContext context, IAstNode parent)
{
TokenStream <RToken> tokens = context.Tokens;
Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.OpenBrace);
this.OpenBrace = RParser.ParseToken(context, this);
this.Content = new Expression(inGroup: true);
this.Content.Parse(context, this);
if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace)
{
this.CloseBrace = RParser.ParseToken(context, this);
}
else
{
context.AddError(new MissingItemParseError(ParseErrorType.CloseBraceExpected, tokens.PreviousToken));
}
return(base.Parse(context, parent));
}
protected override CommaSeparatedItem CreateItem(IAstNode parent, ParseContext context)
{
RToken currentToken = context.Tokens.CurrentToken;
RToken nextToken = context.Tokens.NextToken;
switch (currentToken.TokenType)
{
case RTokenType.Ellipsis:
return(new EllipsisArgument());
case RTokenType.Comma:
return(new MissingArgument());
case RTokenType.Identifier:
case RTokenType.String:
if (nextToken.TokenType == RTokenType.Operator && context.TextProvider.GetText(nextToken) == "=")
{
return(new NamedArgument());
}
break;
case RTokenType.Logical:
case RTokenType.Complex:
case RTokenType.NaN:
case RTokenType.Null:
case RTokenType.Number:
case RTokenType.Infinity:
if (nextToken.TokenType == RTokenType.Operator && context.TextProvider.GetText(nextToken) == "=")
{
context.AddError(new ParseError(ParseErrorType.IndentifierExpected, ErrorLocation.Token, currentToken));
return(new ErrorArgument(CollectErrorArgumentTokens(context)));
}
break;
case RTokenType.CloseBrace:
return(null); // no arguments supplied
}
return(new ExpressionArgument());
}
/// <summary>
/// Abstract factory
/// </summary>
public static IStatement CreateStatement(ParseContext context, IAstNode parent)
{
RToken currentToken = context.Tokens.CurrentToken;
string keyword = context.TextProvider.GetText(currentToken);
IStatement statement = null;
switch (keyword)
{
case "if":
statement = new If();
break;
case "for":
statement = new For();
break;
case "while":
statement = new KeywordExpressionScopeStatement();
break;
case "repeat":
statement = new KeywordScopeStatement(allowsSimpleScope: false);
break;
case "break":
case "next":
statement = new KeywordStatement();
break;
case "function":
statement = new FunctionStatement();
break;
default:
context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, currentToken));
break;
}
return(statement);
}
public override bool Parse(ParseContext context, IAstNode parent)
{
TokenStream <RToken> tokens = context.Tokens;
RToken currentToken = tokens.CurrentToken;
context.Scopes.Push(this);
if (!(this is GlobalScope) && currentToken.TokenType == RTokenType.OpenCurlyBrace)
{
this.OpenCurlyBrace = RParser.ParseToken(context, this);
}
while (!tokens.IsEndOfStream())
{
currentToken = context.Tokens.CurrentToken;
switch (currentToken.TokenType)
{
case RTokenType.CloseCurlyBrace:
if (this.OpenCurlyBrace != null)
{
this.CloseCurlyBrace = RParser.ParseToken(context, this);
}
else
{
context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, currentToken));
context.Tokens.MoveToNextToken();
}
break;
case RTokenType.OpenCurlyBrace:
IScope scope = new Scope(string.Empty);
scope.Parse(context, this);
break;
default:
IStatement statement = Statement.Create(context, this, null);
if (statement != null)
{
if (statement.Parse(context, this))
{
this.statements.Add(statement);
}
else
{
statement = null;
}
}
if (statement == null)
{
if (!context.TextProvider.IsNewLineBeforePosition(context.Tokens.CurrentToken.Start))
{
// try recovering at the next line or past nearest
// semicolon or closing curly brace
tokens.MoveToNextLine(context.TextProvider,
(TokenStream <RToken> ts) => {
return(ts.CurrentToken.TokenType == RTokenType.Semicolon ||
ts.NextToken.TokenType == RTokenType.CloseCurlyBrace);
});
}
else
{
tokens.MoveToNextToken();
}
}
break;
}
if (this.CloseCurlyBrace != null)
{
break;
}
}
context.Scopes.Pop();
if (this.OpenCurlyBrace != null && this.CloseCurlyBrace == null)
{
context.AddError(new MissingItemParseError(ParseErrorType.CloseCurlyBraceExpected, context.Tokens.PreviousToken));
}
// TODO: process content and fill out declared variables
// and functions and get data to the classifier for colorization.
return(base.Parse(context, parent));
}
private bool ParseExpression(ParseContext context)
{
// https://en.wikipedia.org/wiki/Shunting-yard_algorithm
// http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm
// Instead of evaluating expressions like calculator would do,
// we create tree nodes with operator and its operands.
var tokens = context.Tokens;
var currentOperationType = OperationType.None;
var errorType = ParseErrorType.None;
var errorLocation = ErrorLocation.AfterToken;
var endOfExpression = false;
// Push sentinel
_operators.Push(Expression.Sentinel);
while (!tokens.IsEndOfStream() && errorType == ParseErrorType.None && !endOfExpression)
{
var token = tokens.CurrentToken;
switch (token.TokenType)
{
// Terminal constants
case RTokenType.Number:
case RTokenType.Complex:
case RTokenType.Logical:
case RTokenType.String:
case RTokenType.Null:
case RTokenType.Missing:
case RTokenType.NaN:
case RTokenType.Infinity:
currentOperationType = HandleConstant(context);
break;
// Variables and function calls
case RTokenType.Identifier:
if (context.ExpressionTermFilter.IsInertRange(context.Tokens.CurrentToken))
{
// This is a workaround for constructs like ```{r x = 1, y = FALSE} where the { }
// block is treated as R fragment. The fragment is syntactually incorrect since
// 'r' is indentifier and there is an operator expected between 'r' and 'x'.
// In order to avoid parsing errors expression parser will use this flag and
// allow standalone indentifier 'r' or 'R' right after the opening curly brace.
context.Tokens.MoveToNextToken();
continue;
}
currentOperationType = HandleIdentifier(context);
break;
case RTokenType.Ellipsis:
currentOperationType = HandleIdentifier(context);
break;
// Nested expression such as a*(b+c) or a nameless
// function call like a[2](x, y) or func(a, b)(c, d)
case RTokenType.OpenBrace:
currentOperationType = HandleOpenBrace(context, out errorType);
break;
case RTokenType.OpenCurlyBrace:
currentOperationType = HandleLambda(context, out errorType);
break;
case RTokenType.OpenSquareBracket:
case RTokenType.OpenDoubleSquareBracket:
currentOperationType = HandleSquareBrackets(context, out errorType);
break;
case RTokenType.Operator:
currentOperationType = HandleOperator(context, out errorType);
break;
case RTokenType.CloseBrace:
currentOperationType = OperationType.EndOfExpression;
endOfExpression = true;
break;
case RTokenType.CloseCurlyBrace:
case RTokenType.CloseSquareBracket:
case RTokenType.CloseDoubleSquareBracket:
currentOperationType = OperationType.EndOfExpression;
endOfExpression = true;
break;
case RTokenType.Comma:
case RTokenType.Semicolon:
currentOperationType = OperationType.EndOfExpression;
endOfExpression = true;
break;
case RTokenType.Keyword:
currentOperationType = HandleKeyword(context, out errorType);
endOfExpression = true;
break;
default:
errorType = ParseErrorType.UnexpectedToken;
errorLocation = ErrorLocation.Token;
break;
}
if (errorType == ParseErrorType.None && !IsConsistentOperationSequence(context, currentOperationType))
{
return(false);
}
if (errorType != ParseErrorType.None || endOfExpression)
{
break;
}
_previousOperationType = currentOperationType;
endOfExpression = CheckEndOfExpression(context, currentOperationType);
}
if (errorType != ParseErrorType.None)
{
if (errorLocation == ErrorLocation.AfterToken)
{
context.AddError(new ParseError(errorType, ErrorLocation.AfterToken, tokens.PreviousToken));
}
else
{
context.AddError(new ParseError(errorType, ErrorLocation.Token, tokens.CurrentToken));
}
}
if (_operators.Count > 1)
{
// If there are still operators to process, construct the final subtree.
// After that only the sentinel operator should be in the operators stack
// and a single final root node in the operand stack.
errorType = this.ProcessHigherPrecendenceOperators(context, Expression.Sentinel);
}
if (errorType != ParseErrorType.None)
{
if (errorType != ParseErrorType.LeftOperandExpected)
{
context.AddError(new ParseError(errorType, ErrorLocation.Token, GetErrorRange(context)));
}
// Although there may be errors such as incomplete function
// we still want to include the result into the tree since
// even in the code like 'func(,,, for(' we want intellisense
// and parameter to work.
if (_operands.Count == 1)
{
Content = _operands.Pop();
AppendChild(this.Content);
}
}
else
{
Debug.Assert(_operators.Count == 1);
// If operand stack ie empty and there is no error
// then the expression is empty.
if (_operands.Count > 0)
{
Debug.Assert(_operands.Count == 1);
Content = _operands.Pop();
AppendChild(Content);
}
}
return(true);
}
private bool ParseExpression(ParseContext context)
{
// https://en.wikipedia.org/wiki/Shunting-yard_algorithm
// http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm
// Instead of evaluating expressions like calculator would do,
// we create tree nodes with operator and its operands.
TokenStream <RToken> tokens = context.Tokens;
OperationType currentOperationType = OperationType.None;
ParseErrorType errorType = ParseErrorType.None;
ErrorLocation errorLocation = ErrorLocation.AfterToken;
bool endOfExpression = false;
// Push sentinel
_operators.Push(Expression.Sentinel);
while (!tokens.IsEndOfStream() && errorType == ParseErrorType.None && !endOfExpression)
{
RToken token = tokens.CurrentToken;
switch (token.TokenType)
{
// Terminal constants
case RTokenType.Number:
case RTokenType.Complex:
case RTokenType.Logical:
case RTokenType.String:
case RTokenType.Null:
case RTokenType.Missing:
case RTokenType.NaN:
case RTokenType.Infinity:
currentOperationType = HandleConstant(context);
break;
// Variables and function calls
case RTokenType.Identifier:
currentOperationType = HandleIdentifier(context);
break;
// Nested expression such as a*(b+c) or a nameless
// function call like a[2](x, y) or func(a, b)(c, d)
case RTokenType.OpenBrace:
currentOperationType = HandleOpenBrace(context, out errorType);
break;
case RTokenType.OpenCurlyBrace:
currentOperationType = HandleLambda(context, out errorType);
break;
case RTokenType.OpenSquareBracket:
case RTokenType.OpenDoubleSquareBracket:
currentOperationType = HandleSquareBrackets(context, out errorType);
break;
case RTokenType.Operator:
currentOperationType = HandleOperator(context, out errorType);
break;
case RTokenType.CloseBrace:
currentOperationType = OperationType.EndOfExpression;
endOfExpression = true;
break;
case RTokenType.CloseCurlyBrace:
case RTokenType.CloseSquareBracket:
case RTokenType.CloseDoubleSquareBracket:
currentOperationType = OperationType.EndOfExpression;
endOfExpression = true;
break;
case RTokenType.Comma:
case RTokenType.Semicolon:
currentOperationType = OperationType.EndOfExpression;
endOfExpression = true;
break;
case RTokenType.Keyword:
currentOperationType = HandleKeyword(context, out errorType);
endOfExpression = true;
break;
default:
errorType = ParseErrorType.UnexpectedToken;
errorLocation = ErrorLocation.Token;
break;
}
if (errorType == ParseErrorType.None && !IsConsistentOperationSequence(context, currentOperationType))
{
return(false);
}
if (errorType != ParseErrorType.None || endOfExpression)
{
break;
}
_previousOperationType = currentOperationType;
if (!endOfExpression)
{
endOfExpression = CheckEndOfExpression(context, currentOperationType);
}
}
if (errorType != ParseErrorType.None)
{
if (errorLocation == ErrorLocation.AfterToken)
{
context.AddError(new ParseError(errorType, ErrorLocation.AfterToken, tokens.PreviousToken));
}
else
{
context.AddError(new ParseError(errorType, ErrorLocation.Token, tokens.CurrentToken));
}
}
if (_operators.Count > 1)
{
// If there are still operators to process,
// construct final node. After this only sentil
// operator should be in the operators stack
// and a single final root node in the operand stack.
errorType = this.ProcessHigherPrecendenceOperators(context, Expression.Sentinel);
}
if (errorType != ParseErrorType.None)
{
if (errorType != ParseErrorType.LeftOperandExpected)
{
context.AddError(new ParseError(errorType, ErrorLocation.Token, GetErrorRange(context)));
}
// Although there may be errors such as incomplete function
// we still want to include the result into the tree since
// even in the code like 'func(,,, for(' we want intellisense
// and parameter to work.
if (_operands.Count == 1)
{
Content = _operands.Pop();
AppendChild(this.Content);
}
}
else
{
Debug.Assert(_operators.Count == 1);
// If operand stack ie empty and there is no error
// then the expression is empty.
if (_operands.Count > 0)
{
Debug.Assert(_operands.Count == 1);
Content = _operands.Pop();
AppendChild(Content);
}
}
return(true);
}
