public override bool Parse(ParseContext context, IAstNode parent) {
TokenStream<RToken> tokens = context.Tokens;
Debug.Assert(context.Tokens.CurrentToken.TokenType == RTokenType.Identifier ||
context.Tokens.CurrentToken.TokenType == RTokenType.String);
this.Identifier = RParser.ParseToken(context, this);
this.EqualsSign = RParser.ParseToken(context, this);
if (context.Tokens.CurrentToken.TokenType != RTokenType.Comma && context.Tokens.CurrentToken.TokenType != RTokenType.CloseBrace) {
Expression exp = new Expression(inGroup: true);
if (exp.Parse(context, this)) {
this.DefaultValue = exp;
}
} else {
this.DefaultValue = new NullExpression();
if (context.Tokens.IsEndOfStream()) {
context.AddError(new ParseError(ParseErrorType.ExpressionExpected, ErrorLocation.Token, context.Tokens.CurrentToken));
} else {
context.AddError(new ParseError(ParseErrorType.ExpressionExpected, ErrorLocation.Token, EqualsSign));
}
}
return base.Parse(context, parent);
}
public override bool Parse(ParseContext context, IAstNode parent) {
TokenStream<RToken> tokens = context.Tokens;
if (tokens.CurrentToken.IsVariableKind()) {
var v = new Variable();
v.Parse(context, this);
// Variables don't set parent since during complex
// exression parsing parent is determined by the
// expression parser based on precedence and grouping.
v.Parent = this;
this.Variable = v;
if (tokens.CurrentToken.IsKeywordText(context.TextProvider, "in")) {
this.InOperator = new TokenNode();
this.InOperator.Parse(context, this);
this.Expression = new Expression(inGroup: true);
if (this.Expression.Parse(context, this)) {
return base.Parse(context, parent);
}
} else {
context.AddError(new MissingItemParseError(ParseErrorType.InKeywordExpected, tokens.CurrentToken));
}
} else {
context.AddError(new MissingItemParseError(ParseErrorType.IndentifierExpected, tokens.PreviousToken));
}
return false;
}
public override bool Parse(ParseContext context, IAstNode parent) {
if (context.Tokens.CurrentToken.TokenType == RTokenType.OpenBrace) {
context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, context.Tokens.CurrentToken));
}
return false;
}
public override bool Parse(ParseContext context, IAstNode parent) {
// First parse base which should pick up keyword, braces, inner
// expression and either full or simple (single statement) scope
if (!base.Parse(context, parent)) {
return false;
}
// At this point we should be either at 'else' token or
// at the next statement. In the latter case we are done.
TokenStream<RToken> tokens = context.Tokens;
if (tokens.CurrentToken.IsKeywordText(context.TextProvider, "else")) {
bool allowLineBreak = AllowLineBreakBeforeElse(context);
if (!allowLineBreak) {
// Verify that there is no line break before the 'else'
if (context.Tokens.IsLineBreakAfter(context.TextProvider, tokens.Position - 1)) {
context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, tokens.CurrentToken));
return true;
}
}
this.Else = new KeywordScopeStatement(allowsSimpleScope: true);
return this.Else.Parse(context, this);
}
// Not at 'else' so we are done here
return true;
}
public static IScope ParseScope(ParseContext context, IAstNode parent, bool allowsSimpleScope, string terminatingKeyword)
{
TokenStream <RToken> tokens = context.Tokens;
IScope scope;
if (tokens.CurrentToken.TokenType == RTokenType.OpenCurlyBrace)
{
scope = new Scope(string.Empty);
if (scope.Parse(context, parent))
{
return(scope);
}
}
else if (allowsSimpleScope)
{
// Try simple on-line scope as in 'for(...) statement # comment'
scope = new SimpleScope(terminatingKeyword);
if (scope.Parse(context, parent))
{
return(scope);
}
}
else
{
context.AddError(new MissingItemParseError(ParseErrorType.OpenCurlyBraceExpected, tokens.PreviousToken));
}
return(null);
}
public static TokenNode ParseCloseBraceSequence(ParseContext context, IAstNode parent) {
TokenStream<RToken> tokens = context.Tokens;
if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace) {
return RParser.ParseToken(context, parent);
}
context.AddError(new MissingItemParseError(ParseErrorType.CloseBraceExpected, tokens.PreviousToken));
return null;
}
public static TokenNode ParseCloseBraceSequence(ParseContext context, IAstNode parent)
{
TokenStream <RToken> tokens = context.Tokens;
if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace)
{
return(RParser.ParseToken(context, parent));
}
context.AddError(new MissingItemParseError(ParseErrorType.CloseBraceExpected, tokens.PreviousToken));
return(null);
}
protected bool ParseSemicolon(ParseContext context, IAstNode parent) {
if (!context.Tokens.IsEndOfStream()) {
if (!context.Tokens.IsLineBreakAfter(context.TextProvider, context.Tokens.Position - 1)) {
if (context.Tokens.CurrentToken.TokenType == RTokenType.Semicolon) {
this.Semicolon = RParser.ParseToken(context, this);
} else {
context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, context.Tokens.CurrentToken));
return false;
}
}
}
return true;
}
public static TokenNode ParseOpenBraceSequence(ParseContext context, IAstNode parent) {
TokenStream<RToken> tokens = context.Tokens;
if (tokens.CurrentToken.TokenType == RTokenType.OpenBrace) {
TokenNode openBrace = new TokenNode();
openBrace.Parse(context, parent);
return openBrace;
} else {
context.AddError(new MissingItemParseError(ParseErrorType.OpenBraceExpected, tokens.PreviousToken));
}
return null;
}
public override bool Parse(ParseContext context, IAstNode parent) {
TokenStream<RToken> tokens = context.Tokens;
if (tokens.CurrentToken.TokenType == RTokenType.Identifier) {
this.VariableName = new TokenNode();
this.VariableName.Parse(context, this);
if (tokens.CurrentToken.IsKeywordText(context.TextProvider, "in")) {
this.InOperator = new TokenNode();
this.InOperator.Parse(context, this);
this.Expression = new Expression(inGroup: true);
if (this.Expression.Parse(context, this)) {
return base.Parse(context, parent);
}
} else {
context.AddError(new MissingItemParseError(ParseErrorType.InKeywordExpected, tokens.CurrentToken));
}
} else {
context.AddError(new MissingItemParseError(ParseErrorType.IndentifierExpected, tokens.PreviousToken));
}
return false;
}
public override bool Parse(ParseContext context, IAstNode parent) {
TokenStream<RToken> tokens = context.Tokens;
Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.OpenBrace);
this.OpenBrace = RParser.ParseToken(context, this);
this.Content = new Expression(inGroup: true);
this.Content.Parse(context, this);
if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace) {
this.CloseBrace = RParser.ParseToken(context, this);
} else {
context.AddError(new MissingItemParseError(ParseErrorType.CloseBraceExpected, tokens.PreviousToken));
}
return base.Parse(context, parent);
}
public static TokenNode ParseOpenBraceSequence(ParseContext context, IAstNode parent)
{
TokenStream <RToken> tokens = context.Tokens;
if (tokens.CurrentToken.TokenType == RTokenType.OpenBrace)
{
TokenNode openBrace = new TokenNode();
openBrace.Parse(context, parent);
return(openBrace);
}
else
{
context.AddError(new MissingItemParseError(ParseErrorType.OpenBraceExpected, tokens.PreviousToken));
}
return(null);
}
public override bool Parse(ParseContext context, IAstNode parent) {
TokenStream<RToken> tokens = context.Tokens;
Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.OpenSquareBracket ||
tokens.CurrentToken.TokenType == RTokenType.OpenDoubleSquareBracket);
this.LeftBrackets = RParser.ParseToken(context, this);
RTokenType terminatingTokenType = RParser.GetTerminatingTokenType(this.LeftBrackets.Token.TokenType);
this.Arguments = new ArgumentList(terminatingTokenType);
this.Arguments.Parse(context, this);
if (tokens.CurrentToken.TokenType == terminatingTokenType) {
this.RightBrackets = RParser.ParseToken(context, this);
} else {
context.AddError(new MissingItemParseError(ParseErrorType.CloseSquareBracketExpected, tokens.PreviousToken));
}
return base.Parse(context, parent);
}
public static IScope ParseScope(ParseContext context, IAstNode parent, bool allowsSimpleScope, string terminatingKeyword) {
TokenStream<RToken> tokens = context.Tokens;
IScope scope;
if (tokens.CurrentToken.TokenType == RTokenType.OpenCurlyBrace) {
scope = new Scope(string.Empty);
if (scope.Parse(context, parent)) {
return scope;
}
} else if (allowsSimpleScope) {
// Try simple on-line scope as in 'for(...) statement # comment'
scope = new SimpleScope(terminatingKeyword);
if (scope.Parse(context, parent)) {
return scope;
}
} else {
context.AddError(new MissingItemParseError(ParseErrorType.OpenCurlyBraceExpected, tokens.PreviousToken));
}
return null;
}
public override bool Parse(ParseContext context, IAstNode parent) {
if (ParseKeyword(context, this)) {
this.OpenBrace = RParser.ParseOpenBraceSequence(context, this);
if (this.OpenBrace != null) {
RToken token = context.Tokens.CurrentToken;
if (token.TokenType == RTokenType.Identifier || token.TokenType == RTokenType.String) {
this.Identifier = RParser.ParseToken(context, this);
this.CloseBrace = RParser.ParseCloseBraceSequence(context, this);
if (this.CloseBrace != null) {
this.Parent = parent;
return true;
}
} else {
context.AddError(new ParseError(ParseErrorType.IndentifierExpected, ErrorLocation.Token, context.Tokens.CurrentToken));
}
}
}
return false;
}
/// <summary>
/// Abstract factory
/// </summary>
public static IStatement CreateStatement(ParseContext context, IAstNode parent) {
RToken currentToken = context.Tokens.CurrentToken;
string keyword = context.TextProvider.GetText(currentToken);
IStatement statement = null;
switch (keyword) {
case "if":
statement = new If();
break;
case "for":
statement = new For();
break;
case "while":
statement = new KeywordExpressionScopeStatement();
break;
case "repeat":
statement = new KeywordScopeStatement(allowsSimpleScope: false);
break;
case "break":
case "next":
statement = new KeywordStatement();
break;
case "function":
statement = new FunctionStatement();
break;
default:
context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, currentToken));
break;
}
return statement;
}
protected override CommaSeparatedItem CreateItem(IAstNode parent, ParseContext context) {
RToken currentToken = context.Tokens.CurrentToken;
RToken nextToken = context.Tokens.NextToken;
switch (currentToken.TokenType) {
case RTokenType.Ellipsis:
return new EllipsisArgument();
case RTokenType.Comma:
return new MissingArgument();
case RTokenType.Identifier:
case RTokenType.String:
if (nextToken.TokenType == RTokenType.Operator && context.TextProvider.GetText(nextToken) == "=") {
return new NamedArgument();
}
break;
case RTokenType.Logical:
case RTokenType.Complex:
case RTokenType.NaN:
case RTokenType.Null:
case RTokenType.Number:
case RTokenType.Infinity:
if (nextToken.TokenType == RTokenType.Operator && context.TextProvider.GetText(nextToken) == "=") {
context.AddError(new ParseError(ParseErrorType.IndentifierExpected, ErrorLocation.Token, currentToken));
return new ErrorArgument(CollectErrorArgumentTokens(context));
}
break;
case RTokenType.CloseBrace:
return null; // no arguments supplied
}
return new ExpressionArgument();
}
private bool ParseExpression(ParseContext context) {
// https://en.wikipedia.org/wiki/Shunting-yard_algorithm
// http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm
// Instead of evaluating expressions like calculator would do,
// we create tree nodes with operator and its operands.
TokenStream<RToken> tokens = context.Tokens;
OperationType currentOperationType = OperationType.None;
ParseErrorType errorType = ParseErrorType.None;
ErrorLocation errorLocation = ErrorLocation.AfterToken;
bool endOfExpression = false;
// Push sentinel
_operators.Push(Expression.Sentinel);
while (!tokens.IsEndOfStream() && errorType == ParseErrorType.None && !endOfExpression) {
RToken token = tokens.CurrentToken;
switch (token.TokenType) {
// Terminal constants
case RTokenType.Number:
case RTokenType.Complex:
case RTokenType.Logical:
case RTokenType.String:
case RTokenType.Null:
case RTokenType.Missing:
case RTokenType.NaN:
case RTokenType.Infinity:
currentOperationType = HandleConstant(context);
break;
// Variables and function calls
case RTokenType.Identifier:
currentOperationType = HandleIdentifier(context);
break;
// Nested expression such as a*(b+c) or a nameless
// function call like a[2](x, y) or func(a, b)(c, d)
case RTokenType.OpenBrace:
currentOperationType = HandleOpenBrace(context, out errorType);
break;
case RTokenType.OpenCurlyBrace:
currentOperationType = HandleLambda(context, out errorType);
break;
case RTokenType.OpenSquareBracket:
case RTokenType.OpenDoubleSquareBracket:
currentOperationType = HandleSquareBrackets(context, out errorType);
break;
case RTokenType.Operator:
currentOperationType = HandleOperator(context, out errorType);
break;
case RTokenType.CloseBrace:
currentOperationType = OperationType.EndOfExpression;
endOfExpression = true;
break;
case RTokenType.CloseCurlyBrace:
case RTokenType.CloseSquareBracket:
case RTokenType.CloseDoubleSquareBracket:
currentOperationType = OperationType.EndOfExpression;
endOfExpression = true;
break;
case RTokenType.Comma:
case RTokenType.Semicolon:
currentOperationType = OperationType.EndOfExpression;
endOfExpression = true;
break;
case RTokenType.Keyword:
currentOperationType = HandleKeyword(context, out errorType);
endOfExpression = true;
break;
default:
errorType = ParseErrorType.UnexpectedToken;
errorLocation = ErrorLocation.Token;
break;
}
if (errorType == ParseErrorType.None && !IsConsistentOperationSequence(context, currentOperationType)) {
return false;
}
if (errorType != ParseErrorType.None || endOfExpression) {
break;
}
_previousOperationType = currentOperationType;
if (!endOfExpression) {
endOfExpression = CheckEndOfExpression(context, currentOperationType);
}
}
if (errorType != ParseErrorType.None) {
if (errorLocation == ErrorLocation.AfterToken) {
context.AddError(new ParseError(errorType, ErrorLocation.AfterToken, tokens.PreviousToken));
} else {
context.AddError(new ParseError(errorType, ErrorLocation.Token, tokens.CurrentToken));
}
}
if (_operators.Count > 1) {
// If there are still operators to process,
// construct final node. After this only sentil
// operator should be in the operators stack
// and a single final root node in the operand stack.
errorType = this.ProcessHigherPrecendenceOperators(context, Expression.Sentinel);
}
if (errorType != ParseErrorType.None) {
if (errorType != ParseErrorType.LeftOperandExpected) {
context.AddError(new ParseError(errorType, ErrorLocation.Token, GetErrorRange(context)));
}
// Although there may be errors such as incomplete function
// we still want to include the result into the tree since
// even in the code like 'func(,,, for(' we want intellisense
// and parameter to work.
if (_operands.Count == 1) {
Content = _operands.Pop();
AppendChild(this.Content);
}
} else {
Debug.Assert(_operators.Count == 1);
// If operand stack ie empty and there is no error
// then the expression is empty.
if (_operands.Count > 0) {
Debug.Assert(_operands.Count == 1);
Content = _operands.Pop();
AppendChild(Content);
}
}
return true;
}
private ParseErrorType MakeNode(ParseContext context) {
IOperator operatorNode = _operators.Pop();
IRValueNode rightOperand = this.SafeGetOperand();
if (rightOperand == null) {
// Oddly, no operands
return ParseErrorType.RightOperandExpected;
}
if (operatorNode.IsUnary) {
operatorNode.AppendChild(rightOperand);
operatorNode.RightOperand = rightOperand;
} else {
IRValueNode leftOperand = this.SafeGetOperand();
if (leftOperand == null) {
// Operand is missing in expression like x <- [].
// Operator on top of the stack is <- since [] was not
// successfully parsed. So we need to mark right operand
// as error token.
context.AddError(new ParseError(ParseErrorType.LeftOperandExpected, ErrorLocation.Token, GetIndexerOrFunctionErrorRange(context, operatorNode)));
return ParseErrorType.LeftOperandExpected;
}
operatorNode.LeftOperand = leftOperand;
operatorNode.RightOperand = rightOperand;
operatorNode.AppendChild(leftOperand);
operatorNode.AppendChild(rightOperand);
}
_operands.Push(operatorNode);
return ParseErrorType.None;
}
private bool IsConsistentOperationSequence(ParseContext context, OperationType currentOperationType) {
// Binary operator followed by another binary like 'a +/ b' is an error.
// 'func()()' or 'func() +' is allowed but 'func() operand' is not.
// Binary operator without anything behind it is an error;
if ((_previousOperationType == OperationType.Function && currentOperationType == OperationType.Operand) ||
(_previousOperationType == currentOperationType && currentOperationType != OperationType.Function)) {
switch (currentOperationType) {
case OperationType.Operand:
// 'operand operand' sequence is an error
context.AddError(new ParseError(ParseErrorType.OperatorExpected, ErrorLocation.Token, GetOperandErrorRange(context)));
break;
case OperationType.UnaryOperator:
// 'unary unary' and 'binary unary' are OK
return true;
default:
// 'operator operator' sequence is an error
context.AddError(new ParseError(ParseErrorType.RightOperandExpected, ErrorLocation.Token, GetOperatorErrorRange(context)));
break;
}
return false;
} else if (currentOperationType == OperationType.BinaryOperator && context.Tokens.IsEndOfStream()) {
// 'operator <EOF>' sequence is an error
context.AddError(new ParseError(ParseErrorType.RightOperandExpected, ErrorLocation.Token, GetOperatorErrorRange(context)));
return false;
} else if (_previousOperationType == OperationType.UnaryOperator && currentOperationType == OperationType.BinaryOperator) {
// unary followed by binary doesn't make sense
context.AddError(new ParseError(ParseErrorType.IndentifierExpected, ErrorLocation.Token, GetOperatorErrorRange(context)));
return false;
} else if (_previousOperationType == OperationType.BinaryOperator && currentOperationType == OperationType.EndOfExpression) {
// missing list selector: z$ }
context.AddError(new ParseError(ParseErrorType.RightOperandExpected, ErrorLocation.Token, GetErrorRange(context)));
return false;
}
return true;
}
public override bool Parse(ParseContext context, IAstNode parent) {
TokenStream<RToken> tokens = context.Tokens;
RToken currentToken = tokens.CurrentToken;
context.Scopes.Push(this);
if (!(this is GlobalScope) && currentToken.TokenType == RTokenType.OpenCurlyBrace) {
this.OpenCurlyBrace = RParser.ParseToken(context, this);
}
while (!tokens.IsEndOfStream()) {
currentToken = context.Tokens.CurrentToken;
switch (currentToken.TokenType) {
case RTokenType.CloseCurlyBrace:
if (this.OpenCurlyBrace != null) {
this.CloseCurlyBrace = RParser.ParseToken(context, this);
} else {
context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, currentToken));
context.Tokens.MoveToNextToken();
}
break;
case RTokenType.OpenCurlyBrace:
IScope scope = new Scope(string.Empty);
scope.Parse(context, this);
break;
default:
IStatement statement = Statement.Create(context, this, null);
if (statement != null) {
if (statement.Parse(context, this)) {
this.statements.Add(statement);
} else {
statement = null;
}
}
if (statement == null) {
if (!context.TextProvider.IsNewLineBeforePosition(context.Tokens.CurrentToken.Start)) {
// try recovering at the next line or past nearest
// semicolon or closing curly brace
tokens.MoveToNextLine(context.TextProvider,
(TokenStream<RToken> ts) => {
return ts.CurrentToken.TokenType == RTokenType.Semicolon ||
ts.NextToken.TokenType == RTokenType.CloseCurlyBrace;
});
} else {
tokens.MoveToNextToken();
}
}
break;
}
if (this.CloseCurlyBrace != null) {
break;
}
}
context.Scopes.Pop();
if (this.OpenCurlyBrace != null && this.CloseCurlyBrace == null) {
context.AddError(new MissingItemParseError(ParseErrorType.CloseCurlyBraceExpected, context.Tokens.PreviousToken));
}
// TODO: process content and fill out declared variables
// and functions and get data to the classifier for colorization.
return base.Parse(context, parent);
}
public override bool Parse(ParseContext context, IAstNode parent) {
TokenStream<RToken> tokens = context.Tokens;
Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.Keyword);
this.Keyword = RParser.ParseKeyword(context, this);
this.Text = context.TextProvider.GetText(this.Keyword);
if (tokens.CurrentToken.TokenType == RTokenType.OpenBrace) {
this.OpenBrace = RParser.ParseToken(context, this);
this.Arguments = new ArgumentList(RTokenType.CloseBrace);
this.Arguments.Parse(context, this);
if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace) {
this.CloseBrace = RParser.ParseToken(context, this);
this.Scope = RParser.ParseScope(context, this, allowsSimpleScope: true, terminatingKeyword: null);
if (this.Scope != null) {
return base.Parse(context, parent);
} else {
context.AddError(new ParseError(ParseErrorType.FunctionBodyExpected, ErrorLocation.Token, tokens.PreviousToken));
}
} else {
context.AddError(new ParseError(ParseErrorType.CloseBraceExpected, ErrorLocation.Token, tokens.CurrentToken));
}
} else {
context.AddError(new ParseError(ParseErrorType.OpenBraceExpected, ErrorLocation.Token, tokens.CurrentToken));
}
return false;
}
public override bool Parse(ParseContext context, IAstNode parent) {
TokenStream<RToken> tokens = context.Tokens;
Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.OpenBrace);
this.OpenBrace = RParser.ParseToken(context, this);
this.Arguments = new ArgumentList(RTokenType.CloseBrace);
bool argumentsParsed = this.Arguments.Parse(context, this);
if (argumentsParsed) {
if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace) {
this.CloseBrace = RParser.ParseToken(context, this);
}
}
if (!argumentsParsed || this.CloseBrace == null) {
CalculateVirtualEnd(context);
}
if (this.CloseBrace == null) {
context.AddError(new MissingItemParseError(ParseErrorType.CloseBraceExpected, tokens.PreviousToken));
}
return base.Parse(context, parent);
}