public override bool Parse(ParseContext context, IAstNode parent) { TokenStream<RToken> tokens = context.Tokens; Debug.Assert(context.Tokens.CurrentToken.TokenType == RTokenType.Identifier || context.Tokens.CurrentToken.TokenType == RTokenType.String); this.Identifier = RParser.ParseToken(context, this); this.EqualsSign = RParser.ParseToken(context, this); if (context.Tokens.CurrentToken.TokenType != RTokenType.Comma && context.Tokens.CurrentToken.TokenType != RTokenType.CloseBrace) { Expression exp = new Expression(inGroup: true); if (exp.Parse(context, this)) { this.DefaultValue = exp; } } else { this.DefaultValue = new NullExpression(); if (context.Tokens.IsEndOfStream()) { context.AddError(new ParseError(ParseErrorType.ExpressionExpected, ErrorLocation.Token, context.Tokens.CurrentToken)); } else { context.AddError(new ParseError(ParseErrorType.ExpressionExpected, ErrorLocation.Token, EqualsSign)); } } return base.Parse(context, parent); }
public override bool Parse(ParseContext context, IAstNode parent) { TokenStream<RToken> tokens = context.Tokens; if (tokens.CurrentToken.IsVariableKind()) { var v = new Variable(); v.Parse(context, this); // Variables don't set parent since during complex // exression parsing parent is determined by the // expression parser based on precedence and grouping. v.Parent = this; this.Variable = v; if (tokens.CurrentToken.IsKeywordText(context.TextProvider, "in")) { this.InOperator = new TokenNode(); this.InOperator.Parse(context, this); this.Expression = new Expression(inGroup: true); if (this.Expression.Parse(context, this)) { return base.Parse(context, parent); } } else { context.AddError(new MissingItemParseError(ParseErrorType.InKeywordExpected, tokens.CurrentToken)); } } else { context.AddError(new MissingItemParseError(ParseErrorType.IndentifierExpected, tokens.PreviousToken)); } return false; }
public override bool Parse(ParseContext context, IAstNode parent) { if (context.Tokens.CurrentToken.TokenType == RTokenType.OpenBrace) { context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, context.Tokens.CurrentToken)); } return false; }
public override bool Parse(ParseContext context, IAstNode parent) { // First parse base which should pick up keyword, braces, inner // expression and either full or simple (single statement) scope if (!base.Parse(context, parent)) { return false; } // At this point we should be either at 'else' token or // at the next statement. In the latter case we are done. TokenStream<RToken> tokens = context.Tokens; if (tokens.CurrentToken.IsKeywordText(context.TextProvider, "else")) { bool allowLineBreak = AllowLineBreakBeforeElse(context); if (!allowLineBreak) { // Verify that there is no line break before the 'else' if (context.Tokens.IsLineBreakAfter(context.TextProvider, tokens.Position - 1)) { context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, tokens.CurrentToken)); return true; } } this.Else = new KeywordScopeStatement(allowsSimpleScope: true); return this.Else.Parse(context, this); } // Not at 'else' so we are done here return true; }
public static IScope ParseScope(ParseContext context, IAstNode parent, bool allowsSimpleScope, string terminatingKeyword) { TokenStream <RToken> tokens = context.Tokens; IScope scope; if (tokens.CurrentToken.TokenType == RTokenType.OpenCurlyBrace) { scope = new Scope(string.Empty); if (scope.Parse(context, parent)) { return(scope); } } else if (allowsSimpleScope) { // Try simple on-line scope as in 'for(...) statement # comment' scope = new SimpleScope(terminatingKeyword); if (scope.Parse(context, parent)) { return(scope); } } else { context.AddError(new MissingItemParseError(ParseErrorType.OpenCurlyBraceExpected, tokens.PreviousToken)); } return(null); }
public static TokenNode ParseCloseBraceSequence(ParseContext context, IAstNode parent) { TokenStream<RToken> tokens = context.Tokens; if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace) { return RParser.ParseToken(context, parent); } context.AddError(new MissingItemParseError(ParseErrorType.CloseBraceExpected, tokens.PreviousToken)); return null; }
public static TokenNode ParseCloseBraceSequence(ParseContext context, IAstNode parent) { TokenStream <RToken> tokens = context.Tokens; if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace) { return(RParser.ParseToken(context, parent)); } context.AddError(new MissingItemParseError(ParseErrorType.CloseBraceExpected, tokens.PreviousToken)); return(null); }
protected bool ParseSemicolon(ParseContext context, IAstNode parent) { if (!context.Tokens.IsEndOfStream()) { if (!context.Tokens.IsLineBreakAfter(context.TextProvider, context.Tokens.Position - 1)) { if (context.Tokens.CurrentToken.TokenType == RTokenType.Semicolon) { this.Semicolon = RParser.ParseToken(context, this); } else { context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, context.Tokens.CurrentToken)); return false; } } } return true; }
public static TokenNode ParseOpenBraceSequence(ParseContext context, IAstNode parent) { TokenStream<RToken> tokens = context.Tokens; if (tokens.CurrentToken.TokenType == RTokenType.OpenBrace) { TokenNode openBrace = new TokenNode(); openBrace.Parse(context, parent); return openBrace; } else { context.AddError(new MissingItemParseError(ParseErrorType.OpenBraceExpected, tokens.PreviousToken)); } return null; }
public override bool Parse(ParseContext context, IAstNode parent) { TokenStream<RToken> tokens = context.Tokens; if (tokens.CurrentToken.TokenType == RTokenType.Identifier) { this.VariableName = new TokenNode(); this.VariableName.Parse(context, this); if (tokens.CurrentToken.IsKeywordText(context.TextProvider, "in")) { this.InOperator = new TokenNode(); this.InOperator.Parse(context, this); this.Expression = new Expression(inGroup: true); if (this.Expression.Parse(context, this)) { return base.Parse(context, parent); } } else { context.AddError(new MissingItemParseError(ParseErrorType.InKeywordExpected, tokens.CurrentToken)); } } else { context.AddError(new MissingItemParseError(ParseErrorType.IndentifierExpected, tokens.PreviousToken)); } return false; }
public override bool Parse(ParseContext context, IAstNode parent) { TokenStream<RToken> tokens = context.Tokens; Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.OpenBrace); this.OpenBrace = RParser.ParseToken(context, this); this.Content = new Expression(inGroup: true); this.Content.Parse(context, this); if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace) { this.CloseBrace = RParser.ParseToken(context, this); } else { context.AddError(new MissingItemParseError(ParseErrorType.CloseBraceExpected, tokens.PreviousToken)); } return base.Parse(context, parent); }
public static TokenNode ParseOpenBraceSequence(ParseContext context, IAstNode parent) { TokenStream <RToken> tokens = context.Tokens; if (tokens.CurrentToken.TokenType == RTokenType.OpenBrace) { TokenNode openBrace = new TokenNode(); openBrace.Parse(context, parent); return(openBrace); } else { context.AddError(new MissingItemParseError(ParseErrorType.OpenBraceExpected, tokens.PreviousToken)); } return(null); }
public override bool Parse(ParseContext context, IAstNode parent) { TokenStream<RToken> tokens = context.Tokens; Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.OpenSquareBracket || tokens.CurrentToken.TokenType == RTokenType.OpenDoubleSquareBracket); this.LeftBrackets = RParser.ParseToken(context, this); RTokenType terminatingTokenType = RParser.GetTerminatingTokenType(this.LeftBrackets.Token.TokenType); this.Arguments = new ArgumentList(terminatingTokenType); this.Arguments.Parse(context, this); if (tokens.CurrentToken.TokenType == terminatingTokenType) { this.RightBrackets = RParser.ParseToken(context, this); } else { context.AddError(new MissingItemParseError(ParseErrorType.CloseSquareBracketExpected, tokens.PreviousToken)); } return base.Parse(context, parent); }
public static IScope ParseScope(ParseContext context, IAstNode parent, bool allowsSimpleScope, string terminatingKeyword) { TokenStream<RToken> tokens = context.Tokens; IScope scope; if (tokens.CurrentToken.TokenType == RTokenType.OpenCurlyBrace) { scope = new Scope(string.Empty); if (scope.Parse(context, parent)) { return scope; } } else if (allowsSimpleScope) { // Try simple on-line scope as in 'for(...) statement # comment' scope = new SimpleScope(terminatingKeyword); if (scope.Parse(context, parent)) { return scope; } } else { context.AddError(new MissingItemParseError(ParseErrorType.OpenCurlyBraceExpected, tokens.PreviousToken)); } return null; }
public override bool Parse(ParseContext context, IAstNode parent) { if (ParseKeyword(context, this)) { this.OpenBrace = RParser.ParseOpenBraceSequence(context, this); if (this.OpenBrace != null) { RToken token = context.Tokens.CurrentToken; if (token.TokenType == RTokenType.Identifier || token.TokenType == RTokenType.String) { this.Identifier = RParser.ParseToken(context, this); this.CloseBrace = RParser.ParseCloseBraceSequence(context, this); if (this.CloseBrace != null) { this.Parent = parent; return true; } } else { context.AddError(new ParseError(ParseErrorType.IndentifierExpected, ErrorLocation.Token, context.Tokens.CurrentToken)); } } } return false; }
/// <summary> /// Abstract factory /// </summary> public static IStatement CreateStatement(ParseContext context, IAstNode parent) { RToken currentToken = context.Tokens.CurrentToken; string keyword = context.TextProvider.GetText(currentToken); IStatement statement = null; switch (keyword) { case "if": statement = new If(); break; case "for": statement = new For(); break; case "while": statement = new KeywordExpressionScopeStatement(); break; case "repeat": statement = new KeywordScopeStatement(allowsSimpleScope: false); break; case "break": case "next": statement = new KeywordStatement(); break; case "function": statement = new FunctionStatement(); break; default: context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, currentToken)); break; } return statement; }
protected override CommaSeparatedItem CreateItem(IAstNode parent, ParseContext context) { RToken currentToken = context.Tokens.CurrentToken; RToken nextToken = context.Tokens.NextToken; switch (currentToken.TokenType) { case RTokenType.Ellipsis: return new EllipsisArgument(); case RTokenType.Comma: return new MissingArgument(); case RTokenType.Identifier: case RTokenType.String: if (nextToken.TokenType == RTokenType.Operator && context.TextProvider.GetText(nextToken) == "=") { return new NamedArgument(); } break; case RTokenType.Logical: case RTokenType.Complex: case RTokenType.NaN: case RTokenType.Null: case RTokenType.Number: case RTokenType.Infinity: if (nextToken.TokenType == RTokenType.Operator && context.TextProvider.GetText(nextToken) == "=") { context.AddError(new ParseError(ParseErrorType.IndentifierExpected, ErrorLocation.Token, currentToken)); return new ErrorArgument(CollectErrorArgumentTokens(context)); } break; case RTokenType.CloseBrace: return null; // no arguments supplied } return new ExpressionArgument(); }
private bool ParseExpression(ParseContext context) { // https://en.wikipedia.org/wiki/Shunting-yard_algorithm // http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm // Instead of evaluating expressions like calculator would do, // we create tree nodes with operator and its operands. TokenStream<RToken> tokens = context.Tokens; OperationType currentOperationType = OperationType.None; ParseErrorType errorType = ParseErrorType.None; ErrorLocation errorLocation = ErrorLocation.AfterToken; bool endOfExpression = false; // Push sentinel _operators.Push(Expression.Sentinel); while (!tokens.IsEndOfStream() && errorType == ParseErrorType.None && !endOfExpression) { RToken token = tokens.CurrentToken; switch (token.TokenType) { // Terminal constants case RTokenType.Number: case RTokenType.Complex: case RTokenType.Logical: case RTokenType.String: case RTokenType.Null: case RTokenType.Missing: case RTokenType.NaN: case RTokenType.Infinity: currentOperationType = HandleConstant(context); break; // Variables and function calls case RTokenType.Identifier: currentOperationType = HandleIdentifier(context); break; // Nested expression such as a*(b+c) or a nameless // function call like a[2](x, y) or func(a, b)(c, d) case RTokenType.OpenBrace: currentOperationType = HandleOpenBrace(context, out errorType); break; case RTokenType.OpenCurlyBrace: currentOperationType = HandleLambda(context, out errorType); break; case RTokenType.OpenSquareBracket: case RTokenType.OpenDoubleSquareBracket: currentOperationType = HandleSquareBrackets(context, out errorType); break; case RTokenType.Operator: currentOperationType = HandleOperator(context, out errorType); break; case RTokenType.CloseBrace: currentOperationType = OperationType.EndOfExpression; endOfExpression = true; break; case RTokenType.CloseCurlyBrace: case RTokenType.CloseSquareBracket: case RTokenType.CloseDoubleSquareBracket: currentOperationType = OperationType.EndOfExpression; endOfExpression = true; break; case RTokenType.Comma: case RTokenType.Semicolon: currentOperationType = OperationType.EndOfExpression; endOfExpression = true; break; case RTokenType.Keyword: currentOperationType = HandleKeyword(context, out errorType); endOfExpression = true; break; default: errorType = ParseErrorType.UnexpectedToken; errorLocation = ErrorLocation.Token; break; } if (errorType == ParseErrorType.None && !IsConsistentOperationSequence(context, currentOperationType)) { return false; } if (errorType != ParseErrorType.None || endOfExpression) { break; } _previousOperationType = currentOperationType; if (!endOfExpression) { endOfExpression = CheckEndOfExpression(context, currentOperationType); } } if (errorType != ParseErrorType.None) { if (errorLocation == ErrorLocation.AfterToken) { context.AddError(new ParseError(errorType, ErrorLocation.AfterToken, tokens.PreviousToken)); } else { context.AddError(new ParseError(errorType, ErrorLocation.Token, tokens.CurrentToken)); } } if (_operators.Count > 1) { // If there are still operators to process, // construct final node. After this only sentil // operator should be in the operators stack // and a single final root node in the operand stack. errorType = this.ProcessHigherPrecendenceOperators(context, Expression.Sentinel); } if (errorType != ParseErrorType.None) { if (errorType != ParseErrorType.LeftOperandExpected) { context.AddError(new ParseError(errorType, ErrorLocation.Token, GetErrorRange(context))); } // Although there may be errors such as incomplete function // we still want to include the result into the tree since // even in the code like 'func(,,, for(' we want intellisense // and parameter to work. if (_operands.Count == 1) { Content = _operands.Pop(); AppendChild(this.Content); } } else { Debug.Assert(_operators.Count == 1); // If operand stack ie empty and there is no error // then the expression is empty. if (_operands.Count > 0) { Debug.Assert(_operands.Count == 1); Content = _operands.Pop(); AppendChild(Content); } } return true; }
private ParseErrorType MakeNode(ParseContext context) { IOperator operatorNode = _operators.Pop(); IRValueNode rightOperand = this.SafeGetOperand(); if (rightOperand == null) { // Oddly, no operands return ParseErrorType.RightOperandExpected; } if (operatorNode.IsUnary) { operatorNode.AppendChild(rightOperand); operatorNode.RightOperand = rightOperand; } else { IRValueNode leftOperand = this.SafeGetOperand(); if (leftOperand == null) { // Operand is missing in expression like x <- []. // Operator on top of the stack is <- since [] was not // successfully parsed. So we need to mark right operand // as error token. context.AddError(new ParseError(ParseErrorType.LeftOperandExpected, ErrorLocation.Token, GetIndexerOrFunctionErrorRange(context, operatorNode))); return ParseErrorType.LeftOperandExpected; } operatorNode.LeftOperand = leftOperand; operatorNode.RightOperand = rightOperand; operatorNode.AppendChild(leftOperand); operatorNode.AppendChild(rightOperand); } _operands.Push(operatorNode); return ParseErrorType.None; }
private bool IsConsistentOperationSequence(ParseContext context, OperationType currentOperationType) { // Binary operator followed by another binary like 'a +/ b' is an error. // 'func()()' or 'func() +' is allowed but 'func() operand' is not. // Binary operator without anything behind it is an error; if ((_previousOperationType == OperationType.Function && currentOperationType == OperationType.Operand) || (_previousOperationType == currentOperationType && currentOperationType != OperationType.Function)) { switch (currentOperationType) { case OperationType.Operand: // 'operand operand' sequence is an error context.AddError(new ParseError(ParseErrorType.OperatorExpected, ErrorLocation.Token, GetOperandErrorRange(context))); break; case OperationType.UnaryOperator: // 'unary unary' and 'binary unary' are OK return true; default: // 'operator operator' sequence is an error context.AddError(new ParseError(ParseErrorType.RightOperandExpected, ErrorLocation.Token, GetOperatorErrorRange(context))); break; } return false; } else if (currentOperationType == OperationType.BinaryOperator && context.Tokens.IsEndOfStream()) { // 'operator <EOF>' sequence is an error context.AddError(new ParseError(ParseErrorType.RightOperandExpected, ErrorLocation.Token, GetOperatorErrorRange(context))); return false; } else if (_previousOperationType == OperationType.UnaryOperator && currentOperationType == OperationType.BinaryOperator) { // unary followed by binary doesn't make sense context.AddError(new ParseError(ParseErrorType.IndentifierExpected, ErrorLocation.Token, GetOperatorErrorRange(context))); return false; } else if (_previousOperationType == OperationType.BinaryOperator && currentOperationType == OperationType.EndOfExpression) { // missing list selector: z$ } context.AddError(new ParseError(ParseErrorType.RightOperandExpected, ErrorLocation.Token, GetErrorRange(context))); return false; } return true; }
public override bool Parse(ParseContext context, IAstNode parent) { TokenStream<RToken> tokens = context.Tokens; RToken currentToken = tokens.CurrentToken; context.Scopes.Push(this); if (!(this is GlobalScope) && currentToken.TokenType == RTokenType.OpenCurlyBrace) { this.OpenCurlyBrace = RParser.ParseToken(context, this); } while (!tokens.IsEndOfStream()) { currentToken = context.Tokens.CurrentToken; switch (currentToken.TokenType) { case RTokenType.CloseCurlyBrace: if (this.OpenCurlyBrace != null) { this.CloseCurlyBrace = RParser.ParseToken(context, this); } else { context.AddError(new ParseError(ParseErrorType.UnexpectedToken, ErrorLocation.Token, currentToken)); context.Tokens.MoveToNextToken(); } break; case RTokenType.OpenCurlyBrace: IScope scope = new Scope(string.Empty); scope.Parse(context, this); break; default: IStatement statement = Statement.Create(context, this, null); if (statement != null) { if (statement.Parse(context, this)) { this.statements.Add(statement); } else { statement = null; } } if (statement == null) { if (!context.TextProvider.IsNewLineBeforePosition(context.Tokens.CurrentToken.Start)) { // try recovering at the next line or past nearest // semicolon or closing curly brace tokens.MoveToNextLine(context.TextProvider, (TokenStream<RToken> ts) => { return ts.CurrentToken.TokenType == RTokenType.Semicolon || ts.NextToken.TokenType == RTokenType.CloseCurlyBrace; }); } else { tokens.MoveToNextToken(); } } break; } if (this.CloseCurlyBrace != null) { break; } } context.Scopes.Pop(); if (this.OpenCurlyBrace != null && this.CloseCurlyBrace == null) { context.AddError(new MissingItemParseError(ParseErrorType.CloseCurlyBraceExpected, context.Tokens.PreviousToken)); } // TODO: process content and fill out declared variables // and functions and get data to the classifier for colorization. return base.Parse(context, parent); }
public override bool Parse(ParseContext context, IAstNode parent) { TokenStream<RToken> tokens = context.Tokens; Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.Keyword); this.Keyword = RParser.ParseKeyword(context, this); this.Text = context.TextProvider.GetText(this.Keyword); if (tokens.CurrentToken.TokenType == RTokenType.OpenBrace) { this.OpenBrace = RParser.ParseToken(context, this); this.Arguments = new ArgumentList(RTokenType.CloseBrace); this.Arguments.Parse(context, this); if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace) { this.CloseBrace = RParser.ParseToken(context, this); this.Scope = RParser.ParseScope(context, this, allowsSimpleScope: true, terminatingKeyword: null); if (this.Scope != null) { return base.Parse(context, parent); } else { context.AddError(new ParseError(ParseErrorType.FunctionBodyExpected, ErrorLocation.Token, tokens.PreviousToken)); } } else { context.AddError(new ParseError(ParseErrorType.CloseBraceExpected, ErrorLocation.Token, tokens.CurrentToken)); } } else { context.AddError(new ParseError(ParseErrorType.OpenBraceExpected, ErrorLocation.Token, tokens.CurrentToken)); } return false; }
public override bool Parse(ParseContext context, IAstNode parent) { TokenStream<RToken> tokens = context.Tokens; Debug.Assert(tokens.CurrentToken.TokenType == RTokenType.OpenBrace); this.OpenBrace = RParser.ParseToken(context, this); this.Arguments = new ArgumentList(RTokenType.CloseBrace); bool argumentsParsed = this.Arguments.Parse(context, this); if (argumentsParsed) { if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace) { this.CloseBrace = RParser.ParseToken(context, this); } } if (!argumentsParsed || this.CloseBrace == null) { CalculateVirtualEnd(context); } if (this.CloseBrace == null) { context.AddError(new MissingItemParseError(ParseErrorType.CloseBraceExpected, tokens.PreviousToken)); } return base.Parse(context, parent); }