protected void ForceEndOfFile() { _currentToken = SyntaxFactory.Token(SyntaxKind.EndOfFileToken); }
private InterpolationSyntax ParseInterpolation( string text, Lexer.Interpolation interpolation, bool isVerbatim ) { SyntaxToken openBraceToken; ExpressionSyntax expression; InterpolationAlignmentClauseSyntax alignment = null; InterpolationFormatClauseSyntax format = null; var closeBraceToken = interpolation.CloseBraceMissing ? SyntaxFactory.MissingToken(SyntaxKind.CloseBraceToken) : SyntaxFactory.Token(SyntaxKind.CloseBraceToken); var parsedText = Substring( text, interpolation.OpenBracePosition, interpolation.HasColon ? interpolation.ColonPosition - 1 : interpolation.CloseBracePosition - 1 ); using ( var tempLexer = new Lexer( Text.SourceText.From(parsedText), this.Options, allowPreprocessorDirectives: false, interpolationFollowedByColon: interpolation.HasColon ) ) { // TODO: some of the trivia in the interpolation maybe should be trailing trivia of the openBraceToken using (var tempParser = new LanguageParser(tempLexer, null, null)) { SyntaxToken commaToken = null; ExpressionSyntax alignmentExpression = null; tempParser.ParseInterpolationStart( out openBraceToken, out expression, out commaToken, out alignmentExpression ); if (alignmentExpression != null) { alignment = SyntaxFactory.InterpolationAlignmentClause( commaToken, alignmentExpression ); } var extraTrivia = tempParser.CurrentToken.GetLeadingTrivia(); if (interpolation.HasColon) { var colonToken = SyntaxFactory .Token(SyntaxKind.ColonToken) .TokenWithLeadingTrivia(extraTrivia); var formatText = Substring( text, interpolation.ColonPosition + 1, interpolation.FormatEndPosition ); var formatString = MakeStringToken( formatText, formatText, isVerbatim, SyntaxKind.InterpolatedStringTextToken ); format = SyntaxFactory.InterpolationFormatClause(colonToken, formatString); } else { // Move the leading trivia from the insertion's EOF token to the following token. closeBraceToken = closeBraceToken.TokenWithLeadingTrivia(extraTrivia); } } } var result = SyntaxFactory.Interpolation( openBraceToken, expression, alignment, format, closeBraceToken ); Debug.Assert( Substring(text, interpolation.OpenBracePosition, interpolation.LastPosition) == result.ToFullString() ); // yield from text equals yield from node return(result); }
private ExpressionSyntax ParseInterpolatedStringToken() { // We don't want to make the scanner stateful (between tokens) if we can possibly avoid it. // The approach implemented here is // // (1) Scan the whole interpolated string literal as a single token. Now the statefulness of // the scanner (to match { }'s) is limited to its behavior while scanning a single token. // // (2) When the parser gets such a token, here, it spins up another scanner / parser on each of // the holes and builds a tree for the whole thing (resulting in an InterpolatedStringExpressionSyntax). // // (3) The parser discards the original token and replaces it with this tree. (In other words, // it replaces one token with a different set of tokens that have already been parsed) // // (4) On an incremental change, we widen the invalidated region to include any enclosing interpolated // string nonterminal so that we never reuse tokens inside a changed interpolated string. // // This has the secondary advantage that it can reasonably be specified. // // The substitution will end up being invisible to external APIs and clients such as the IDE, as // they have no way to ask for the stream of tokens before parsing. // var originalToken = this.EatToken(); var originalText = originalToken.ValueText; // this is actually the source text Debug.Assert(originalText[0] == '$' || originalText[0] == '@'); var isAltInterpolatedVerbatim = originalText.Length > 2 && originalText[0] == '@'; // @$ var isVerbatim = isAltInterpolatedVerbatim || (originalText.Length > 2 && originalText[1] == '@'); Debug.Assert(originalToken.Kind == SyntaxKind.InterpolatedStringToken); var interpolations = ArrayBuilder <Lexer.Interpolation> .GetInstance(); SyntaxDiagnosticInfo error = null; bool closeQuoteMissing; using ( var tempLexer = new Lexer( Text.SourceText.From(originalText), this.Options, allowPreprocessorDirectives: false ) ) { // compute the positions of the interpolations in the original string literal, and also compute/preserve // lexical errors var info = default(Lexer.TokenInfo); tempLexer.ScanInterpolatedStringLiteralTop( interpolations, isVerbatim, ref info, ref error, out closeQuoteMissing ); } // Make a token for the open quote $" or $@" or @$" var openQuoteIndex = isVerbatim ? 2 : 1; Debug.Assert(originalText[openQuoteIndex] == '"'); var openQuoteKind = isVerbatim ? SyntaxKind.InterpolatedVerbatimStringStartToken // $@ or @$ : SyntaxKind.InterpolatedStringStartToken; // $ var openQuoteText = isAltInterpolatedVerbatim ? "@$\"" : isVerbatim ? "$@\"" : "$\""; var openQuote = SyntaxFactory.Token( originalToken.GetLeadingTrivia(), openQuoteKind, openQuoteText, openQuoteText, trailing: null ); if (isAltInterpolatedVerbatim) { openQuote = CheckFeatureAvailability( openQuote, MessageID.IDS_FeatureAltInterpolatedVerbatimStrings ); } // Make a token for the close quote " (even if it was missing) var closeQuoteIndex = closeQuoteMissing ? originalText.Length : originalText.Length - 1; Debug.Assert(closeQuoteMissing || originalText[closeQuoteIndex] == '"'); var closeQuote = closeQuoteMissing ? SyntaxFactory .MissingToken(SyntaxKind.InterpolatedStringEndToken) .TokenWithTrailingTrivia(originalToken.GetTrailingTrivia()) : SyntaxFactory.Token( null, SyntaxKind.InterpolatedStringEndToken, originalToken.GetTrailingTrivia() ); var builder = _pool.Allocate <InterpolatedStringContentSyntax>(); if (interpolations.Count == 0) { // In the special case when there are no interpolations, we just construct a format string // with no inserts. We must still use String.Format to get its handling of escapes such as {{, // so we still treat it as a composite format string. var text = Substring(originalText, openQuoteIndex + 1, closeQuoteIndex - 1); if (text.Length > 0) { var token = MakeStringToken( text, text, isVerbatim, SyntaxKind.InterpolatedStringTextToken ); builder.Add(SyntaxFactory.InterpolatedStringText(token)); } } else { for (int i = 0; i < interpolations.Count; i++) { var interpolation = interpolations[i]; // Add a token for text preceding the interpolation var text = Substring( originalText, (i == 0) ? (openQuoteIndex + 1) : (interpolations[i - 1].CloseBracePosition + 1), interpolation.OpenBracePosition - 1 ); if (text.Length > 0) { var token = MakeStringToken( text, text, isVerbatim, SyntaxKind.InterpolatedStringTextToken ); builder.Add(SyntaxFactory.InterpolatedStringText(token)); } // Add an interpolation var interp = ParseInterpolation(originalText, interpolation, isVerbatim); builder.Add(interp); } // Add a token for text following the last interpolation var lastText = Substring( originalText, interpolations[interpolations.Count - 1].CloseBracePosition + 1, closeQuoteIndex - 1 ); if (lastText.Length > 0) { var token = MakeStringToken( lastText, lastText, isVerbatim, SyntaxKind.InterpolatedStringTextToken ); builder.Add(SyntaxFactory.InterpolatedStringText(token)); } } interpolations.Free(); var result = SyntaxFactory.InterpolatedStringExpression(openQuote, builder, closeQuote); _pool.Free(builder); if (error != null) { result = result.WithDiagnosticsGreen(new[] { error }); } Debug.Assert(originalToken.ToFullString() == result.ToFullString()); // yield from text equals yield from node return(CheckFeatureAvailability(result, MessageID.IDS_FeatureInterpolatedStrings)); }
private ExpressionSyntax ParseSubExpression(uint precedence, bool allowDeclarationExpressionAtTheBeginning = true, bool contextRequiresVariable = false) { ExpressionSyntax leftOperand = null; uint newPrecedence = 0; SyntaxKind opKind = SyntaxKind.None; // all of these are tokens that start statements and are invalid // to start a expression with. if we see one, then we must have // something like: // // return // if (... // parse out a missing name node for the expression, and keep on going var tk = this.CurrentToken.Kind; if (IsInvalidSubExpression(tk)) { return(this.AddError(this.CreateMissingIdentifierName(), ErrorCode.ERR_InvalidExprTerm, SyntaxKindFacts.GetText(tk))); } // No left operand, so we need to parse one -- possibly preceded by a // unary operator. if (IsExpectedPrefixUnaryOperator(tk)) { opKind = SyntaxKindFacts.GetPrefixUnaryExpression(tk); newPrecedence = GetPrecedence(opKind); var opToken = this.EatToken(); var operand = this.ParseSubExpression(newPrecedence); leftOperand = _syntaxFactory.PrefixUnaryExpression(opKind, opToken, operand); } else { // Not a unary operator - get a primary expression. leftOperand = this.ParseTerm(precedence, allowDeclarationExpressionAtTheBeginning, contextRequiresVariable); } while (true) { // We either have a binary operator here, or we're finished. tk = this.CurrentToken.Kind; if (!IsExpectedBinaryOperator(tk)) { break; } opKind = SyntaxKindFacts.GetBinaryExpression(tk); newPrecedence = GetPrecedence(opKind); Debug.Assert(newPrecedence > 0); // All binary operators must have precedence > 0! // check for >> or >>= or >>> or >>>= bool doubleOp = false; bool doubleOpPlusOne = false; if (tk == SyntaxKind.GreaterThanToken && (this.PeekToken(1).Kind == SyntaxKind.GreaterThanToken) && (this.PeekToken(2).Kind == SyntaxKind.GreaterThanToken || this.PeekToken(2).Kind == SyntaxKind.GreaterThanEqualsToken)) { // check to see if they really are adjacent if (this.CurrentToken.GetTrailingTriviaWidth() == 0 && this.PeekToken(1).GetLeadingTriviaWidth() == 0 && this.PeekToken(1).GetTrailingTriviaWidth() == 0 && this.PeekToken(2).GetLeadingTriviaWidth() == 0 ) { opKind = SyntaxKindFacts.GetBinaryExpression(this.PeekToken(2).Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken); newPrecedence = GetPrecedence(opKind); doubleOp = true; doubleOpPlusOne = true; } } else if (tk == SyntaxKind.GreaterThanToken && (this.PeekToken(1).Kind == SyntaxKind.GreaterThanToken || this.PeekToken(1).Kind == SyntaxKind.GreaterThanEqualsToken)) { // check to see if they really are adjacent if (this.CurrentToken.GetTrailingTriviaWidth() == 0 && this.PeekToken(1).GetLeadingTriviaWidth() == 0) { opKind = SyntaxKindFacts.GetBinaryExpression(this.PeekToken(1).Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken); newPrecedence = GetPrecedence(opKind); doubleOp = true; } } // Check the precedence to see if we should "take" this operator if (newPrecedence < precedence) { break; } // Same precedence, but not right-associative -- deal with this "later" if ((newPrecedence == precedence) && !IsRightAssociative(opKind)) { break; } // Precedence is okay, so we'll "take" this operator. var opToken = this.EatToken(); if (doubleOp) { // combine tokens into a single token var opToken2 = this.EatToken(); if (doubleOpPlusOne) { var opToken3 = this.EatToken(); var kind = opToken3.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken; opToken = SyntaxFactory.Token(opToken.GetLeadingTrivia(), kind, opToken3.GetTrailingTrivia()); } else { var kind = opToken2.Kind == SyntaxKind.GreaterThanToken ? SyntaxKind.GreaterThanGreaterThanToken : SyntaxKind.GreaterThanGreaterThanEqualsToken; opToken = SyntaxFactory.Token(opToken.GetLeadingTrivia(), kind, opToken2.GetTrailingTrivia()); } } if (opKind == SyntaxKind.InstanceOfExpression) { leftOperand = _syntaxFactory.BinaryExpression(opKind, leftOperand, opToken, this.ParseTypeCore(parentIsParameter: false, isInstanceOfOrAs: true, expectSizes: false, isArrayCreation: false)); } else { leftOperand = _syntaxFactory.BinaryExpression(opKind, leftOperand, opToken, this.ParseSubExpression(newPrecedence)); } } // From the language spec: // // conditional-expression: // null-coalescing-expression // null-coalescing-expression ? expression : expression // // Only take the ternary if we're at a precedence less than the null coelescing // expression. var nullCoalescingPrecedence = 2U; //GetPrecedence(SyntaxKind.CoalesceExpression); if (tk == SyntaxKind.QuestionToken && precedence < 2) { var questionToken = this.EatToken(); var colonLeft = this.ParseSubExpression(nullCoalescingPrecedence - 1); var colon = this.EatToken(SyntaxKind.ColonToken); var colonRight = this.ParseSubExpression(nullCoalescingPrecedence - 1); leftOperand = _syntaxFactory.ConditionalExpression(leftOperand, questionToken, colonLeft, colon, colonRight); } return(leftOperand); }