예제 #1
0
        private SyntaxTree WithChanges(SourceText newText, IReadOnlyList <TextChangeRange> changes)
        {
            if (changes == null)
            {
                throw new ArgumentNullException("changes");
            }

            var oldTree = this;

            // if changes is entire text do a full reparse
            if (changes.Count == 1 && changes[0].Span == new TextSpan(0, this.Length) && changes[0].NewLength == newText.Length)
            {
                // parser will do a full parse if we give it no changes
                changes = null;
                oldTree = null;
            }

            using (var lexer = new InternalSyntax.Lexer(newText, this.Options))
            {
                CSharp.CSharpSyntaxNode oldRoot = oldTree != null?oldTree.GetRoot() : null;

                using (var parser = new InternalSyntax.LanguageParser(lexer, oldRoot, changes))
                {
                    var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
                    var tree            = new ParsedSyntaxTree(newText, this.FilePath, this.Options, compilationUnit, parser.Directives);
                    tree.VerifySource(changes);
                    return(tree);
                }
            }
        }
예제 #2
0
        private SyntaxTree WithChanges(SourceText newText, IReadOnlyList <TextChangeRange> changes)
        {
            if (changes == null)
            {
                throw new ArgumentNullException(nameof(changes));
            }

            var oldTree = this;

            // if changes is entire text do a full reparse
            if (changes.Count == 1 && changes[0].Span == new TextSpan(0, this.Length) && changes[0].NewLength == newText.Length)
            {
                // parser will do a full parse if we give it no changes
                changes = null;
                oldTree = null;
            }

            using (var lexer = new InternalSyntax.Lexer(newText, this.Options))
                using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree?.GetRoot(), changes))
                {
                    var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
                    var tree            = new ParsedSyntaxTree(
                        newText,
                        newText.Encoding,
                        newText.ChecksumAlgorithm,
                        FilePath,
                        Options,
                        compilationUnit,
                        parser.Directives,
                        DiagnosticOptions);
                    tree.VerifySource(changes);
                    return(tree);
                }
        }
예제 #3
0
        /// <summary>
        /// Produces a syntax tree by parsing the source text.
        /// </summary>
        public static SyntaxTree ParseText(
            SourceText text,
            CSharpParseOptions options = null,
            string path = "",
            CancellationToken cancellationToken = default(CancellationToken))
        {
            if (text == null)
            {
                throw new ArgumentNullException("text");
            }

            if (path == null)
            {
                throw new ArgumentNullException("path");
            }

            options = options ?? CSharpParseOptions.Default;

            using (var lexer = new InternalSyntax.Lexer(text, options))
            {
                using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken))
                {
                    var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
                    var tree            = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, parser.Directives);
                    tree.VerifySource();
                    return(tree);
                }
            }
        }
예제 #4
0
        private static InternalSyntax.ExpressionSyntax ParseDebuggerExpressionInternal(
            SourceText source,
            bool consumeFullText
            )
        {
            using var lexer = new InternalSyntax.Lexer(
                      source,
                      ParseOptions,
                      allowPreprocessorDirectives: false
                      );
            using var parser = new InternalSyntax.LanguageParser(
                      lexer,
                      oldTree: null,
                      changes: null,
                      lexerMode: InternalSyntax.LexerMode.DebuggerSyntax
                      );

            var node = parser.ParseExpression();

            if (consumeFullText)
            {
                node = parser.ConsumeUnexpectedTokens(node);
            }
            return(node);
        }
예제 #5
0
        private static StatementSyntax ParseDebuggerStatement(string text)
        {
            var source = SourceText.From(text);

            using var lexer  = new InternalSyntax.Lexer(source, ParseOptions);
            using var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax);

            var statement  = parser.ParseStatement();
            var syntaxTree = statement.CreateSyntaxTree(source);

            return((StatementSyntax)syntaxTree.GetRoot());
        }
        private InterpolationSyntax ParseInterpolation(string text, Lexer.Interpolation interpolation, bool isVerbatim)
        {
            SyntaxToken      openBraceToken;
            ExpressionSyntax expression;
            InterpolationAlignmentClauseSyntax alignment = null;
            InterpolationFormatClauseSyntax    format    = null;
            var closeBraceToken = interpolation.CloseBraceMissing
                ? SyntaxFactory.MissingToken(SyntaxKind.CloseBraceToken)
                : SyntaxFactory.Token(SyntaxKind.CloseBraceToken);

            var parsedText = Substring(text, interpolation.OpenBracePosition, interpolation.HasColon ? interpolation.ColonPosition - 1 : interpolation.CloseBracePosition - 1);

            using (var tempLexer = new Lexer(Text.SourceText.From(parsedText), this.Options, allowPreprocessorDirectives: false, interpolationFollowedByColon: interpolation.HasColon))
            {
                // TODO: some of the trivia in the interpolation maybe should be trailing trivia of the openBraceToken
                using (var tempParser = new LanguageParser(tempLexer, null, null))
                {
                    SyntaxToken      commaToken          = null;
                    ExpressionSyntax alignmentExpression = null;
                    tempParser.ParseInterpolationStart(out openBraceToken, out expression, out commaToken, out alignmentExpression);
                    if (alignmentExpression != null)
                    {
                        alignment = SyntaxFactory.InterpolationAlignmentClause(commaToken, alignmentExpression);
                    }

                    var extraTrivia = tempParser.CurrentToken.GetLeadingTrivia();
                    if (interpolation.HasColon)
                    {
                        var colonToken   = SyntaxFactory.Token(SyntaxKind.ColonToken).TokenWithLeadingTrivia(extraTrivia);
                        var formatText   = Substring(text, interpolation.ColonPosition + 1, interpolation.FormatEndPosition);
                        var formatString = MakeStringToken(formatText, formatText, isVerbatim, SyntaxKind.InterpolatedStringTextToken);
                        format = SyntaxFactory.InterpolationFormatClause(colonToken, formatString);
                    }
                    else
                    {
                        // Move the leading trivia from the insertion's EOF token to the following token.
                        closeBraceToken = closeBraceToken.TokenWithLeadingTrivia(extraTrivia);
                    }
                }
            }

            var result = SyntaxFactory.Interpolation(openBraceToken, expression, alignment, format, closeBraceToken);

            Debug.Assert(Substring(text, interpolation.OpenBracePosition, interpolation.LastPosition) == result.ToFullString()); // yield from text equals yield from node
            return(result);
        }
예제 #7
0
        /// <summary>
        /// Produces a syntax tree by parsing the source text.
        /// </summary>
        public static SyntaxTree ParseText(
            SourceText text,
            CSharpParseOptions options = null,
            string path = "",
            ImmutableDictionary <string, ReportDiagnostic> diagnosticOptions = null,
            bool?isGeneratedCode = null,
            CancellationToken cancellationToken = default)
        {
            if (text == null)
            {
                throw new ArgumentNullException(nameof(text));
            }

            options = options ?? CSharpParseOptions.Default;

            using (var lexer = new InternalSyntax.Lexer(text, options))
            {
                using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken))
                {
                    var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
                    var tree            = new ParsedSyntaxTree(
                        text,
                        text.Encoding,
                        text.ChecksumAlgorithm,
                        path,
                        options,
                        compilationUnit,
                        parser.Directives,
                        diagnosticOptions: diagnosticOptions,
                        isGeneratedCode: isGeneratedCode,
                        cloneRoot: true);
                    tree.VerifySource();
                    return(tree);
                }
            }
        }
        private ExpressionSyntax ParseInterpolatedStringToken()
        {
            // We don't want to make the scanner stateful (between tokens) if we can possibly avoid it.
            // The approach implemented here is
            //
            // (1) Scan the whole interpolated string literal as a single token. Now the statefulness of
            // the scanner (to match { }'s) is limited to its behavior while scanning a single token.
            //
            // (2) When the parser gets such a token, here, it spins up another scanner / parser on each of
            // the holes and builds a tree for the whole thing (resulting in an interpolated string nonterminal node).
            //
            // (3) The parser discards the original token and replaces it with this tree. (In other words,
            // it replaces one token with a different set of tokens that have already been parsed)
            //
            // (4) On an incremental change, we widen the invalidated region to include any enclosing interpolated
            // string nonterminal so that we never reuse tokens inside a changed interpolated string.
            //
            // This has the secondary advantage that it can reasonably be specified.
            //
            // The substitution will end up being invisible to external APIs and clients such as the IDE, as
            // they have no way to ask for the stream of tokens before parsing.
            //
            // Take special not of the handling of trivia. With one exception, what happens in a hole stays
            // in the hole. That means that the first token in a hole can have leading trivia, even though
            // it is not the first thing on a line. The single exception occurs when the hole is completely
            // empty of tokens, but contains some trivia. In that case we move the trivia (from the fake EOF
            // token inside the hole) to be leading trivia of the following literal part so that it doesn't
            // get dropped on the floor.
            //

            var originalToken = this.EatToken();

            Debug.Assert(originalToken.Kind == SyntaxKind.InterpolatedStringToken);
            var originalText   = originalToken.ValueText;
            var interpolations = ArrayBuilder <Lexer.Interpolation> .GetInstance();

            SyntaxDiagnosticInfo error = null;

            using (var tempLexer = new Lexer(Text.SourceText.From(originalText), this.Options))
            {
                // compute the positions of the interpolations in the original string literal
                var info = default(Lexer.TokenInfo);
                tempLexer.ScanISLTop(interpolations, ref info, ref error);
            }

            Debug.Assert(interpolations.Count != 0);
            var builder = this.pool.AllocateSeparated <InterpolatedStringInsertSyntax>();

            try
            {
                SyntaxToken stringStart = null;
                SyntaxToken stringEnd   = null;

                for (int i = 0; i < interpolations.Count; i++)
                {
                    var interpolation = interpolations[i];
                    var first         = i == 0;
                    var last          = i == (interpolations.Count - 1);

                    if (first)
                    {
                        // compute stringStart
                        var startText = originalText.Substring(0, interpolation.Start + 1);
                        stringStart = MakeStringToken(startText, SyntaxKind.InterpolatedStringStartToken).WithLeadingTrivia(originalToken.GetLeadingTrivia());
                        Debug.Assert(stringStart.Kind == SyntaxKind.InterpolatedStringStartToken);
                    }

                    CSharpSyntaxNode additionalTrivia;
                    var interpText = originalText.Substring(interpolation.Start + 1, interpolation.End - interpolation.Start - 1);
                    using (var tempLexer = new Lexer(Text.SourceText.From(interpText), this.Options))
                    {
                        using (var tempParser = new LanguageParser(tempLexer, null, null))
                        {
                            var insert = tempParser.ParseInterpolatedStringInsert();
                            insert = tempParser.ConsumeUnexpectedTokens(insert);
                            // In case the insert is empty, move the leading trivia from its EOF token to the following literal part.
                            additionalTrivia = tempParser.CurrentToken.GetLeadingTrivia();
                            builder.Add(insert);
                        }
                    }

                    if (last)
                    {
                        // compute stringEnd
                        var endText = originalText.Substring(interpolation.End);
                        stringEnd = MakeStringToken(endText, SyntaxKind.InterpolatedStringEndToken).WithLeadingTrivia(additionalTrivia).WithTrailingTrivia(originalToken.GetTrailingTrivia());
                        Debug.Assert(stringEnd.Kind == SyntaxKind.InterpolatedStringEndToken);
                    }
                    else
                    {
                        // add an interpolated string mid token for the following }...{ part
                        var midText   = originalText.Substring(interpolation.End, interpolations[i + 1].Start - interpolation.End + 1);
                        var stringMid = MakeStringToken(midText, SyntaxKind.InterpolatedStringMidToken).WithLeadingTrivia(additionalTrivia);
                        Debug.Assert(stringMid.Kind == SyntaxKind.InterpolatedStringMidToken);
                        builder.AddSeparator(stringMid);
                    }
                }

                interpolations.Free();
                var result = SyntaxFactory.InterpolatedString(stringStart, builder.ToList(), stringEnd);
                if (error != null)
                {
                    result = result.WithDiagnosticsGreen(new[] { error });
                }
                return(result);
            }
            finally
            {
                this.pool.Free(builder);
            }
        }
예제 #9
0
 private static StatementSyntax ParseDebuggerStatement(string text)
 {
     var source = SourceText.From(text);
     using (var lexer = new InternalSyntax.Lexer(source, CSharpParseOptions.Default))
     {
         using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax))
         {
             var statement = parser.ParseStatement();
             var syntaxTree = statement.CreateSyntaxTree(source);
             return (StatementSyntax)syntaxTree.GetRoot();
         }
     }
 }
예제 #10
0
 private static InternalSyntax.ExpressionSyntax ParseDebuggerExpressionInternal(SourceText source, bool consumeFullText)
 {
     using (var lexer = new InternalSyntax.Lexer(source, CSharpParseOptions.Default, allowPreprocessorDirectives: false))
     {
         using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax))
         {
             var node = parser.ParseExpression();
             if (consumeFullText) node = parser.ConsumeUnexpectedTokens(node);
             return node;
         }
     }
 }
        private InterpolationSyntax ParseInterpolation(string text, Lexer.Interpolation interpolation, bool isVerbatim)
        {
            SyntaxToken openBraceToken;
            ExpressionSyntax expression;
            InterpolationAlignmentClauseSyntax alignment = null;
            InterpolationFormatClauseSyntax format = null;
            var closeBraceToken = interpolation.CloseBraceMissing
                ? SyntaxFactory.MissingToken(SyntaxKind.CloseBraceToken)
                : SyntaxFactory.Token(SyntaxKind.CloseBraceToken);

            var parsedText = Substring(text, interpolation.OpenBracePosition, interpolation.HasColon ? interpolation.ColonPosition - 1 : interpolation.CloseBracePosition - 1);
            using (var tempLexer = new Lexer(Text.SourceText.From(parsedText), this.Options, allowPreprocessorDirectives: false))
            {
                // TODO: some of the trivia in the interpolation maybe should be trailing trivia of the openBraceToken
                using (var tempParser = new LanguageParser(tempLexer, null, null))
                {
                    SyntaxToken commaToken = null;
                    ExpressionSyntax alignmentExpression = null;
                    tempParser.ParseInterpolationStart(out openBraceToken, out expression, out commaToken, out alignmentExpression);
                    if (alignmentExpression != null)
                    {
                        alignment = SyntaxFactory.InterpolationAlignmentClause(commaToken, alignmentExpression);
                    }

                    var extraTrivia = tempParser.CurrentToken.GetLeadingTrivia();
                    if (interpolation.HasColon)
                    {
                        var colonToken = SyntaxFactory.Token(SyntaxKind.ColonToken).TokenWithLeadingTrivia(extraTrivia);
                        var formatText = Substring(text, interpolation.ColonPosition + 1, interpolation.FormatEndPosition);
                        var formatString = MakeStringToken(formatText, formatText, isVerbatim, SyntaxKind.InterpolatedStringTextToken);
                        format = SyntaxFactory.InterpolationFormatClause(colonToken, formatString);
                    }
                    else
                    {
                        // Move the leading trivia from the insertion's EOF token to the following token.
                        closeBraceToken = closeBraceToken.TokenWithLeadingTrivia(extraTrivia);
                    }
                }
            }

            var result = SyntaxFactory.Interpolation(openBraceToken, expression, alignment, format, closeBraceToken);
            Debug.Assert(Substring(text, interpolation.OpenBracePosition, interpolation.LastPosition) == result.ToFullString()); // yield from text equals yield from node
            return result;
        }
예제 #12
0
        /// <summary>
        /// Produces a syntax tree by parsing the source text.
        /// </summary>
        public static SyntaxTree ParseText(
            SourceText text,
            CSharpParseOptions options = null,
            string path = "",
            CancellationToken cancellationToken = default(CancellationToken))
        {
            if (text == null)
            {
                throw new ArgumentNullException(nameof(text));
            }

#if XSHARPPRE
            {
                string s = text.ToString();
                s    = s.Replace("Microsoft.CodeAnalysis.CSharp", "LanguageService.CodeAnalysis.XSharp");
                s    = s.Replace("Microsoft.CodeAnalysis", "LanguageService.CodeAnalysis");
                s    = s.Replace("Microsoft.Cci", "LanguageService.Cci");
                s    = (new System.Text.RegularExpressions.Regex(@"(?<!\.)CSharpResources")).Replace(s, "LanguageService.CodeAnalysis.XSharpResources");
                s    = (new System.Text.RegularExpressions.Regex(@"(?<!Microsoft[\._][A-Za-z0-9_\.]*)CSharp")).Replace(s, "XSharp");
                s    = s.Replace("XSharp.XSharpResources", "XSharpResources");
                s    = s.Replace("Antlr4.Runtime", "LanguageService.SyntaxTree");
                s    = s.Replace("XSHARP_RUNTIME", "true");
                text = SourceText.From(s, text.Encoding);
            }
#endif

            options = options ?? CSharpParseOptions.Default;

#if XSHARP
            using (var parser = new InternalSyntax.XSharpLanguageParser(path, text, options, oldTree: null, changes: null, cancellationToken: cancellationToken))
            {
                var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
                var tree            = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, default(InternalSyntax.DirectiveStack));
                //tree.VerifySource();
                if (options.SaveAsCSharp)
                {
                    path = System.IO.Path.ChangeExtension(path, ".cs");
                    string source = compilationUnit.ToString();
                    source = source.Replace(";", ";\r\n");
                    source = source.Replace("{", "\r\n{\r\n");
                    source = source.Replace("}", "\r\n}\r\n");
                    source = source.Replace(" . ", ".");
                    source = source.Replace(" :: ", "::");
                    source = source.Replace("}", "}\r\n");
                    source = source.Replace("$", "_");
                    System.IO.File.WriteAllText(path, source);
                }
                return(tree);
            }
#else
            using (var lexer = new InternalSyntax.Lexer(text, options))
            {
                using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken))
                {
                    var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
                    var tree            = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, parser.Directives);
                    tree.VerifySource();
                    return(tree);
                }
            }
#endif
        }