private SyntaxTree WithChanges(SourceText newText, IReadOnlyList <TextChangeRange> changes)
{
if (changes == null)
{
throw new ArgumentNullException("changes");
}
var oldTree = this;
// if changes is entire text do a full reparse
if (changes.Count == 1 && changes[0].Span == new TextSpan(0, this.Length) && changes[0].NewLength == newText.Length)
{
// parser will do a full parse if we give it no changes
changes = null;
oldTree = null;
}
using (var lexer = new InternalSyntax.Lexer(newText, this.Options))
{
CSharp.CSharpSyntaxNode oldRoot = oldTree != null?oldTree.GetRoot() : null;
using (var parser = new InternalSyntax.LanguageParser(lexer, oldRoot, changes))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(newText, this.FilePath, this.Options, compilationUnit, parser.Directives);
tree.VerifySource(changes);
return(tree);
}
}
}
private SyntaxTree WithChanges(SourceText newText, IReadOnlyList <TextChangeRange> changes)
{
if (changes == null)
{
throw new ArgumentNullException(nameof(changes));
}
var oldTree = this;
// if changes is entire text do a full reparse
if (changes.Count == 1 && changes[0].Span == new TextSpan(0, this.Length) && changes[0].NewLength == newText.Length)
{
// parser will do a full parse if we give it no changes
changes = null;
oldTree = null;
}
using (var lexer = new InternalSyntax.Lexer(newText, this.Options))
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree?.GetRoot(), changes))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(
newText,
newText.Encoding,
newText.ChecksumAlgorithm,
FilePath,
Options,
compilationUnit,
parser.Directives,
DiagnosticOptions);
tree.VerifySource(changes);
return(tree);
}
}
/// <summary>
/// Produces a syntax tree by parsing the source text.
/// </summary>
public static SyntaxTree ParseText(
SourceText text,
CSharpParseOptions options = null,
string path = "",
CancellationToken cancellationToken = default(CancellationToken))
{
if (text == null)
{
throw new ArgumentNullException("text");
}
if (path == null)
{
throw new ArgumentNullException("path");
}
options = options ?? CSharpParseOptions.Default;
using (var lexer = new InternalSyntax.Lexer(text, options))
{
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, parser.Directives);
tree.VerifySource();
return(tree);
}
}
}
private static InternalSyntax.ExpressionSyntax ParseDebuggerExpressionInternal(
SourceText source,
bool consumeFullText
)
{
using var lexer = new InternalSyntax.Lexer(
source,
ParseOptions,
allowPreprocessorDirectives: false
);
using var parser = new InternalSyntax.LanguageParser(
lexer,
oldTree: null,
changes: null,
lexerMode: InternalSyntax.LexerMode.DebuggerSyntax
);
var node = parser.ParseExpression();
if (consumeFullText)
{
node = parser.ConsumeUnexpectedTokens(node);
}
return(node);
}
private static StatementSyntax ParseDebuggerStatement(string text)
{
var source = SourceText.From(text);
using var lexer = new InternalSyntax.Lexer(source, ParseOptions);
using var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax);
var statement = parser.ParseStatement();
var syntaxTree = statement.CreateSyntaxTree(source);
return((StatementSyntax)syntaxTree.GetRoot());
}
private InterpolationSyntax ParseInterpolation(string text, Lexer.Interpolation interpolation, bool isVerbatim)
{
SyntaxToken openBraceToken;
ExpressionSyntax expression;
InterpolationAlignmentClauseSyntax alignment = null;
InterpolationFormatClauseSyntax format = null;
var closeBraceToken = interpolation.CloseBraceMissing
? SyntaxFactory.MissingToken(SyntaxKind.CloseBraceToken)
: SyntaxFactory.Token(SyntaxKind.CloseBraceToken);
var parsedText = Substring(text, interpolation.OpenBracePosition, interpolation.HasColon ? interpolation.ColonPosition - 1 : interpolation.CloseBracePosition - 1);
using (var tempLexer = new Lexer(Text.SourceText.From(parsedText), this.Options, allowPreprocessorDirectives: false, interpolationFollowedByColon: interpolation.HasColon))
{
// TODO: some of the trivia in the interpolation maybe should be trailing trivia of the openBraceToken
using (var tempParser = new LanguageParser(tempLexer, null, null))
{
SyntaxToken commaToken = null;
ExpressionSyntax alignmentExpression = null;
tempParser.ParseInterpolationStart(out openBraceToken, out expression, out commaToken, out alignmentExpression);
if (alignmentExpression != null)
{
alignment = SyntaxFactory.InterpolationAlignmentClause(commaToken, alignmentExpression);
}
var extraTrivia = tempParser.CurrentToken.GetLeadingTrivia();
if (interpolation.HasColon)
{
var colonToken = SyntaxFactory.Token(SyntaxKind.ColonToken).TokenWithLeadingTrivia(extraTrivia);
var formatText = Substring(text, interpolation.ColonPosition + 1, interpolation.FormatEndPosition);
var formatString = MakeStringToken(formatText, formatText, isVerbatim, SyntaxKind.InterpolatedStringTextToken);
format = SyntaxFactory.InterpolationFormatClause(colonToken, formatString);
}
else
{
// Move the leading trivia from the insertion's EOF token to the following token.
closeBraceToken = closeBraceToken.TokenWithLeadingTrivia(extraTrivia);
}
}
}
var result = SyntaxFactory.Interpolation(openBraceToken, expression, alignment, format, closeBraceToken);
Debug.Assert(Substring(text, interpolation.OpenBracePosition, interpolation.LastPosition) == result.ToFullString()); // yield from text equals yield from node
return(result);
}
/// <summary>
/// Produces a syntax tree by parsing the source text.
/// </summary>
public static SyntaxTree ParseText(
SourceText text,
CSharpParseOptions options = null,
string path = "",
ImmutableDictionary <string, ReportDiagnostic> diagnosticOptions = null,
bool?isGeneratedCode = null,
CancellationToken cancellationToken = default)
{
if (text == null)
{
throw new ArgumentNullException(nameof(text));
}
options = options ?? CSharpParseOptions.Default;
using (var lexer = new InternalSyntax.Lexer(text, options))
{
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(
text,
text.Encoding,
text.ChecksumAlgorithm,
path,
options,
compilationUnit,
parser.Directives,
diagnosticOptions: diagnosticOptions,
isGeneratedCode: isGeneratedCode,
cloneRoot: true);
tree.VerifySource();
return(tree);
}
}
}
private ExpressionSyntax ParseInterpolatedStringToken()
{
// We don't want to make the scanner stateful (between tokens) if we can possibly avoid it.
// The approach implemented here is
//
// (1) Scan the whole interpolated string literal as a single token. Now the statefulness of
// the scanner (to match { }'s) is limited to its behavior while scanning a single token.
//
// (2) When the parser gets such a token, here, it spins up another scanner / parser on each of
// the holes and builds a tree for the whole thing (resulting in an interpolated string nonterminal node).
//
// (3) The parser discards the original token and replaces it with this tree. (In other words,
// it replaces one token with a different set of tokens that have already been parsed)
//
// (4) On an incremental change, we widen the invalidated region to include any enclosing interpolated
// string nonterminal so that we never reuse tokens inside a changed interpolated string.
//
// This has the secondary advantage that it can reasonably be specified.
//
// The substitution will end up being invisible to external APIs and clients such as the IDE, as
// they have no way to ask for the stream of tokens before parsing.
//
// Take special not of the handling of trivia. With one exception, what happens in a hole stays
// in the hole. That means that the first token in a hole can have leading trivia, even though
// it is not the first thing on a line. The single exception occurs when the hole is completely
// empty of tokens, but contains some trivia. In that case we move the trivia (from the fake EOF
// token inside the hole) to be leading trivia of the following literal part so that it doesn't
// get dropped on the floor.
//
var originalToken = this.EatToken();
Debug.Assert(originalToken.Kind == SyntaxKind.InterpolatedStringToken);
var originalText = originalToken.ValueText;
var interpolations = ArrayBuilder <Lexer.Interpolation> .GetInstance();
SyntaxDiagnosticInfo error = null;
using (var tempLexer = new Lexer(Text.SourceText.From(originalText), this.Options))
{
// compute the positions of the interpolations in the original string literal
var info = default(Lexer.TokenInfo);
tempLexer.ScanISLTop(interpolations, ref info, ref error);
}
Debug.Assert(interpolations.Count != 0);
var builder = this.pool.AllocateSeparated <InterpolatedStringInsertSyntax>();
try
{
SyntaxToken stringStart = null;
SyntaxToken stringEnd = null;
for (int i = 0; i < interpolations.Count; i++)
{
var interpolation = interpolations[i];
var first = i == 0;
var last = i == (interpolations.Count - 1);
if (first)
{
// compute stringStart
var startText = originalText.Substring(0, interpolation.Start + 1);
stringStart = MakeStringToken(startText, SyntaxKind.InterpolatedStringStartToken).WithLeadingTrivia(originalToken.GetLeadingTrivia());
Debug.Assert(stringStart.Kind == SyntaxKind.InterpolatedStringStartToken);
}
CSharpSyntaxNode additionalTrivia;
var interpText = originalText.Substring(interpolation.Start + 1, interpolation.End - interpolation.Start - 1);
using (var tempLexer = new Lexer(Text.SourceText.From(interpText), this.Options))
{
using (var tempParser = new LanguageParser(tempLexer, null, null))
{
var insert = tempParser.ParseInterpolatedStringInsert();
insert = tempParser.ConsumeUnexpectedTokens(insert);
// In case the insert is empty, move the leading trivia from its EOF token to the following literal part.
additionalTrivia = tempParser.CurrentToken.GetLeadingTrivia();
builder.Add(insert);
}
}
if (last)
{
// compute stringEnd
var endText = originalText.Substring(interpolation.End);
stringEnd = MakeStringToken(endText, SyntaxKind.InterpolatedStringEndToken).WithLeadingTrivia(additionalTrivia).WithTrailingTrivia(originalToken.GetTrailingTrivia());
Debug.Assert(stringEnd.Kind == SyntaxKind.InterpolatedStringEndToken);
}
else
{
// add an interpolated string mid token for the following }...{ part
var midText = originalText.Substring(interpolation.End, interpolations[i + 1].Start - interpolation.End + 1);
var stringMid = MakeStringToken(midText, SyntaxKind.InterpolatedStringMidToken).WithLeadingTrivia(additionalTrivia);
Debug.Assert(stringMid.Kind == SyntaxKind.InterpolatedStringMidToken);
builder.AddSeparator(stringMid);
}
}
interpolations.Free();
var result = SyntaxFactory.InterpolatedString(stringStart, builder.ToList(), stringEnd);
if (error != null)
{
result = result.WithDiagnosticsGreen(new[] { error });
}
return(result);
}
finally
{
this.pool.Free(builder);
}
}
private static StatementSyntax ParseDebuggerStatement(string text)
{
var source = SourceText.From(text);
using (var lexer = new InternalSyntax.Lexer(source, CSharpParseOptions.Default))
{
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax))
{
var statement = parser.ParseStatement();
var syntaxTree = statement.CreateSyntaxTree(source);
return (StatementSyntax)syntaxTree.GetRoot();
}
}
}
private static InternalSyntax.ExpressionSyntax ParseDebuggerExpressionInternal(SourceText source, bool consumeFullText)
{
using (var lexer = new InternalSyntax.Lexer(source, CSharpParseOptions.Default, allowPreprocessorDirectives: false))
{
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax))
{
var node = parser.ParseExpression();
if (consumeFullText) node = parser.ConsumeUnexpectedTokens(node);
return node;
}
}
}
private InterpolationSyntax ParseInterpolation(string text, Lexer.Interpolation interpolation, bool isVerbatim)
{
SyntaxToken openBraceToken;
ExpressionSyntax expression;
InterpolationAlignmentClauseSyntax alignment = null;
InterpolationFormatClauseSyntax format = null;
var closeBraceToken = interpolation.CloseBraceMissing
? SyntaxFactory.MissingToken(SyntaxKind.CloseBraceToken)
: SyntaxFactory.Token(SyntaxKind.CloseBraceToken);
var parsedText = Substring(text, interpolation.OpenBracePosition, interpolation.HasColon ? interpolation.ColonPosition - 1 : interpolation.CloseBracePosition - 1);
using (var tempLexer = new Lexer(Text.SourceText.From(parsedText), this.Options, allowPreprocessorDirectives: false))
{
// TODO: some of the trivia in the interpolation maybe should be trailing trivia of the openBraceToken
using (var tempParser = new LanguageParser(tempLexer, null, null))
{
SyntaxToken commaToken = null;
ExpressionSyntax alignmentExpression = null;
tempParser.ParseInterpolationStart(out openBraceToken, out expression, out commaToken, out alignmentExpression);
if (alignmentExpression != null)
{
alignment = SyntaxFactory.InterpolationAlignmentClause(commaToken, alignmentExpression);
}
var extraTrivia = tempParser.CurrentToken.GetLeadingTrivia();
if (interpolation.HasColon)
{
var colonToken = SyntaxFactory.Token(SyntaxKind.ColonToken).TokenWithLeadingTrivia(extraTrivia);
var formatText = Substring(text, interpolation.ColonPosition + 1, interpolation.FormatEndPosition);
var formatString = MakeStringToken(formatText, formatText, isVerbatim, SyntaxKind.InterpolatedStringTextToken);
format = SyntaxFactory.InterpolationFormatClause(colonToken, formatString);
}
else
{
// Move the leading trivia from the insertion's EOF token to the following token.
closeBraceToken = closeBraceToken.TokenWithLeadingTrivia(extraTrivia);
}
}
}
var result = SyntaxFactory.Interpolation(openBraceToken, expression, alignment, format, closeBraceToken);
Debug.Assert(Substring(text, interpolation.OpenBracePosition, interpolation.LastPosition) == result.ToFullString()); // yield from text equals yield from node
return result;
}
/// <summary>
/// Produces a syntax tree by parsing the source text.
/// </summary>
public static SyntaxTree ParseText(
SourceText text,
CSharpParseOptions options = null,
string path = "",
CancellationToken cancellationToken = default(CancellationToken))
{
if (text == null)
{
throw new ArgumentNullException(nameof(text));
}
#if XSHARPPRE
{
string s = text.ToString();
s = s.Replace("Microsoft.CodeAnalysis.CSharp", "LanguageService.CodeAnalysis.XSharp");
s = s.Replace("Microsoft.CodeAnalysis", "LanguageService.CodeAnalysis");
s = s.Replace("Microsoft.Cci", "LanguageService.Cci");
s = (new System.Text.RegularExpressions.Regex(@"(?<!\.)CSharpResources")).Replace(s, "LanguageService.CodeAnalysis.XSharpResources");
s = (new System.Text.RegularExpressions.Regex(@"(?<!Microsoft[\._][A-Za-z0-9_\.]*)CSharp")).Replace(s, "XSharp");
s = s.Replace("XSharp.XSharpResources", "XSharpResources");
s = s.Replace("Antlr4.Runtime", "LanguageService.SyntaxTree");
s = s.Replace("XSHARP_RUNTIME", "true");
text = SourceText.From(s, text.Encoding);
}
#endif
options = options ?? CSharpParseOptions.Default;
#if XSHARP
using (var parser = new InternalSyntax.XSharpLanguageParser(path, text, options, oldTree: null, changes: null, cancellationToken: cancellationToken))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, default(InternalSyntax.DirectiveStack));
//tree.VerifySource();
if (options.SaveAsCSharp)
{
path = System.IO.Path.ChangeExtension(path, ".cs");
string source = compilationUnit.ToString();
source = source.Replace(";", ";\r\n");
source = source.Replace("{", "\r\n{\r\n");
source = source.Replace("}", "\r\n}\r\n");
source = source.Replace(" . ", ".");
source = source.Replace(" :: ", "::");
source = source.Replace("}", "}\r\n");
source = source.Replace("$", "_");
System.IO.File.WriteAllText(path, source);
}
return(tree);
}
#else
using (var lexer = new InternalSyntax.Lexer(text, options))
{
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, parser.Directives);
tree.VerifySource();
return(tree);
}
}
#endif
}
