private SyntaxToken DebuggerLex(string text)
{
using (var lexer = new InternalSyntax.Lexer(SourceText.From(text), _options))
{
return new SyntaxToken(lexer.Lex(InternalSyntax.LexerMode.DebuggerSyntax));
}
}
/// <summary>
/// Produces a syntax tree by parsing the source text.
/// </summary>
public static SyntaxTree ParseText(
SourceText text,
CSharpParseOptions options = null,
string path = "",
CancellationToken cancellationToken = default(CancellationToken))
{
if (text == null)
{
throw new ArgumentNullException("text");
}
if (path == null)
{
throw new ArgumentNullException("path");
}
options = options ?? CSharpParseOptions.Default;
using (var lexer = new InternalSyntax.Lexer(text, options))
{
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, parser.Directives);
tree.VerifySource();
return(tree);
}
}
}
internal override IEnumerable <InternalSyntax.SyntaxToken> GetTokens(string text)
{
Assert.DoesNotContain("'", text, StringComparison.Ordinal);
using (
var lexer = new InternalSyntax.Lexer(
SourceText.From(text + "'"),
TestOptions.RegularWithDocumentationComments
)
)
{
while (true)
{
var token = lexer.Lex(
InternalSyntax.LexerMode.XmlNameQuote
| InternalSyntax.LexerMode.XmlDocCommentStyleSingleLine
| InternalSyntax.LexerMode.XmlDocCommentLocationInterior
);
if (token.Kind == SyntaxKind.SingleQuoteToken)
{
break;
}
yield return(token);
}
}
}
private static InternalSyntax.ExpressionSyntax ParseDebuggerExpressionInternal(
SourceText source,
bool consumeFullText
)
{
using var lexer = new InternalSyntax.Lexer(
source,
ParseOptions,
allowPreprocessorDirectives: false
);
using var parser = new InternalSyntax.LanguageParser(
lexer,
oldTree: null,
changes: null,
lexerMode: InternalSyntax.LexerMode.DebuggerSyntax
);
var node = parser.ParseExpression();
if (consumeFullText)
{
node = parser.ConsumeUnexpectedTokens(node);
}
return(node);
}
private SyntaxTree WithChanges(SourceText newText, IReadOnlyList <TextChangeRange> changes)
{
if (changes == null)
{
throw new ArgumentNullException(nameof(changes));
}
var oldTree = this;
// if changes is entire text do a full reparse
if (changes.Count == 1 && changes[0].Span == new TextSpan(0, this.Length) && changes[0].NewLength == newText.Length)
{
// parser will do a full parse if we give it no changes
changes = null;
oldTree = null;
}
using (var lexer = new InternalSyntax.Lexer(newText, this.Options))
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree?.GetRoot(), changes))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(
newText,
newText.Encoding,
newText.ChecksumAlgorithm,
FilePath,
Options,
compilationUnit,
parser.Directives,
DiagnosticOptions);
tree.VerifySource(changes);
return(tree);
}
}
private SyntaxTree WithChanges(SourceText newText, IReadOnlyList <TextChangeRange> changes)
{
if (changes == null)
{
throw new ArgumentNullException("changes");
}
var oldTree = this;
// if changes is entire text do a full reparse
if (changes.Count == 1 && changes[0].Span == new TextSpan(0, this.Length) && changes[0].NewLength == newText.Length)
{
// parser will do a full parse if we give it no changes
changes = null;
oldTree = null;
}
using (var lexer = new InternalSyntax.Lexer(newText, this.Options))
{
CSharp.CSharpSyntaxNode oldRoot = oldTree != null?oldTree.GetRoot() : null;
using (var parser = new InternalSyntax.LanguageParser(lexer, oldRoot, changes))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(newText, this.FilePath, this.Options, compilationUnit, parser.Directives);
tree.VerifySource(changes);
return(tree);
}
}
}
public Reader(Blender blender)
{
this.lexer = blender.lexer;
this.oldTreeCursor = blender.oldTreeCursor;
this.changes = blender.changes;
this.newPosition = blender.newPosition;
this.changeDelta = blender.changeDelta;
this.newDirectives = blender.newDirectives;
this.oldDirectives = blender.oldDirectives;
this.newLexerDrivenMode = blender.newLexerDrivenMode;
}
public Reader(Blender blender)
{
_lexer = blender._lexer;
_oldTreeCursor = blender._oldTreeCursor;
_changes = blender._changes;
_newPosition = blender._newPosition;
_changeDelta = blender._changeDelta;
_newDirectives = blender._newDirectives;
_oldDirectives = blender._oldDirectives;
_newLexerDrivenMode = blender._newLexerDrivenMode;
}
private static StatementSyntax ParseDebuggerStatement(string text)
{
var source = SourceText.From(text);
using var lexer = new InternalSyntax.Lexer(source, ParseOptions);
using var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax);
var statement = parser.ParseStatement();
var syntaxTree = statement.CreateSyntaxTree(source);
return((StatementSyntax)syntaxTree.GetRoot());
}
private IEnumerable<InternalSyntax.BlendedNode> Blend(string text)
{
using (var lexer = new InternalSyntax.Lexer(SourceText.From(text), _options))
{
var blender = new InternalSyntax.Blender(lexer, null, null);
InternalSyntax.BlendedNode result;
do
{
result = blender.ReadToken(InternalSyntax.LexerMode.Syntax);
blender = result.Blender;
yield return result;
}
while (result.Token.Kind != SyntaxKind.EndOfFileToken);
}
}
public Blender(Lexer lexer, CSharp.CSharpSyntaxNode oldTree, IEnumerable<TextChangeRange> changes)
{
Debug.Assert(lexer != null);
_lexer = lexer;
_changes = ImmutableStack.Create<TextChangeRange>();
if (changes != null)
{
// TODO: Consider implementing NormalizedChangeCollection for TextSpan. the real
// reason why we are collapsing is because we want to extend change ranges and
// cannot allow them to overlap. This does not seem to be a big deal since multiple
// changes are infrequent and typically close to each other. However if we have
// NormalizedChangeCollection for TextSpan we can have both - we can extend ranges
// and not require collapsing them. NormalizedChangeCollection would also ensure
// that changes are always normalized.
// TODO: this is a temporary measure to prevent individual change spans from
// overlapping after they are widened to effective width (+1 token at the start).
// once we have normalized collection for TextSpan we will not need to collapse all
// the change spans.
var collapsed = TextChangeRange.Collapse(changes);
// extend the change to its affected range. This will make it easier
// to filter out affected nodes since we will be able simply check
// if node intersects with a change.
var affectedRange = ExtendToAffectedRange(oldTree, collapsed);
_changes = _changes.Push(affectedRange);
}
if (oldTree == null)
{
// start at lexer current position if no nodes specified
_oldTreeCursor = new Cursor();
_newPosition = lexer.TextWindow.Position;
}
else
{
_oldTreeCursor = Cursor.FromRoot(oldTree).MoveToFirstChild();
_newPosition = 0;
}
_changeDelta = 0;
_newDirectives = default(DirectiveStack);
_oldDirectives = default(DirectiveStack);
_newLexerDrivenMode = 0;
}
private Blender(
Lexer lexer,
Cursor oldTreeCursor,
ImmutableStack<TextChangeRange> changes,
int newPosition,
int changeDelta,
DirectiveStack newDirectives,
DirectiveStack oldDirectives,
LexerMode newLexerDrivenMode)
{
Debug.Assert(lexer != null);
Debug.Assert(changes != null);
Debug.Assert(newPosition >= 0);
_lexer = lexer;
_oldTreeCursor = oldTreeCursor;
_changes = changes;
_newPosition = newPosition;
_changeDelta = changeDelta;
_newDirectives = newDirectives;
_oldDirectives = oldDirectives;
_newLexerDrivenMode = newLexerDrivenMode & (LexerMode.MaskXmlDocCommentLocation | LexerMode.MaskXmlDocCommentStyle);
}
/// <summary>
/// Produces a syntax tree by parsing the source text.
/// </summary>
public static SyntaxTree ParseText(
SourceText text,
CSharpParseOptions options = null,
string path = "",
ImmutableDictionary <string, ReportDiagnostic> diagnosticOptions = null,
bool?isGeneratedCode = null,
CancellationToken cancellationToken = default)
{
if (text == null)
{
throw new ArgumentNullException(nameof(text));
}
options = options ?? CSharpParseOptions.Default;
using (var lexer = new InternalSyntax.Lexer(text, options))
{
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(
text,
text.Encoding,
text.ChecksumAlgorithm,
path,
options,
compilationUnit,
parser.Directives,
diagnosticOptions: diagnosticOptions,
isGeneratedCode: isGeneratedCode,
cloneRoot: true);
tree.VerifySource();
return(tree);
}
}
}
internal DirectiveParser(Lexer lexer, DirectiveStack context)
: base(lexer, LexerMode.Directive, null, null, false)
{
this.context = context;
}
/// <summary>
/// Produces a syntax tree by parsing the source text.
/// </summary>
public static SyntaxTree ParseText(
SourceText text,
CSharpParseOptions options = null,
string path = "",
CancellationToken cancellationToken = default(CancellationToken))
{
if (text == null)
{
throw new ArgumentNullException(nameof(text));
}
#if XSHARPPRE
{
string s = text.ToString();
s = s.Replace("Microsoft.CodeAnalysis.CSharp", "LanguageService.CodeAnalysis.XSharp");
s = s.Replace("Microsoft.CodeAnalysis", "LanguageService.CodeAnalysis");
s = s.Replace("Microsoft.Cci", "LanguageService.Cci");
s = (new System.Text.RegularExpressions.Regex(@"(?<!\.)CSharpResources")).Replace(s, "LanguageService.CodeAnalysis.XSharpResources");
s = (new System.Text.RegularExpressions.Regex(@"(?<!Microsoft[\._][A-Za-z0-9_\.]*)CSharp")).Replace(s, "XSharp");
s = s.Replace("XSharp.XSharpResources", "XSharpResources");
s = s.Replace("Antlr4.Runtime", "LanguageService.SyntaxTree");
s = s.Replace("XSHARP_RUNTIME", "true");
text = SourceText.From(s, text.Encoding);
}
#endif
options = options ?? CSharpParseOptions.Default;
#if XSHARP
using (var parser = new InternalSyntax.XSharpLanguageParser(path, text, options, oldTree: null, changes: null, cancellationToken: cancellationToken))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, default(InternalSyntax.DirectiveStack));
//tree.VerifySource();
if (options.SaveAsCSharp)
{
path = System.IO.Path.ChangeExtension(path, ".cs");
string source = compilationUnit.ToString();
source = source.Replace(";", ";\r\n");
source = source.Replace("{", "\r\n{\r\n");
source = source.Replace("}", "\r\n}\r\n");
source = source.Replace(" . ", ".");
source = source.Replace(" :: ", "::");
source = source.Replace("}", "}\r\n");
source = source.Replace("$", "_");
System.IO.File.WriteAllText(path, source);
}
return(tree);
}
#else
using (var lexer = new InternalSyntax.Lexer(text, options))
{
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken))
{
var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed();
var tree = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, parser.Directives);
tree.VerifySource();
return(tree);
}
}
#endif
}
internal DirectiveParser(Lexer lexer, DirectiveStack context)
: base(lexer, LexerMode.Directive, null, null, false)
{
_context = context;
}
private InterpolationSyntax ParseInterpolation(
string text,
Lexer.Interpolation interpolation,
bool isVerbatim
)
{
SyntaxToken openBraceToken;
ExpressionSyntax expression;
InterpolationAlignmentClauseSyntax alignment = null;
InterpolationFormatClauseSyntax format = null;
var closeBraceToken = interpolation.CloseBraceMissing
? SyntaxFactory.MissingToken(SyntaxKind.CloseBraceToken)
: SyntaxFactory.Token(SyntaxKind.CloseBraceToken);
var parsedText = Substring(
text,
interpolation.OpenBracePosition,
interpolation.HasColon
? interpolation.ColonPosition - 1
: interpolation.CloseBracePosition - 1
);
using (
var tempLexer = new Lexer(
Text.SourceText.From(parsedText),
this.Options,
allowPreprocessorDirectives: false,
interpolationFollowedByColon: interpolation.HasColon
)
)
{
// TODO: some of the trivia in the interpolation maybe should be trailing trivia of the openBraceToken
using (var tempParser = new LanguageParser(tempLexer, null, null))
{
SyntaxToken commaToken = null;
ExpressionSyntax alignmentExpression = null;
tempParser.ParseInterpolationStart(
out openBraceToken,
out expression,
out commaToken,
out alignmentExpression
);
if (alignmentExpression != null)
{
alignment = SyntaxFactory.InterpolationAlignmentClause(
commaToken,
alignmentExpression
);
}
var extraTrivia = tempParser.CurrentToken.GetLeadingTrivia();
if (interpolation.HasColon)
{
var colonToken = SyntaxFactory
.Token(SyntaxKind.ColonToken)
.TokenWithLeadingTrivia(extraTrivia);
var formatText = Substring(
text,
interpolation.ColonPosition + 1,
interpolation.FormatEndPosition
);
var formatString = MakeStringToken(
formatText,
formatText,
isVerbatim,
SyntaxKind.InterpolatedStringTextToken
);
format = SyntaxFactory.InterpolationFormatClause(colonToken, formatString);
}
else
{
// Move the leading trivia from the insertion's EOF token to the following token.
closeBraceToken = closeBraceToken.TokenWithLeadingTrivia(extraTrivia);
}
}
}
var result = SyntaxFactory.Interpolation(
openBraceToken,
expression,
alignment,
format,
closeBraceToken
);
Debug.Assert(
Substring(text, interpolation.OpenBracePosition, interpolation.LastPosition)
== result.ToFullString()
); // yield from text equals yield from node
return(result);
}
private static StatementSyntax ParseDebuggerStatement(string text)
{
var source = SourceText.From(text);
using (var lexer = new InternalSyntax.Lexer(source, CSharpParseOptions.Default))
{
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax))
{
var statement = parser.ParseStatement();
var syntaxTree = statement.CreateSyntaxTree(source);
return (StatementSyntax)syntaxTree.GetRoot();
}
}
}
public InterpolatedStringScanner(
Lexer lexer,
bool isVerbatim)
{
this.lexer = lexer;
this.isVerbatim = isVerbatim;
this.allowNewlines = isVerbatim;
}
private ExpressionSyntax ParseInterpolatedStringToken()
{
// We don't want to make the scanner stateful (between tokens) if we can possibly avoid it.
// The approach implemented here is
//
// (1) Scan the whole interpolated string literal as a single token. Now the statefulness of
// the scanner (to match { }'s) is limited to its behavior while scanning a single token.
//
// (2) When the parser gets such a token, here, it spins up another scanner / parser on each of
// the holes and builds a tree for the whole thing (resulting in an InterpolatedStringExpressionSyntax).
//
// (3) The parser discards the original token and replaces it with this tree. (In other words,
// it replaces one token with a different set of tokens that have already been parsed)
//
// (4) On an incremental change, we widen the invalidated region to include any enclosing interpolated
// string nonterminal so that we never reuse tokens inside a changed interpolated string.
//
// This has the secondary advantage that it can reasonably be specified.
//
// The substitution will end up being invisible to external APIs and clients such as the IDE, as
// they have no way to ask for the stream of tokens before parsing.
//
var originalToken = this.EatToken();
var originalText = originalToken.ValueText; // this is actually the source text
Debug.Assert(originalText[0] == '$');
var isVerbatim = originalText.Length > 2 && originalText[1] == '@';
Debug.Assert(originalToken.Kind == SyntaxKind.InterpolatedStringToken);
var interpolations = ArrayBuilder<Lexer.Interpolation>.GetInstance();
SyntaxDiagnosticInfo error = null;
bool closeQuoteMissing;
using (var tempLexer = new Lexer(Text.SourceText.From(originalText), this.Options, allowPreprocessorDirectives: false))
{
// compute the positions of the interpolations in the original string literal, and also compute/preserve
// lexical errors
var info = default(Lexer.TokenInfo);
tempLexer.ScanInterpolatedStringLiteralTop(interpolations, isVerbatim, ref info, ref error, out closeQuoteMissing);
}
// Make a token for the open quote $" or $@"
var openQuoteIndex = isVerbatim ? 2 : 1;
Debug.Assert(originalText[openQuoteIndex] == '"');
var openQuote = SyntaxFactory.Token(
originalToken.GetLeadingTrivia(), isVerbatim ? SyntaxKind.InterpolatedVerbatimStringStartToken : SyntaxKind.InterpolatedStringStartToken, null);
// Make a token for the close quote " (even if it was missing)
var closeQuoteIndex = closeQuoteMissing ? originalText.Length : originalText.Length - 1;
Debug.Assert(closeQuoteMissing || originalText[closeQuoteIndex] == '"');
var closeQuote = closeQuoteMissing
? SyntaxFactory.MissingToken(SyntaxKind.InterpolatedStringEndToken).TokenWithTrailingTrivia(originalToken.GetTrailingTrivia())
: SyntaxFactory.Token(null, SyntaxKind.InterpolatedStringEndToken, originalToken.GetTrailingTrivia());
var builder = _pool.Allocate<InterpolatedStringContentSyntax>();
if (interpolations.Count == 0)
{
// In the special case when there are no interpolations, we just construct a format string
// with no inserts. We must still use String.Format to get its handling of escapes such as {{,
// so we still treat it as a composite format string.
var text = Substring(originalText, openQuoteIndex + 1, closeQuoteIndex - 1);
if (text.Length > 0)
{
var token = MakeStringToken(text, text, isVerbatim, SyntaxKind.InterpolatedStringTextToken);
builder.Add(SyntaxFactory.InterpolatedStringText(token));
}
}
else
{
for (int i = 0; i < interpolations.Count; i++)
{
var interpolation = interpolations[i];
// Add a token for text preceding the interpolation
var text = Substring(originalText, (i == 0) ? (openQuoteIndex + 1) : (interpolations[i - 1].CloseBracePosition + 1), interpolation.OpenBracePosition - 1);
if (text.Length > 0)
{
var token = MakeStringToken(text, text, isVerbatim, SyntaxKind.InterpolatedStringTextToken);
builder.Add(SyntaxFactory.InterpolatedStringText(token));
}
// Add an interpolation
var interp = ParseInterpolation(originalText, interpolation, isVerbatim);
builder.Add(interp);
}
// Add a token for text following the last interpolation
var lastText = Substring(originalText, interpolations[interpolations.Count - 1].CloseBracePosition + 1, closeQuoteIndex - 1);
if (lastText.Length > 0)
{
var token = MakeStringToken(lastText, lastText, isVerbatim, SyntaxKind.InterpolatedStringTextToken);
builder.Add(SyntaxFactory.InterpolatedStringText(token));
}
}
interpolations.Free();
var result = SyntaxFactory.InterpolatedStringExpression(openQuote, builder, closeQuote);
_pool.Free(builder);
if (error != null)
{
result = result.WithDiagnosticsGreen(new[] { error });
}
Debug.Assert(originalToken.ToFullString() == result.ToFullString()); // yield from text equals yield from node
return CheckFeatureAvailability(result, MessageID.IDS_FeatureInterpolatedStrings);
}
/// <summary>
/// Take the given text and treat it as the contents of a string literal, returning a token for that.
/// </summary>
/// <param name="text">The text for the full string literal, including the quotes and contents</param>
/// <param name="bodyText">The text for the string literal's contents, excluding surrounding quotes</param>
/// <param name="isVerbatim">True if the string contents should be scanned using the rules for verbatim strings</param>
/// <param name="kind">The token kind to be assigned to the resulting token</param>
private SyntaxToken MakeStringToken(string text, string bodyText, bool isVerbatim, SyntaxKind kind)
{
var prefix = isVerbatim ? "@\"" : "\"";
var fakeString = prefix + bodyText + "\"";
using (var tempLexer = new Lexer(Text.SourceText.From(fakeString), this.Options, allowPreprocessorDirectives: false))
{
LexerMode mode = LexerMode.Syntax;
SyntaxToken token = tempLexer.Lex(ref mode);
Debug.Assert(token.Kind == SyntaxKind.StringLiteralToken);
var result = SyntaxFactory.Literal(null, text, kind, token.ValueText, null);
if (token.ContainsDiagnostics)
{
result = result.WithDiagnosticsGreen(MoveDiagnostics(token.GetDiagnostics(), -prefix.Length));
}
return result;
}
}
private InterpolationSyntax ParseInterpolation(string text, Lexer.Interpolation interpolation, bool isVerbatim)
{
SyntaxToken openBraceToken;
ExpressionSyntax expression;
InterpolationAlignmentClauseSyntax alignment = null;
InterpolationFormatClauseSyntax format = null;
var closeBraceToken = interpolation.CloseBraceMissing
? SyntaxFactory.MissingToken(SyntaxKind.CloseBraceToken)
: SyntaxFactory.Token(SyntaxKind.CloseBraceToken);
var parsedText = Substring(text, interpolation.OpenBracePosition, interpolation.HasColon ? interpolation.ColonPosition - 1 : interpolation.CloseBracePosition - 1);
using (var tempLexer = new Lexer(Text.SourceText.From(parsedText), this.Options, allowPreprocessorDirectives: false))
{
// TODO: some of the trivia in the interpolation maybe should be trailing trivia of the openBraceToken
using (var tempParser = new LanguageParser(tempLexer, null, null))
{
SyntaxToken commaToken = null;
ExpressionSyntax alignmentExpression = null;
tempParser.ParseInterpolationStart(out openBraceToken, out expression, out commaToken, out alignmentExpression);
if (alignmentExpression != null)
{
alignment = SyntaxFactory.InterpolationAlignmentClause(commaToken, alignmentExpression);
}
var extraTrivia = tempParser.CurrentToken.GetLeadingTrivia();
if (interpolation.HasColon)
{
var colonToken = SyntaxFactory.Token(SyntaxKind.ColonToken).TokenWithLeadingTrivia(extraTrivia);
var formatText = Substring(text, interpolation.ColonPosition + 1, interpolation.FormatEndPosition);
var formatString = MakeStringToken(formatText, formatText, isVerbatim, SyntaxKind.InterpolatedStringTextToken);
format = SyntaxFactory.InterpolationFormatClause(colonToken, formatString);
}
else
{
// Move the leading trivia from the insertion's EOF token to the following token.
closeBraceToken = closeBraceToken.TokenWithLeadingTrivia(extraTrivia);
}
}
}
var result = SyntaxFactory.Interpolation(openBraceToken, expression, alignment, format, closeBraceToken);
Debug.Assert(Substring(text, interpolation.OpenBracePosition, interpolation.LastPosition) == result.ToFullString()); // yield from text equals yield from node
return result;
}
public InterpolatedStringScanner(Lexer lexer, bool isVerbatim)
{
_lexer = lexer;
_isVerbatim = isVerbatim;
}
private static InternalSyntax.ExpressionSyntax ParseDebuggerExpressionInternal(SourceText source, bool consumeFullText)
{
using (var lexer = new InternalSyntax.Lexer(source, CSharpParseOptions.Default, allowPreprocessorDirectives: false))
{
using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax))
{
var node = parser.ParseExpression();
if (consumeFullText) node = parser.ConsumeUnexpectedTokens(node);
return node;
}
}
}
private ExpressionSyntax ParseInterpolatedStringToken()
{
// We don't want to make the scanner stateful (between tokens) if we can possibly avoid it.
// The approach implemented here is
//
// (1) Scan the whole interpolated string literal as a single token. Now the statefulness of
// the scanner (to match { }'s) is limited to its behavior while scanning a single token.
//
// (2) When the parser gets such a token, here, it spins up another scanner / parser on each of
// the holes and builds a tree for the whole thing (resulting in an InterpolatedStringExpressionSyntax).
//
// (3) The parser discards the original token and replaces it with this tree. (In other words,
// it replaces one token with a different set of tokens that have already been parsed)
//
// (4) On an incremental change, we widen the invalidated region to include any enclosing interpolated
// string nonterminal so that we never reuse tokens inside a changed interpolated string.
//
// This has the secondary advantage that it can reasonably be specified.
//
// The substitution will end up being invisible to external APIs and clients such as the IDE, as
// they have no way to ask for the stream of tokens before parsing.
//
var originalToken = this.EatToken();
var originalText = originalToken.ValueText; // this is actually the source text
Debug.Assert(originalText[0] == '$' || originalText[0] == '@');
var isAltInterpolatedVerbatim = originalText.Length > 2 && originalText[0] == '@'; // @$
var isVerbatim =
isAltInterpolatedVerbatim || (originalText.Length > 2 && originalText[1] == '@');
Debug.Assert(originalToken.Kind == SyntaxKind.InterpolatedStringToken);
var interpolations = ArrayBuilder <Lexer.Interpolation> .GetInstance();
SyntaxDiagnosticInfo error = null;
bool closeQuoteMissing;
using (
var tempLexer = new Lexer(
Text.SourceText.From(originalText),
this.Options,
allowPreprocessorDirectives: false
)
)
{
// compute the positions of the interpolations in the original string literal, and also compute/preserve
// lexical errors
var info = default(Lexer.TokenInfo);
tempLexer.ScanInterpolatedStringLiteralTop(
interpolations,
isVerbatim,
ref info,
ref error,
out closeQuoteMissing
);
}
// Make a token for the open quote $" or $@" or @$"
var openQuoteIndex = isVerbatim ? 2 : 1;
Debug.Assert(originalText[openQuoteIndex] == '"');
var openQuoteKind = isVerbatim
? SyntaxKind.InterpolatedVerbatimStringStartToken // $@ or @$
: SyntaxKind.InterpolatedStringStartToken; // $
var openQuoteText = isAltInterpolatedVerbatim
? "@$\""
: isVerbatim
? "$@\""
: "$\"";
var openQuote = SyntaxFactory.Token(
originalToken.GetLeadingTrivia(),
openQuoteKind,
openQuoteText,
openQuoteText,
trailing: null
);
if (isAltInterpolatedVerbatim)
{
openQuote = CheckFeatureAvailability(
openQuote,
MessageID.IDS_FeatureAltInterpolatedVerbatimStrings
);
}
// Make a token for the close quote " (even if it was missing)
var closeQuoteIndex = closeQuoteMissing ? originalText.Length : originalText.Length - 1;
Debug.Assert(closeQuoteMissing || originalText[closeQuoteIndex] == '"');
var closeQuote = closeQuoteMissing
? SyntaxFactory
.MissingToken(SyntaxKind.InterpolatedStringEndToken)
.TokenWithTrailingTrivia(originalToken.GetTrailingTrivia())
: SyntaxFactory.Token(
null,
SyntaxKind.InterpolatedStringEndToken,
originalToken.GetTrailingTrivia()
);
var builder = _pool.Allocate <InterpolatedStringContentSyntax>();
if (interpolations.Count == 0)
{
// In the special case when there are no interpolations, we just construct a format string
// with no inserts. We must still use String.Format to get its handling of escapes such as {{,
// so we still treat it as a composite format string.
var text = Substring(originalText, openQuoteIndex + 1, closeQuoteIndex - 1);
if (text.Length > 0)
{
var token = MakeStringToken(
text,
text,
isVerbatim,
SyntaxKind.InterpolatedStringTextToken
);
builder.Add(SyntaxFactory.InterpolatedStringText(token));
}
}
else
{
for (int i = 0; i < interpolations.Count; i++)
{
var interpolation = interpolations[i];
// Add a token for text preceding the interpolation
var text = Substring(
originalText,
(i == 0)
? (openQuoteIndex + 1)
: (interpolations[i - 1].CloseBracePosition + 1),
interpolation.OpenBracePosition - 1
);
if (text.Length > 0)
{
var token = MakeStringToken(
text,
text,
isVerbatim,
SyntaxKind.InterpolatedStringTextToken
);
builder.Add(SyntaxFactory.InterpolatedStringText(token));
}
// Add an interpolation
var interp = ParseInterpolation(originalText, interpolation, isVerbatim);
builder.Add(interp);
}
// Add a token for text following the last interpolation
var lastText = Substring(
originalText,
interpolations[interpolations.Count - 1].CloseBracePosition + 1,
closeQuoteIndex - 1
);
if (lastText.Length > 0)
{
var token = MakeStringToken(
lastText,
lastText,
isVerbatim,
SyntaxKind.InterpolatedStringTextToken
);
builder.Add(SyntaxFactory.InterpolatedStringText(token));
}
}
interpolations.Free();
var result = SyntaxFactory.InterpolatedStringExpression(openQuote, builder, closeQuote);
_pool.Free(builder);
if (error != null)
{
result = result.WithDiagnosticsGreen(new[] { error });
}
Debug.Assert(originalToken.ToFullString() == result.ToFullString()); // yield from text equals yield from node
return(CheckFeatureAvailability(result, MessageID.IDS_FeatureInterpolatedStrings));
}
internal override IEnumerable<InternalSyntax.SyntaxToken> GetTokens(string text)
{
Assert.DoesNotContain("'", text, StringComparison.Ordinal);
using (var lexer = new InternalSyntax.Lexer(SourceText.From(text + "'"), TestOptions.RegularWithDocumentationComments))
{
while (true)
{
var token = lexer.Lex(InternalSyntax.LexerMode.XmlNameQuote | InternalSyntax.LexerMode.XmlDocCommentStyleSingleLine | InternalSyntax.LexerMode.XmlDocCommentLocationInterior);
if (token.Kind == SyntaxKind.SingleQuoteToken)
{
break;
}
yield return token;
}
}
}
public InterpolatedStringScanner(Lexer lexer, bool isVerbatim)
{
this.lexer = lexer;
this.isVerbatim = isVerbatim;
this.allowNewlines = isVerbatim;
}
