private SyntaxToken DebuggerLex(string text) { using (var lexer = new InternalSyntax.Lexer(SourceText.From(text), _options)) { return new SyntaxToken(lexer.Lex(InternalSyntax.LexerMode.DebuggerSyntax)); } }
/// <summary> /// Produces a syntax tree by parsing the source text. /// </summary> public static SyntaxTree ParseText( SourceText text, CSharpParseOptions options = null, string path = "", CancellationToken cancellationToken = default(CancellationToken)) { if (text == null) { throw new ArgumentNullException("text"); } if (path == null) { throw new ArgumentNullException("path"); } options = options ?? CSharpParseOptions.Default; using (var lexer = new InternalSyntax.Lexer(text, options)) { using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken)) { var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed(); var tree = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, parser.Directives); tree.VerifySource(); return(tree); } } }
internal override IEnumerable <InternalSyntax.SyntaxToken> GetTokens(string text) { Assert.DoesNotContain("'", text, StringComparison.Ordinal); using ( var lexer = new InternalSyntax.Lexer( SourceText.From(text + "'"), TestOptions.RegularWithDocumentationComments ) ) { while (true) { var token = lexer.Lex( InternalSyntax.LexerMode.XmlNameQuote | InternalSyntax.LexerMode.XmlDocCommentStyleSingleLine | InternalSyntax.LexerMode.XmlDocCommentLocationInterior ); if (token.Kind == SyntaxKind.SingleQuoteToken) { break; } yield return(token); } } }
private static InternalSyntax.ExpressionSyntax ParseDebuggerExpressionInternal( SourceText source, bool consumeFullText ) { using var lexer = new InternalSyntax.Lexer( source, ParseOptions, allowPreprocessorDirectives: false ); using var parser = new InternalSyntax.LanguageParser( lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax ); var node = parser.ParseExpression(); if (consumeFullText) { node = parser.ConsumeUnexpectedTokens(node); } return(node); }
private SyntaxTree WithChanges(SourceText newText, IReadOnlyList <TextChangeRange> changes) { if (changes == null) { throw new ArgumentNullException(nameof(changes)); } var oldTree = this; // if changes is entire text do a full reparse if (changes.Count == 1 && changes[0].Span == new TextSpan(0, this.Length) && changes[0].NewLength == newText.Length) { // parser will do a full parse if we give it no changes changes = null; oldTree = null; } using (var lexer = new InternalSyntax.Lexer(newText, this.Options)) using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree?.GetRoot(), changes)) { var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed(); var tree = new ParsedSyntaxTree( newText, newText.Encoding, newText.ChecksumAlgorithm, FilePath, Options, compilationUnit, parser.Directives, DiagnosticOptions); tree.VerifySource(changes); return(tree); } }
private SyntaxTree WithChanges(SourceText newText, IReadOnlyList <TextChangeRange> changes) { if (changes == null) { throw new ArgumentNullException("changes"); } var oldTree = this; // if changes is entire text do a full reparse if (changes.Count == 1 && changes[0].Span == new TextSpan(0, this.Length) && changes[0].NewLength == newText.Length) { // parser will do a full parse if we give it no changes changes = null; oldTree = null; } using (var lexer = new InternalSyntax.Lexer(newText, this.Options)) { CSharp.CSharpSyntaxNode oldRoot = oldTree != null?oldTree.GetRoot() : null; using (var parser = new InternalSyntax.LanguageParser(lexer, oldRoot, changes)) { var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed(); var tree = new ParsedSyntaxTree(newText, this.FilePath, this.Options, compilationUnit, parser.Directives); tree.VerifySource(changes); return(tree); } } }
public Reader(Blender blender) { this.lexer = blender.lexer; this.oldTreeCursor = blender.oldTreeCursor; this.changes = blender.changes; this.newPosition = blender.newPosition; this.changeDelta = blender.changeDelta; this.newDirectives = blender.newDirectives; this.oldDirectives = blender.oldDirectives; this.newLexerDrivenMode = blender.newLexerDrivenMode; }
public Reader(Blender blender) { _lexer = blender._lexer; _oldTreeCursor = blender._oldTreeCursor; _changes = blender._changes; _newPosition = blender._newPosition; _changeDelta = blender._changeDelta; _newDirectives = blender._newDirectives; _oldDirectives = blender._oldDirectives; _newLexerDrivenMode = blender._newLexerDrivenMode; }
private static StatementSyntax ParseDebuggerStatement(string text) { var source = SourceText.From(text); using var lexer = new InternalSyntax.Lexer(source, ParseOptions); using var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax); var statement = parser.ParseStatement(); var syntaxTree = statement.CreateSyntaxTree(source); return((StatementSyntax)syntaxTree.GetRoot()); }
private IEnumerable<InternalSyntax.BlendedNode> Blend(string text) { using (var lexer = new InternalSyntax.Lexer(SourceText.From(text), _options)) { var blender = new InternalSyntax.Blender(lexer, null, null); InternalSyntax.BlendedNode result; do { result = blender.ReadToken(InternalSyntax.LexerMode.Syntax); blender = result.Blender; yield return result; } while (result.Token.Kind != SyntaxKind.EndOfFileToken); } }
public Blender(Lexer lexer, CSharp.CSharpSyntaxNode oldTree, IEnumerable<TextChangeRange> changes) { Debug.Assert(lexer != null); _lexer = lexer; _changes = ImmutableStack.Create<TextChangeRange>(); if (changes != null) { // TODO: Consider implementing NormalizedChangeCollection for TextSpan. the real // reason why we are collapsing is because we want to extend change ranges and // cannot allow them to overlap. This does not seem to be a big deal since multiple // changes are infrequent and typically close to each other. However if we have // NormalizedChangeCollection for TextSpan we can have both - we can extend ranges // and not require collapsing them. NormalizedChangeCollection would also ensure // that changes are always normalized. // TODO: this is a temporary measure to prevent individual change spans from // overlapping after they are widened to effective width (+1 token at the start). // once we have normalized collection for TextSpan we will not need to collapse all // the change spans. var collapsed = TextChangeRange.Collapse(changes); // extend the change to its affected range. This will make it easier // to filter out affected nodes since we will be able simply check // if node intersects with a change. var affectedRange = ExtendToAffectedRange(oldTree, collapsed); _changes = _changes.Push(affectedRange); } if (oldTree == null) { // start at lexer current position if no nodes specified _oldTreeCursor = new Cursor(); _newPosition = lexer.TextWindow.Position; } else { _oldTreeCursor = Cursor.FromRoot(oldTree).MoveToFirstChild(); _newPosition = 0; } _changeDelta = 0; _newDirectives = default(DirectiveStack); _oldDirectives = default(DirectiveStack); _newLexerDrivenMode = 0; }
private Blender( Lexer lexer, Cursor oldTreeCursor, ImmutableStack<TextChangeRange> changes, int newPosition, int changeDelta, DirectiveStack newDirectives, DirectiveStack oldDirectives, LexerMode newLexerDrivenMode) { Debug.Assert(lexer != null); Debug.Assert(changes != null); Debug.Assert(newPosition >= 0); _lexer = lexer; _oldTreeCursor = oldTreeCursor; _changes = changes; _newPosition = newPosition; _changeDelta = changeDelta; _newDirectives = newDirectives; _oldDirectives = oldDirectives; _newLexerDrivenMode = newLexerDrivenMode & (LexerMode.MaskXmlDocCommentLocation | LexerMode.MaskXmlDocCommentStyle); }
/// <summary> /// Produces a syntax tree by parsing the source text. /// </summary> public static SyntaxTree ParseText( SourceText text, CSharpParseOptions options = null, string path = "", ImmutableDictionary <string, ReportDiagnostic> diagnosticOptions = null, bool?isGeneratedCode = null, CancellationToken cancellationToken = default) { if (text == null) { throw new ArgumentNullException(nameof(text)); } options = options ?? CSharpParseOptions.Default; using (var lexer = new InternalSyntax.Lexer(text, options)) { using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken)) { var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed(); var tree = new ParsedSyntaxTree( text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, parser.Directives, diagnosticOptions: diagnosticOptions, isGeneratedCode: isGeneratedCode, cloneRoot: true); tree.VerifySource(); return(tree); } } }
internal DirectiveParser(Lexer lexer, DirectiveStack context) : base(lexer, LexerMode.Directive, null, null, false) { this.context = context; }
/// <summary> /// Produces a syntax tree by parsing the source text. /// </summary> public static SyntaxTree ParseText( SourceText text, CSharpParseOptions options = null, string path = "", CancellationToken cancellationToken = default(CancellationToken)) { if (text == null) { throw new ArgumentNullException(nameof(text)); } #if XSHARPPRE { string s = text.ToString(); s = s.Replace("Microsoft.CodeAnalysis.CSharp", "LanguageService.CodeAnalysis.XSharp"); s = s.Replace("Microsoft.CodeAnalysis", "LanguageService.CodeAnalysis"); s = s.Replace("Microsoft.Cci", "LanguageService.Cci"); s = (new System.Text.RegularExpressions.Regex(@"(?<!\.)CSharpResources")).Replace(s, "LanguageService.CodeAnalysis.XSharpResources"); s = (new System.Text.RegularExpressions.Regex(@"(?<!Microsoft[\._][A-Za-z0-9_\.]*)CSharp")).Replace(s, "XSharp"); s = s.Replace("XSharp.XSharpResources", "XSharpResources"); s = s.Replace("Antlr4.Runtime", "LanguageService.SyntaxTree"); s = s.Replace("XSHARP_RUNTIME", "true"); text = SourceText.From(s, text.Encoding); } #endif options = options ?? CSharpParseOptions.Default; #if XSHARP using (var parser = new InternalSyntax.XSharpLanguageParser(path, text, options, oldTree: null, changes: null, cancellationToken: cancellationToken)) { var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed(); var tree = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, default(InternalSyntax.DirectiveStack)); //tree.VerifySource(); if (options.SaveAsCSharp) { path = System.IO.Path.ChangeExtension(path, ".cs"); string source = compilationUnit.ToString(); source = source.Replace(";", ";\r\n"); source = source.Replace("{", "\r\n{\r\n"); source = source.Replace("}", "\r\n}\r\n"); source = source.Replace(" . ", "."); source = source.Replace(" :: ", "::"); source = source.Replace("}", "}\r\n"); source = source.Replace("$", "_"); System.IO.File.WriteAllText(path, source); } return(tree); } #else using (var lexer = new InternalSyntax.Lexer(text, options)) { using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, cancellationToken: cancellationToken)) { var compilationUnit = (CompilationUnitSyntax)parser.ParseCompilationUnit().CreateRed(); var tree = new ParsedSyntaxTree(text, text.Encoding, text.ChecksumAlgorithm, path, options, compilationUnit, parser.Directives); tree.VerifySource(); return(tree); } } #endif }
internal DirectiveParser(Lexer lexer, DirectiveStack context) : base(lexer, LexerMode.Directive, null, null, false) { _context = context; }
private InterpolationSyntax ParseInterpolation( string text, Lexer.Interpolation interpolation, bool isVerbatim ) { SyntaxToken openBraceToken; ExpressionSyntax expression; InterpolationAlignmentClauseSyntax alignment = null; InterpolationFormatClauseSyntax format = null; var closeBraceToken = interpolation.CloseBraceMissing ? SyntaxFactory.MissingToken(SyntaxKind.CloseBraceToken) : SyntaxFactory.Token(SyntaxKind.CloseBraceToken); var parsedText = Substring( text, interpolation.OpenBracePosition, interpolation.HasColon ? interpolation.ColonPosition - 1 : interpolation.CloseBracePosition - 1 ); using ( var tempLexer = new Lexer( Text.SourceText.From(parsedText), this.Options, allowPreprocessorDirectives: false, interpolationFollowedByColon: interpolation.HasColon ) ) { // TODO: some of the trivia in the interpolation maybe should be trailing trivia of the openBraceToken using (var tempParser = new LanguageParser(tempLexer, null, null)) { SyntaxToken commaToken = null; ExpressionSyntax alignmentExpression = null; tempParser.ParseInterpolationStart( out openBraceToken, out expression, out commaToken, out alignmentExpression ); if (alignmentExpression != null) { alignment = SyntaxFactory.InterpolationAlignmentClause( commaToken, alignmentExpression ); } var extraTrivia = tempParser.CurrentToken.GetLeadingTrivia(); if (interpolation.HasColon) { var colonToken = SyntaxFactory .Token(SyntaxKind.ColonToken) .TokenWithLeadingTrivia(extraTrivia); var formatText = Substring( text, interpolation.ColonPosition + 1, interpolation.FormatEndPosition ); var formatString = MakeStringToken( formatText, formatText, isVerbatim, SyntaxKind.InterpolatedStringTextToken ); format = SyntaxFactory.InterpolationFormatClause(colonToken, formatString); } else { // Move the leading trivia from the insertion's EOF token to the following token. closeBraceToken = closeBraceToken.TokenWithLeadingTrivia(extraTrivia); } } } var result = SyntaxFactory.Interpolation( openBraceToken, expression, alignment, format, closeBraceToken ); Debug.Assert( Substring(text, interpolation.OpenBracePosition, interpolation.LastPosition) == result.ToFullString() ); // yield from text equals yield from node return(result); }
private static StatementSyntax ParseDebuggerStatement(string text) { var source = SourceText.From(text); using (var lexer = new InternalSyntax.Lexer(source, CSharpParseOptions.Default)) { using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax)) { var statement = parser.ParseStatement(); var syntaxTree = statement.CreateSyntaxTree(source); return (StatementSyntax)syntaxTree.GetRoot(); } } }
public InterpolatedStringScanner( Lexer lexer, bool isVerbatim) { this.lexer = lexer; this.isVerbatim = isVerbatim; this.allowNewlines = isVerbatim; }
private ExpressionSyntax ParseInterpolatedStringToken() { // We don't want to make the scanner stateful (between tokens) if we can possibly avoid it. // The approach implemented here is // // (1) Scan the whole interpolated string literal as a single token. Now the statefulness of // the scanner (to match { }'s) is limited to its behavior while scanning a single token. // // (2) When the parser gets such a token, here, it spins up another scanner / parser on each of // the holes and builds a tree for the whole thing (resulting in an InterpolatedStringExpressionSyntax). // // (3) The parser discards the original token and replaces it with this tree. (In other words, // it replaces one token with a different set of tokens that have already been parsed) // // (4) On an incremental change, we widen the invalidated region to include any enclosing interpolated // string nonterminal so that we never reuse tokens inside a changed interpolated string. // // This has the secondary advantage that it can reasonably be specified. // // The substitution will end up being invisible to external APIs and clients such as the IDE, as // they have no way to ask for the stream of tokens before parsing. // var originalToken = this.EatToken(); var originalText = originalToken.ValueText; // this is actually the source text Debug.Assert(originalText[0] == '$'); var isVerbatim = originalText.Length > 2 && originalText[1] == '@'; Debug.Assert(originalToken.Kind == SyntaxKind.InterpolatedStringToken); var interpolations = ArrayBuilder<Lexer.Interpolation>.GetInstance(); SyntaxDiagnosticInfo error = null; bool closeQuoteMissing; using (var tempLexer = new Lexer(Text.SourceText.From(originalText), this.Options, allowPreprocessorDirectives: false)) { // compute the positions of the interpolations in the original string literal, and also compute/preserve // lexical errors var info = default(Lexer.TokenInfo); tempLexer.ScanInterpolatedStringLiteralTop(interpolations, isVerbatim, ref info, ref error, out closeQuoteMissing); } // Make a token for the open quote $" or $@" var openQuoteIndex = isVerbatim ? 2 : 1; Debug.Assert(originalText[openQuoteIndex] == '"'); var openQuote = SyntaxFactory.Token( originalToken.GetLeadingTrivia(), isVerbatim ? SyntaxKind.InterpolatedVerbatimStringStartToken : SyntaxKind.InterpolatedStringStartToken, null); // Make a token for the close quote " (even if it was missing) var closeQuoteIndex = closeQuoteMissing ? originalText.Length : originalText.Length - 1; Debug.Assert(closeQuoteMissing || originalText[closeQuoteIndex] == '"'); var closeQuote = closeQuoteMissing ? SyntaxFactory.MissingToken(SyntaxKind.InterpolatedStringEndToken).TokenWithTrailingTrivia(originalToken.GetTrailingTrivia()) : SyntaxFactory.Token(null, SyntaxKind.InterpolatedStringEndToken, originalToken.GetTrailingTrivia()); var builder = _pool.Allocate<InterpolatedStringContentSyntax>(); if (interpolations.Count == 0) { // In the special case when there are no interpolations, we just construct a format string // with no inserts. We must still use String.Format to get its handling of escapes such as {{, // so we still treat it as a composite format string. var text = Substring(originalText, openQuoteIndex + 1, closeQuoteIndex - 1); if (text.Length > 0) { var token = MakeStringToken(text, text, isVerbatim, SyntaxKind.InterpolatedStringTextToken); builder.Add(SyntaxFactory.InterpolatedStringText(token)); } } else { for (int i = 0; i < interpolations.Count; i++) { var interpolation = interpolations[i]; // Add a token for text preceding the interpolation var text = Substring(originalText, (i == 0) ? (openQuoteIndex + 1) : (interpolations[i - 1].CloseBracePosition + 1), interpolation.OpenBracePosition - 1); if (text.Length > 0) { var token = MakeStringToken(text, text, isVerbatim, SyntaxKind.InterpolatedStringTextToken); builder.Add(SyntaxFactory.InterpolatedStringText(token)); } // Add an interpolation var interp = ParseInterpolation(originalText, interpolation, isVerbatim); builder.Add(interp); } // Add a token for text following the last interpolation var lastText = Substring(originalText, interpolations[interpolations.Count - 1].CloseBracePosition + 1, closeQuoteIndex - 1); if (lastText.Length > 0) { var token = MakeStringToken(lastText, lastText, isVerbatim, SyntaxKind.InterpolatedStringTextToken); builder.Add(SyntaxFactory.InterpolatedStringText(token)); } } interpolations.Free(); var result = SyntaxFactory.InterpolatedStringExpression(openQuote, builder, closeQuote); _pool.Free(builder); if (error != null) { result = result.WithDiagnosticsGreen(new[] { error }); } Debug.Assert(originalToken.ToFullString() == result.ToFullString()); // yield from text equals yield from node return CheckFeatureAvailability(result, MessageID.IDS_FeatureInterpolatedStrings); }
/// <summary> /// Take the given text and treat it as the contents of a string literal, returning a token for that. /// </summary> /// <param name="text">The text for the full string literal, including the quotes and contents</param> /// <param name="bodyText">The text for the string literal's contents, excluding surrounding quotes</param> /// <param name="isVerbatim">True if the string contents should be scanned using the rules for verbatim strings</param> /// <param name="kind">The token kind to be assigned to the resulting token</param> private SyntaxToken MakeStringToken(string text, string bodyText, bool isVerbatim, SyntaxKind kind) { var prefix = isVerbatim ? "@\"" : "\""; var fakeString = prefix + bodyText + "\""; using (var tempLexer = new Lexer(Text.SourceText.From(fakeString), this.Options, allowPreprocessorDirectives: false)) { LexerMode mode = LexerMode.Syntax; SyntaxToken token = tempLexer.Lex(ref mode); Debug.Assert(token.Kind == SyntaxKind.StringLiteralToken); var result = SyntaxFactory.Literal(null, text, kind, token.ValueText, null); if (token.ContainsDiagnostics) { result = result.WithDiagnosticsGreen(MoveDiagnostics(token.GetDiagnostics(), -prefix.Length)); } return result; } }
private InterpolationSyntax ParseInterpolation(string text, Lexer.Interpolation interpolation, bool isVerbatim) { SyntaxToken openBraceToken; ExpressionSyntax expression; InterpolationAlignmentClauseSyntax alignment = null; InterpolationFormatClauseSyntax format = null; var closeBraceToken = interpolation.CloseBraceMissing ? SyntaxFactory.MissingToken(SyntaxKind.CloseBraceToken) : SyntaxFactory.Token(SyntaxKind.CloseBraceToken); var parsedText = Substring(text, interpolation.OpenBracePosition, interpolation.HasColon ? interpolation.ColonPosition - 1 : interpolation.CloseBracePosition - 1); using (var tempLexer = new Lexer(Text.SourceText.From(parsedText), this.Options, allowPreprocessorDirectives: false)) { // TODO: some of the trivia in the interpolation maybe should be trailing trivia of the openBraceToken using (var tempParser = new LanguageParser(tempLexer, null, null)) { SyntaxToken commaToken = null; ExpressionSyntax alignmentExpression = null; tempParser.ParseInterpolationStart(out openBraceToken, out expression, out commaToken, out alignmentExpression); if (alignmentExpression != null) { alignment = SyntaxFactory.InterpolationAlignmentClause(commaToken, alignmentExpression); } var extraTrivia = tempParser.CurrentToken.GetLeadingTrivia(); if (interpolation.HasColon) { var colonToken = SyntaxFactory.Token(SyntaxKind.ColonToken).TokenWithLeadingTrivia(extraTrivia); var formatText = Substring(text, interpolation.ColonPosition + 1, interpolation.FormatEndPosition); var formatString = MakeStringToken(formatText, formatText, isVerbatim, SyntaxKind.InterpolatedStringTextToken); format = SyntaxFactory.InterpolationFormatClause(colonToken, formatString); } else { // Move the leading trivia from the insertion's EOF token to the following token. closeBraceToken = closeBraceToken.TokenWithLeadingTrivia(extraTrivia); } } } var result = SyntaxFactory.Interpolation(openBraceToken, expression, alignment, format, closeBraceToken); Debug.Assert(Substring(text, interpolation.OpenBracePosition, interpolation.LastPosition) == result.ToFullString()); // yield from text equals yield from node return result; }
public InterpolatedStringScanner(Lexer lexer, bool isVerbatim) { _lexer = lexer; _isVerbatim = isVerbatim; }
private static InternalSyntax.ExpressionSyntax ParseDebuggerExpressionInternal(SourceText source, bool consumeFullText) { using (var lexer = new InternalSyntax.Lexer(source, CSharpParseOptions.Default, allowPreprocessorDirectives: false)) { using (var parser = new InternalSyntax.LanguageParser(lexer, oldTree: null, changes: null, lexerMode: InternalSyntax.LexerMode.DebuggerSyntax)) { var node = parser.ParseExpression(); if (consumeFullText) node = parser.ConsumeUnexpectedTokens(node); return node; } } }
private ExpressionSyntax ParseInterpolatedStringToken() { // We don't want to make the scanner stateful (between tokens) if we can possibly avoid it. // The approach implemented here is // // (1) Scan the whole interpolated string literal as a single token. Now the statefulness of // the scanner (to match { }'s) is limited to its behavior while scanning a single token. // // (2) When the parser gets such a token, here, it spins up another scanner / parser on each of // the holes and builds a tree for the whole thing (resulting in an InterpolatedStringExpressionSyntax). // // (3) The parser discards the original token and replaces it with this tree. (In other words, // it replaces one token with a different set of tokens that have already been parsed) // // (4) On an incremental change, we widen the invalidated region to include any enclosing interpolated // string nonterminal so that we never reuse tokens inside a changed interpolated string. // // This has the secondary advantage that it can reasonably be specified. // // The substitution will end up being invisible to external APIs and clients such as the IDE, as // they have no way to ask for the stream of tokens before parsing. // var originalToken = this.EatToken(); var originalText = originalToken.ValueText; // this is actually the source text Debug.Assert(originalText[0] == '$' || originalText[0] == '@'); var isAltInterpolatedVerbatim = originalText.Length > 2 && originalText[0] == '@'; // @$ var isVerbatim = isAltInterpolatedVerbatim || (originalText.Length > 2 && originalText[1] == '@'); Debug.Assert(originalToken.Kind == SyntaxKind.InterpolatedStringToken); var interpolations = ArrayBuilder <Lexer.Interpolation> .GetInstance(); SyntaxDiagnosticInfo error = null; bool closeQuoteMissing; using ( var tempLexer = new Lexer( Text.SourceText.From(originalText), this.Options, allowPreprocessorDirectives: false ) ) { // compute the positions of the interpolations in the original string literal, and also compute/preserve // lexical errors var info = default(Lexer.TokenInfo); tempLexer.ScanInterpolatedStringLiteralTop( interpolations, isVerbatim, ref info, ref error, out closeQuoteMissing ); } // Make a token for the open quote $" or $@" or @$" var openQuoteIndex = isVerbatim ? 2 : 1; Debug.Assert(originalText[openQuoteIndex] == '"'); var openQuoteKind = isVerbatim ? SyntaxKind.InterpolatedVerbatimStringStartToken // $@ or @$ : SyntaxKind.InterpolatedStringStartToken; // $ var openQuoteText = isAltInterpolatedVerbatim ? "@$\"" : isVerbatim ? "$@\"" : "$\""; var openQuote = SyntaxFactory.Token( originalToken.GetLeadingTrivia(), openQuoteKind, openQuoteText, openQuoteText, trailing: null ); if (isAltInterpolatedVerbatim) { openQuote = CheckFeatureAvailability( openQuote, MessageID.IDS_FeatureAltInterpolatedVerbatimStrings ); } // Make a token for the close quote " (even if it was missing) var closeQuoteIndex = closeQuoteMissing ? originalText.Length : originalText.Length - 1; Debug.Assert(closeQuoteMissing || originalText[closeQuoteIndex] == '"'); var closeQuote = closeQuoteMissing ? SyntaxFactory .MissingToken(SyntaxKind.InterpolatedStringEndToken) .TokenWithTrailingTrivia(originalToken.GetTrailingTrivia()) : SyntaxFactory.Token( null, SyntaxKind.InterpolatedStringEndToken, originalToken.GetTrailingTrivia() ); var builder = _pool.Allocate <InterpolatedStringContentSyntax>(); if (interpolations.Count == 0) { // In the special case when there are no interpolations, we just construct a format string // with no inserts. We must still use String.Format to get its handling of escapes such as {{, // so we still treat it as a composite format string. var text = Substring(originalText, openQuoteIndex + 1, closeQuoteIndex - 1); if (text.Length > 0) { var token = MakeStringToken( text, text, isVerbatim, SyntaxKind.InterpolatedStringTextToken ); builder.Add(SyntaxFactory.InterpolatedStringText(token)); } } else { for (int i = 0; i < interpolations.Count; i++) { var interpolation = interpolations[i]; // Add a token for text preceding the interpolation var text = Substring( originalText, (i == 0) ? (openQuoteIndex + 1) : (interpolations[i - 1].CloseBracePosition + 1), interpolation.OpenBracePosition - 1 ); if (text.Length > 0) { var token = MakeStringToken( text, text, isVerbatim, SyntaxKind.InterpolatedStringTextToken ); builder.Add(SyntaxFactory.InterpolatedStringText(token)); } // Add an interpolation var interp = ParseInterpolation(originalText, interpolation, isVerbatim); builder.Add(interp); } // Add a token for text following the last interpolation var lastText = Substring( originalText, interpolations[interpolations.Count - 1].CloseBracePosition + 1, closeQuoteIndex - 1 ); if (lastText.Length > 0) { var token = MakeStringToken( lastText, lastText, isVerbatim, SyntaxKind.InterpolatedStringTextToken ); builder.Add(SyntaxFactory.InterpolatedStringText(token)); } } interpolations.Free(); var result = SyntaxFactory.InterpolatedStringExpression(openQuote, builder, closeQuote); _pool.Free(builder); if (error != null) { result = result.WithDiagnosticsGreen(new[] { error }); } Debug.Assert(originalToken.ToFullString() == result.ToFullString()); // yield from text equals yield from node return(CheckFeatureAvailability(result, MessageID.IDS_FeatureInterpolatedStrings)); }
internal override IEnumerable<InternalSyntax.SyntaxToken> GetTokens(string text) { Assert.DoesNotContain("'", text, StringComparison.Ordinal); using (var lexer = new InternalSyntax.Lexer(SourceText.From(text + "'"), TestOptions.RegularWithDocumentationComments)) { while (true) { var token = lexer.Lex(InternalSyntax.LexerMode.XmlNameQuote | InternalSyntax.LexerMode.XmlDocCommentStyleSingleLine | InternalSyntax.LexerMode.XmlDocCommentLocationInterior); if (token.Kind == SyntaxKind.SingleQuoteToken) { break; } yield return token; } } }
public InterpolatedStringScanner(Lexer lexer, bool isVerbatim) { this.lexer = lexer; this.isVerbatim = isVerbatim; this.allowNewlines = isVerbatim; }