private void LexToken() { textWindow.Reset(); // important to force enum evaluation here via .ToImmutableArray()! var leadingTrivia = ScanLeadingTrivia().ToImmutableArray(); textWindow.Reset(); var tokenType = ScanToken(); var tokenText = textWindow.GetText(); var tokenSpan = textWindow.GetSpan(); if (tokenType == TokenType.Unrecognized) { if (tokenText == "\"") { AddDiagnostic(b => b.DoubleQuoteToken(tokenText)); } else { AddDiagnostic(b => b.UnrecognizedToken(tokenText)); } } textWindow.Reset(); // important to force enum evaluation here via .ToImmutableArray()! var trailingTrivia = ScanTrailingTrivia().ToImmutableArray(); var token = new Token(tokenType, tokenSpan, tokenText, leadingTrivia, trailingTrivia); this.tokens.Add(token); }
private SyntaxTrivia ScanDisableNextLineDiagnosticsDirective() { textWindow.Reset(); textWindow.Advance(LanguageConstants.DisableNextLineDiagnosticsKeyword.Length + 1); // Length of disable next statement plus # var span = textWindow.GetSpan(); int start = span.Position; int length = span.Length; StringBuilder sb = new StringBuilder(); sb.Append(textWindow.GetText()); textWindow.Reset(); List <Token> codes = new(); while (!textWindow.IsAtEnd()) { var nextChar = textWindow.Peek(); if (IsIdentifierContinuation(nextChar) || nextChar == '-') { switch (textWindow.Peek(1)) { case ' ': case '\t': case '\n': case '\r': case char.MaxValue: textWindow.Advance(); if (GetToken() is { } token) { codes.Add(token); length += token.Span.Length; sb.Append(token.Text); continue; } break; default: textWindow.Advance(); break; } } else if (nextChar == ' ' || nextChar == '\t') { textWindow.Advance(); sb.Append(nextChar); length++; textWindow.Reset(); } else { break; } } if (codes.Count == 0) { AddDiagnostic(b => b.MissingDiagnosticCodes()); } return(GetDisableNextLineDiagnosticsSyntaxTrivia(codes, start, length, sb.ToString())); }
private SyntaxTrivia ScanDisableNextLineDiagnosticsDirective() { textWindow.Reset(); textWindow.Advance(LanguageConstants.DisableNextLineDiagnosticsKeyword.Length + 1); // Length of disable next statement plus # var span = textWindow.GetSpan(); int start = span.Position; int end = span.GetEndPosition(); StringBuilder sb = new StringBuilder(); sb.Append(textWindow.GetText()); textWindow.Reset(); List <Token> codes = new(); while (!textWindow.IsAtEnd()) { var nextChar = textWindow.Peek(); if (IsNewLine(nextChar)) { break; } else if (IsIdentifierContinuation(nextChar) || nextChar == '-') { switch (textWindow.Peek(1)) { case ' ': case '\t': case '\n': case '\r': case char.MaxValue: textWindow.Advance(); if (GetToken() is { } token) { codes.Add(token); end += token.Span.Length; sb.Append(token.Text); continue; } break; default: textWindow.Advance(); break; } } else if (nextChar == ' ' || nextChar == '\t') { textWindow.Advance(); sb.Append(nextChar); end++; textWindow.Reset(); } else { // Handle scenario where nextChar is not one of the following: identifier, '-', space, tab // Eg: '|' in #disable-next-line BCP037| if (GetToken() is { } token) { codes.Add(token); end += token.Span.Length; sb.Append(token.Text); } break; } } if (codes.Count == 0) { AddDiagnostic(b => b.MissingDiagnosticCodes()); } return(GetDisableNextLineDiagnosticsSyntaxTrivia(codes, start, end, sb.ToString())); }