private SyntaxToken DebuggerLex(string text)
{
using (var lexer = new InternalSyntax.Lexer(SourceText.From(text), _options))
{
return new SyntaxToken(lexer.Lex(InternalSyntax.LexerMode.DebuggerSyntax));
}
}
internal override IEnumerable <InternalSyntax.SyntaxToken> GetTokens(string text)
{
Assert.DoesNotContain("'", text, StringComparison.Ordinal);
using (
var lexer = new InternalSyntax.Lexer(
SourceText.From(text + "'"),
TestOptions.RegularWithDocumentationComments
)
)
{
while (true)
{
var token = lexer.Lex(
InternalSyntax.LexerMode.XmlNameQuote
| InternalSyntax.LexerMode.XmlDocCommentStyleSingleLine
| InternalSyntax.LexerMode.XmlDocCommentLocationInterior
);
if (token.Kind == SyntaxKind.SingleQuoteToken)
{
break;
}
yield return(token);
}
}
}
private SyntaxToken LexNewToken(LexerMode mode)
{
if (_lexer.TextWindow.Position != _newPosition)
{
_lexer.Reset(_newPosition, _newDirectives);
}
if (mode >= LexerMode.XmlDocComment)
{
mode |= _newLexerDrivenMode;
}
var token = _lexer.Lex(ref mode);
_newDirectives = _lexer.Directives;
_newLexerDrivenMode = mode & (LexerMode.MaskXmlDocCommentLocation | LexerMode.MaskXmlDocCommentStyle);
return(token);
}
/// <summary>
/// Take the given text and treat it as the contents of a string literal, returning a token for that.
/// </summary>
/// <param name="text">The text for the full string literal, including the quotes and contents</param>
/// <param name="bodyText">The text for the string literal's contents, excluding surrounding quotes</param>
/// <param name="isVerbatim">True if the string contents should be scanned using the rules for verbatim strings</param>
/// <param name="kind">The token kind to be assigned to the resulting token</param>
private SyntaxToken MakeStringToken(string text, string bodyText, bool isVerbatim, SyntaxKind kind)
{
var prefix = isVerbatim ? "@\"" : "\"";
var fakeString = prefix + bodyText + "\"";
using (var tempLexer = new Lexer(Text.SourceText.From(fakeString), this.Options, allowPreprocessorDirectives: false))
{
LexerMode mode = LexerMode.Syntax;
SyntaxToken token = tempLexer.Lex(ref mode);
Debug.Assert(token.Kind == SyntaxKind.StringLiteralToken);
var result = SyntaxFactory.Literal(null, text, kind, token.ValueText, null);
if (token.ContainsDiagnostics)
{
result = result.WithDiagnosticsGreen(MoveDiagnostics(token.GetDiagnostics(), -prefix.Length));
}
return(result);
}
}
internal override IEnumerable<InternalSyntax.SyntaxToken> GetTokens(string text)
{
Assert.DoesNotContain("'", text, StringComparison.Ordinal);
using (var lexer = new InternalSyntax.Lexer(SourceText.From(text + "'"), TestOptions.RegularWithDocumentationComments))
{
while (true)
{
var token = lexer.Lex(InternalSyntax.LexerMode.XmlNameQuote | InternalSyntax.LexerMode.XmlDocCommentStyleSingleLine | InternalSyntax.LexerMode.XmlDocCommentLocationInterior);
if (token.Kind == SyntaxKind.SingleQuoteToken)
{
break;
}
yield return token;
}
}
}
/// <summary>
/// Take the given text and treat it as the contents of a string literal, returning a token for that.
/// </summary>
/// <param name="text">The text for the full string literal, including the quotes and contents</param>
/// <param name="bodyText">The text for the string literal's contents, excluding surrounding quotes</param>
/// <param name="isVerbatim">True if the string contents should be scanned using the rules for verbatim strings</param>
/// <param name="kind">The token kind to be assigned to the resulting token</param>
private SyntaxToken MakeStringToken(string text, string bodyText, bool isVerbatim, SyntaxKind kind)
{
var prefix = isVerbatim ? "@\"" : "\"";
var fakeString = prefix + bodyText + "\"";
using (var tempLexer = new Lexer(Text.SourceText.From(fakeString), this.Options, allowPreprocessorDirectives: false))
{
LexerMode mode = LexerMode.Syntax;
SyntaxToken token = tempLexer.Lex(ref mode);
Debug.Assert(token.Kind == SyntaxKind.StringLiteralToken);
var result = SyntaxFactory.Literal(null, text, kind, token.ValueText, null);
if (token.ContainsDiagnostics)
{
result = result.WithDiagnosticsGreen(MoveDiagnostics(token.GetDiagnostics(), -prefix.Length));
}
return result;
}
}