private SyntaxToken EatToken(TokenKind tokenKind) { SyntaxToken syntax; if (_token.Kind == tokenKind) { syntax = Open <SyntaxToken>(); } else { // Create an invalid token in case we don't match it var invalid = Open <InvalidSyntaxToken>(); invalid.InvalidKind = _token.Kind; syntax = invalid; var tokenText = tokenKind.ToText(); var expectingTokenText = tokenText != null ? $"while expecting `{tokenText}` (token: `{tokenKind.ToString().ToLowerInvariant()}`)" : $"while expecting token `{tokenKind.ToString().ToLowerInvariant()}`"; if (_token.Kind == TokenKind.Invalid) { LogError($"Unexpected token found `{ToPrintable(_token)}` {expectingTokenText}"); } else { LogError($"Unexpected token found `{ToPrintable(_token)}` (token: `{_token.Kind.ToString().ToLowerInvariant()}`) {expectingTokenText}"); } } syntax.TokenKind = tokenKind; syntax.Text = _token.Kind.ToText() ?? _token.GetText(_lexer.Source); if (tokenKind == TokenKind.NewLine) { // Once we have found a new line, we let all the other NewLines as trivias _hideNewLine = true; } NextToken(); return(Close(syntax)); }
/// <summary> /// Creates a token from the specified token kind. /// </summary> /// <param name="kind">The token kind</param> /// <returns>The token</returns> public static SyntaxToken Token(TokenKind kind) { if (kind == TokenKind.NewLine || !kind.IsToken()) { throw new ArgumentOutOfRangeException($"The token kind `{kind}` is not supported for a plain token without a predefined value"); } var text = kind.ToText(); Debug.Assert(text != null); return(new SyntaxToken(kind, text)); }