public TokenInfo(SyntaxKind kind)
{
this.Kind = kind;
this.Text = kind.GetText();
this.ZeroTriviaToken = new LexicalToken(kind, "", this.Text);
this.SingleWhitespaceToken = new LexicalToken(kind, " ", this.Text);
}
public void HonorPrecedences_GivenBinaryExpression(SyntaxKind op1, SyntaxKind op2)
{
var op1Precedence = op1.GetBinaryOperatorPrecedence();
var op2Precedence = op2.GetBinaryOperatorPrecedence();
var op1Text = op1.GetText();
var op2Text = op2.GetText();
var text = $"a {op1Text} b {op2Text} c";
var expressionSyntax = ParseExpression(text);
if (op1Precedence >= op2Precedence)
{
// op2
// / \
// op1 c
// / \
// a b
using (var asserter = new AssertingEnumerator(expressionSyntax))
{
asserter.AssertNode(SyntaxKind.BinaryExpression);
asserter.AssertNode(SyntaxKind.BinaryExpression);
asserter.AssertNode(SyntaxKind.NameExpression);
asserter.AssertToken(SyntaxKind.IdentifierToken, "a");
asserter.AssertToken(op1, op1Text);
asserter.AssertNode(SyntaxKind.NameExpression);
asserter.AssertToken(SyntaxKind.IdentifierToken, "b");
asserter.AssertToken(op2, op2Text);
asserter.AssertNode(SyntaxKind.NameExpression);
asserter.AssertToken(SyntaxKind.IdentifierToken, "c");
}
}
else
{
// op1
// / \
// a op2
// / \
// b c
using (var asserter = new AssertingEnumerator(expressionSyntax))
{
asserter.AssertNode(SyntaxKind.BinaryExpression);
asserter.AssertNode(SyntaxKind.NameExpression);
asserter.AssertToken(SyntaxKind.IdentifierToken, "a");
asserter.AssertToken(op1, op1Text);
asserter.AssertNode(SyntaxKind.BinaryExpression);
asserter.AssertNode(SyntaxKind.NameExpression);
asserter.AssertToken(SyntaxKind.IdentifierToken, "b");
asserter.AssertToken(op2, op2Text);
asserter.AssertNode(SyntaxKind.NameExpression);
asserter.AssertToken(SyntaxKind.IdentifierToken, "c");
}
}
}
public void FlipBinaryOperatorSides_DoesNoTrigger_ForInvalidOperators(SyntaxKind kind)
{
var operatorText = kind.GetText();
var query = $@"
SELECT *
FROM Employees e
WHERE /* prefix */ e.FirstName /* before */ {operatorText}| /* after */ 'Andrew' /* suffix */
";
AssertDoesNotTrigger(query);
}
public void TestPrimaryExpressions(SyntaxKind kind)
{
var text = kind.GetText();
var expr = ParseExpression(text);
Assert.NotNull(expr);
var opKind = SyntaxFacts.GetLiteralExpression(kind);
Assert.AreEqual(opKind, expr.Kind);
Assert.AreEqual(0, expr.GetDiagnostics().Count());
var us = (LiteralExpressionSyntax)expr;
Assert.NotNull(us.Token);
Assert.AreEqual(kind, us.Token.Kind);
}
public void HonorPrecedences_GivenUnaryExpression(SyntaxKind unaaryKind, SyntaxKind binaryKind)
{
var unaryPrecedence = unaaryKind.GetUnaryOperatorPrecedence();
var binaryPrecedence = binaryKind.GetBinaryOperatorPrecedence();
var unaryText = unaaryKind.GetText();
var binaryText = binaryKind.GetText();
var text = $"{unaryText} a {binaryText} b";
var expressionSyntax = ParseExpression(text);
if (unaryPrecedence >= binaryPrecedence)
{
// op2
// / \
// op1 b
// |
// a
using (var asserter = new AssertingEnumerator(expressionSyntax))
{
asserter.AssertNode(SyntaxKind.BinaryExpression);
asserter.AssertNode(SyntaxKind.UnaryExpression);
asserter.AssertToken(unaaryKind, unaryText);
asserter.AssertNode(SyntaxKind.NameExpression);
asserter.AssertToken(SyntaxKind.IdentifierToken, "a");
asserter.AssertToken(binaryKind, binaryText);
asserter.AssertNode(SyntaxKind.NameExpression);
asserter.AssertToken(SyntaxKind.IdentifierToken, "b");
}
}
else
{
// op2
// |
// op1
// / \
// a b
using (var asserter = new AssertingEnumerator(expressionSyntax))
{
asserter.AssertNode(SyntaxKind.UnaryExpression);
asserter.AssertNode(SyntaxKind.BinaryExpression);
asserter.AssertToken(unaaryKind, unaryText);
asserter.AssertNode(SyntaxKind.NameExpression);
asserter.AssertToken(SyntaxKind.IdentifierToken, "a");
asserter.AssertToken(binaryKind, binaryText);
asserter.AssertNode(SyntaxKind.NameExpression);
asserter.AssertToken(SyntaxKind.IdentifierToken, "b");
}
}
}
public void Rountrtrip_GivenSyntaxKind(SyntaxKind kind)
{
var text = kind.GetText();
if (text == null)
{
return;
}
var tokens = SyntaxTree.ParseTokens(text).ToArray();
Assert.That(tokens, Has.One.Items);
Assert.That(tokens[0].Kind, Is.EqualTo(kind));
Assert.That(tokens[0].Text, Is.EqualTo(text));
}
public void TestPrimaryExpressions(SyntaxKind kind)
{
var text = kind.GetText();
var expr = ParseExpression(text);
Assert.NotNull(expr);
var opKind = SyntaxFacts.GetLiteralExpression(kind);
Assert.Equal(opKind, expr.Kind);
Assert.Equal(0, expr.GetDiagnostics().Count());
var us = (LiteralExpressionSyntax)expr;
Assert.NotNull(us.Token);
Assert.Equal(kind, us.Token.Kind);
}
private static void LexPunctuationOrOperator(SyntaxKind kind)
{
var text = kind.GetText();
var token = SyntaxFacts.ParseToken(text);
var expectedContextualKind = kind.IsKeyword()
? SyntaxKind.IdentifierToken
: SyntaxKind.BadToken;
Assert.Equal(kind, token.Kind);
Assert.Equal(expectedContextualKind, token.ContextualKind);
Assert.Equal(text, token.Text);
Assert.False(token.IsMissing);
Assert.True(token.IsTerminated());
Assert.Empty(token.Diagnostics);
}
public void TestBinaryOperators(SyntaxKind kind)
{
var text = "(a) " + kind.GetText() + " b";
var expr = ParseExpression(text);
Assert.NotNull(expr);
var opKind = SyntaxFacts.GetBinaryExpression(kind);
Assert.AreEqual(opKind, expr.Kind);
Assert.AreEqual(text, expr.ToString());
Assert.AreEqual(0, expr.GetDiagnostics().Count());
var b = (BinaryExpressionSyntax)expr;
Assert.NotNull(b.OperatorToken);
Assert.AreEqual(kind, b.OperatorToken.Kind);
Assert.NotNull(b.Left);
Assert.NotNull(b.Right);
Assert.AreEqual("(a)", b.Left.ToString());
Assert.AreEqual("b", b.Right.ToString());
}
public void TestPrefixUnaryOperators(SyntaxKind kind)
{
var text = kind.GetText() + "a";
var expr = ParseExpression(text);
Assert.NotNull(expr);
var opKind = SyntaxFacts.GetPrefixUnaryExpression(kind);
Assert.Equal(opKind, expr.Kind);
Assert.Equal(text, expr.ToString());
Assert.Equal(0, expr.GetDiagnostics().Count());
var us = (PrefixUnaryExpressionSyntax)expr;
Assert.NotNull(us.OperatorToken);
Assert.Equal(kind, us.OperatorToken.Kind);
Assert.NotNull(us.Operand);
Assert.Equal(SyntaxKind.IdentifierName, us.Operand.Kind);
Assert.Equal("a", us.Operand.ToString());
}
private static void LexKeyword(SyntaxKind kind, bool isReserved)
{
var text = kind.GetText();
var token = SyntaxFacts.ParseToken(text);
var expectedKind = isReserved
? kind
: SyntaxKind.IdentifierToken;
var expectedContextualKind = isReserved
? SyntaxKind.IdentifierToken
: kind;
Assert.Equal(expectedKind, token.Kind);
Assert.Equal(expectedContextualKind, token.ContextualKind);
Assert.Equal(text, token.Text);
Assert.False(token.IsMissing);
Assert.True(token.IsTerminated());
Assert.Empty(token.Diagnostics);
}
public void FlipBinaryOperatorSides_SwapsOperator_ToSelf(SyntaxKind kind)
{
var operatorText = kind.GetText();
var query = $@"
SELECT *
FROM Employees e
WHERE /* prefix */ e.FirstName /* before */ {operatorText}| /* after */ 'Andrew' /* suffix */
";
var fixedQuery = $@"
SELECT *
FROM Employees e
WHERE /* prefix */ 'Andrew' /* before */ {operatorText} /* after */ e.FirstName /* suffix */
";
var description = $"Flip '{operatorText}' operands";
AssertFixes(query, fixedQuery, description);
}
public void TestAssignmentOperators(SyntaxKind kind)
{
var text = "(a) " + kind.GetText() + " b";
var expr = ParseExpression(text);
Assert.NotNull(expr);
var opKind = SyntaxFacts.GetAssignmentExpression(kind);
Assert.Equal(opKind, expr.Kind);
Assert.Equal(text, expr.ToString());
Assert.Equal(0, expr.GetDiagnostics().Count());
var b = (AssignmentExpressionSyntax)expr;
Assert.NotNull(b.OperatorToken);
Assert.Equal(kind, b.OperatorToken.Kind);
Assert.NotNull(b.Left);
Assert.NotNull(b.Right);
Assert.Equal("(a)", b.Left.ToString());
Assert.Equal("b", b.Right.ToString());
}
public void TestPostfixUnaryOperator(SyntaxKind kind)
{
var text = "a" + kind.GetText() + ";";
var statement = ParseStatement(text);
Assert.NotNull(statement);
Assert.AreEqual(SyntaxKind.ExpressionStatement, statement.Kind);
Assert.AreEqual(text, statement.ToString());
Assert.AreEqual(0, statement.GetDiagnostics().Count());
var es = (ExpressionStatementSyntax)statement;
Assert.NotNull(es.Expression);
Assert.NotNull(es.SemicolonToken);
Assert.False(es.SemicolonToken.IsMissing);
var opKind = SyntaxFacts.GetPostfixUnaryExpression(kind);
Assert.AreEqual(opKind, es.Expression.Kind);
var us = (PostfixUnaryExpressionSyntax)es.Expression;
Assert.AreEqual("a", us.Operand.ToString());
Assert.AreEqual(kind, us.OperatorToken.Kind);
}
public void FlipBinaryOperatorSides_SwapsOperator_ToOther(SyntaxKind kind)
{
var operatorText = kind.GetText();
var otherOperatorText = SyntaxFacts.SwapBinaryExpressionTokenKind(kind).GetText();
var query = $@"
SELECT *
FROM Employees e
WHERE /* prefix */ e.FirstName /* before */ {operatorText}| /* after */ 'Andrew' /* suffix */
";
var fixedQuery = $@"
SELECT *
FROM Employees e
WHERE /* prefix */ 'Andrew' /* before */ {otherOperatorText} /* after */ e.FirstName /* suffix */
";
var description = $"Flip '{operatorText}' operator to '{otherOperatorText}'";
AssertFixes(query, fixedQuery, description);
}
public void TestPostfixUnaryOperator(SyntaxKind kind)
{
var text = "a" + kind.GetText() + ";";
var statement = ParseStatement(text);
Assert.NotNull(statement);
Assert.Equal(SyntaxKind.ExpressionStatement, statement.Kind);
Assert.Equal(text, statement.ToString());
Assert.Equal(0, statement.GetDiagnostics().Count());
var es = (ExpressionStatementSyntax)statement;
Assert.NotNull(es.Expression);
Assert.NotNull(es.SemicolonToken);
Assert.False(es.SemicolonToken.IsMissing);
var opKind = SyntaxFacts.GetPostfixUnaryExpression(kind);
Assert.Equal(opKind, es.Expression.Kind);
var us = (PostfixUnaryExpressionSyntax)es.Expression;
Assert.Equal("a", us.Operand.ToString());
Assert.Equal(kind, us.OperatorToken.Kind);
}
/// <summary>
/// Gets the default tag to assign a token parser, based on the token's kind.
/// </summary>
private static string GetDefaultTag(SyntaxKind kind)
{
return(kind.GetCategory() == SyntaxCategory.Punctuation
? $"'{kind.GetText()}'"
: kind.GetText() ?? kind.ToString().ToLower());
}
public SyntaxToken Lex()
{
_start = _position;
_kind = SyntaxKind.BadToken;
_value = null;
switch (Current)
{
case '\0':
_kind = SyntaxKind.EofToken;
break;
case '+':
_kind = SyntaxKind.PlusToken;
_position++;
break;
case '-':
_kind = SyntaxKind.MinusToken;
_position++;
break;
case '*':
_kind = SyntaxKind.StarToken;
_position++;
break;
case '/':
_kind = SyntaxKind.SlashToken;
_position++;
break;
case '(':
_kind = SyntaxKind.OpenParensToken;
_position++;
break;
case ')':
_kind = SyntaxKind.CloseParensToken;
_position++;
break;
case '{':
_kind = SyntaxKind.OpenBraceToken;
_position++;
break;
case '}':
_kind = SyntaxKind.CloseBraceToken;
_position++;
break;
case '^':
_kind = SyntaxKind.HatToken;
_position++;
break;
case '~':
_kind = SyntaxKind.TildeToken;
_position++;
break;
case '&':
_position++;
if (Current != '&')
{
_kind = SyntaxKind.AmpersandToken;
}
else
{
_kind = SyntaxKind.AmpersandAmpersandToken;
_position++;
}
break;
case '|':
_position++;
if (Current != '|')
{
_kind = SyntaxKind.PipeToken;
}
else
{
_kind = SyntaxKind.PipePipeToken;
_position++;
}
break;
case '=':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.EqualsToken;
}
else
{
_kind = SyntaxKind.EqualsEqualsToken;
_position++;
}
break;
case '!':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.BangToken;
}
else
{
_kind = SyntaxKind.BangEqualsToken;
_position++;
}
break;
case '<':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.LessThanToken;
}
else
{
_kind = SyntaxKind.LessThanOrEqualToken;
_position++;
}
break;
case '>':
_position++;
if (Current != '=')
{
_kind = SyntaxKind.GreaterThanToken;
}
else
{
_kind = SyntaxKind.GreaterThanOrEqualToken;
_position++;
}
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
ReadNumberToken();
break;
case '"':
ReadString();
break;
case ' ':
case '\t':
case '\n':
case '\r':
ReadWhiteSpace();
break;
default:
if (char.IsLetter(Current))
{
ReadIdentifierOrKeyword();
}
else if (char.IsWhiteSpace(Current))
{
ReadWhiteSpace();
}
else
{
Diagnostics.ReportBadCharacter(_position, Current);
_position++;
}
break;
}
var length = _position - _start;
var text = _kind.GetText() ?? _text.ToString(_start, length);
return(new SyntaxToken(_kind, _start, text, _value));
}
public static void WritePunctuation(this TextWriter writer, SyntaxKind tokenType)
{
writer.WritePunctuation(tokenType.GetText());
}
public void Lexer_Lex_Identifier_Quoted_WithReservedKeyword(SyntaxKind kind)
{
var keyword = kind.GetText();
LexQuotedIdentifier(keyword);
}
public void TestPrefixUnaryOperators(SyntaxKind kind)
{
var text = kind.GetText() + "a";
var expr = ParseExpression(text);
Assert.NotNull(expr);
var opKind = SyntaxFacts.GetPrefixUnaryExpression(kind);
Assert.AreEqual(opKind, expr.Kind);
Assert.AreEqual(text, expr.ToString());
Assert.AreEqual(0, expr.GetDiagnostics().Count());
var us = (PrefixUnaryExpressionSyntax)expr;
Assert.NotNull(us.OperatorToken);
Assert.AreEqual(kind, us.OperatorToken.Kind);
Assert.NotNull(us.Operand);
Assert.AreEqual(SyntaxKind.IdentifierName, us.Operand.Kind);
Assert.AreEqual("a", us.Operand.ToString());
}
public static void WriteKeyword(this TextWriter writer, SyntaxKind tokenType)
{
writer.WriteKeyword(tokenType.GetText());
}
