private Token NextIdentifier()
{
int start = Index;
string id = GetIdentifierName();
Token result;
if (id.CIEquals("true"))
{
result = new TrueLiteralToken();
}
else if (id.CIEquals("null"))
{
result = new NullLiteralToken();
}
else if (id.CIEquals("false"))
{
result = new FalseLiteralToken();
}
else if (id.CIEquals("empty"))
{
result = new EmptyLiteralToken();
}
else if (id.CIEquals("nothing"))
{
result = new NothingLiteralToken();
}
else if (LexerUtils.IsKeyword(id))
{
result = new KeywordToken {
Keyword = LexerUtils.GetKeyword(id),
Name = id,
}
}
;
else if (LexerUtils.IsKeywordAsIdentifier(id))
{
result = new KeywordOrIdentifierToken
{
Keyword = LexerUtils.GetKeywordAsIdentifier(id),
Name = id,
}
}
;
else
{
result = new IdentifierToken {
Name = id
}
};
result.Start = start;
result.End = Index;
result.LineNumber = CurrentLine;
result.LineStart = CurrentLineStart;
return(result);
}
public static (Token Token, Lexer NextLexer) ReadToken(Lexer lexer)
{
var reader = lexer.Reader;
var peek = reader.Peek();
switch (peek.Value)
{
case '$':
return(LexerEngine.ReadAliasIdentifierToken(reader));
case ']':
return(LexerEngine.ReadAttributeCloseToken(reader));
case '[':
return(LexerEngine.ReadAttributeOpenToken(reader));
case '}':
return(LexerEngine.ReadBlockCloseToken(reader));
case '{':
return(LexerEngine.ReadBlockOpenToken(reader));
case ':':
return(LexerEngine.ReadColonToken(reader));
case ',':
return(LexerEngine.ReadCommaToken(reader));
case '/':
return(LexerEngine.ReadCommentToken(reader));
case '=':
return(LexerEngine.ReadEqualsOperatorToken(reader));
case ')':
return(LexerEngine.ReadParenthesisCloseToken(reader));
case '(':
return(LexerEngine.ReadParenthesisOpenToken(reader));
case '#':
return(LexerEngine.ReadPragmaToken(reader));
case ';':
return(LexerEngine.ReadStatementEndToken(reader));
case '"':
return(LexerEngine.ReadStringLiteralToken(reader));
case '+':
case '-':
return(LexerEngine.ReadNumericLiteralToken(reader));
case '.':
// if the next character is a decimalDigit then we're reading a RealLiteralToken with
// no leading digits before the decimal point (e.g. ".45"), otherwise we're reading
// a DotOperatorToken (e.g. the "." in "MyPropertyValue = MyEnum.Value;")
var readAhead = reader.Read().NextReader;
if (!readAhead.Eof() && StringValidator.IsDecimalDigit(readAhead.Peek().Value))
{
return(LexerEngine.ReadNumericLiteralToken(reader));
}
else
{
return(LexerEngine.ReadDotOperatorToken(reader));
}
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
case 'G':
case 'H':
case 'I':
case 'J':
case 'K':
case 'L':
case 'M':
case 'N':
case 'O':
case 'P':
case 'Q':
case 'R':
case 'S':
case 'T':
case 'U':
case 'V':
case 'W':
case 'X':
case 'Y':
case 'Z':
case '_':
// firstIdentifierChar
var(identifierToken, nextLexer) = LexerEngine.ReadIdentifierToken(reader);
var normalized = identifierToken.Name.ToLowerInvariant();
switch (normalized)
{
case "false":
var falseToken = new BooleanLiteralToken(identifierToken.Extent, false);
return(falseToken, nextLexer);
case "true":
var trueToken = new BooleanLiteralToken(identifierToken.Extent, true);
return(trueToken, nextLexer);
case "null":
var nullLiteralToken = new NullLiteralToken(identifierToken.Extent);
return(nullLiteralToken, nextLexer);
default:
return(identifierToken, nextLexer);
}
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
// decimalDigit
return(LexerEngine.ReadNumericLiteralToken(reader));
case '\u0020': // space
case '\u0009': // horizontal tab
case '\u000D': // carriage return
case '\u000A': // line feed
// WS
return(LexerEngine.ReadWhitespaceToken(reader));
default:
throw new UnexpectedCharacterException(peek);
}
}
private NullValueAst(NullLiteralToken token)
{
this.Token = token ?? throw new ArgumentNullException(nameof(token));
}
private static void AssertAreEqualInternal(Token expectedToken, Token actualToken, int index = -1)
{
if ((expectedToken == null) && (actualToken == null))
{
return;
}
if (expectedToken == null)
{
Assert.Fail(LexerHelper.GetAssertErrorMessage("expected is null, but actual is not null", index));
}
if (actualToken == null)
{
Assert.Fail(LexerHelper.GetAssertErrorMessage("expected is not null, but actual is null", index));
}
Assert.AreEqual(expectedToken.GetType(), actualToken.GetType(),
LexerHelper.GetAssertErrorMessage($"actual type does not match expected value", index));
Assert.AreEqual(expectedToken.Extent.StartPosition.Position, actualToken.Extent.StartPosition.Position,
LexerHelper.GetAssertErrorMessage($"actual Start Position does not match expected value", index));
Assert.AreEqual(expectedToken.Extent.StartPosition.LineNumber, actualToken.Extent.StartPosition.LineNumber,
LexerHelper.GetAssertErrorMessage($"actual Start Line does not match expected value", index));
Assert.AreEqual(expectedToken.Extent.StartPosition.ColumnNumber, actualToken.Extent.StartPosition.ColumnNumber,
LexerHelper.GetAssertErrorMessage($"actual Start Column does not match expected value", index));
Assert.AreEqual(expectedToken.Extent.EndPosition.Position, actualToken.Extent.EndPosition.Position,
LexerHelper.GetAssertErrorMessage($"actual End Position does not match expected value", index));
Assert.AreEqual(expectedToken.Extent.EndPosition.LineNumber, actualToken.Extent.EndPosition.LineNumber,
LexerHelper.GetAssertErrorMessage($"actual End Line does not match expected value", index));
Assert.AreEqual(expectedToken.Extent.EndPosition.ColumnNumber, actualToken.Extent.EndPosition.ColumnNumber,
LexerHelper.GetAssertErrorMessage($"actual End Column does not match expected value", index));
Assert.AreEqual(expectedToken.Extent.Text, actualToken.Extent.Text,
LexerHelper.GetAssertErrorMessage($"actual Text does not match expected value", index));
switch (expectedToken)
{
case AliasIdentifierToken token:
Assert.IsTrue(
AliasIdentifierToken.AreEqual((AliasIdentifierToken)expectedToken, (AliasIdentifierToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case AttributeCloseToken token:
Assert.IsTrue(
AttributeCloseToken.AreEqual((AttributeCloseToken)expectedToken, (AttributeCloseToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case AttributeOpenToken token:
Assert.IsTrue(
AttributeOpenToken.AreEqual((AttributeOpenToken)expectedToken, (AttributeOpenToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case BlockCloseToken token:
Assert.IsTrue(
BlockCloseToken.AreEqual((BlockCloseToken)expectedToken, (BlockCloseToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case BlockOpenToken token:
Assert.IsTrue(
BlockOpenToken.AreEqual((BlockOpenToken)expectedToken, (BlockOpenToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case BooleanLiteralToken token:
Assert.IsTrue(
BooleanLiteralToken.AreEqual((BooleanLiteralToken)expectedToken, (BooleanLiteralToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case ColonToken token:
Assert.IsTrue(
ColonToken.AreEqual((ColonToken)expectedToken, (ColonToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case CommaToken token:
Assert.IsTrue(
CommaToken.AreEqual((CommaToken)expectedToken, (CommaToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case CommentToken token:
Assert.IsTrue(
CommentToken.AreEqual((CommentToken)expectedToken, (CommentToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case DotOperatorToken token:
Assert.IsTrue(
DotOperatorToken.AreEqual((DotOperatorToken)expectedToken, (DotOperatorToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case EqualsOperatorToken token:
Assert.IsTrue(
EqualsOperatorToken.AreEqual((EqualsOperatorToken)expectedToken, (EqualsOperatorToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case IdentifierToken token:
Assert.IsTrue(
IdentifierToken.AreEqual((IdentifierToken)expectedToken, (IdentifierToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case IntegerLiteralToken token:
Assert.IsTrue(
IntegerLiteralToken.AreEqual((IntegerLiteralToken)expectedToken, (IntegerLiteralToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case NullLiteralToken token:
Assert.IsTrue(
NullLiteralToken.AreEqual((NullLiteralToken)expectedToken, (NullLiteralToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case ParenthesisCloseToken token:
Assert.IsTrue(
ParenthesisCloseToken.AreEqual((ParenthesisCloseToken)expectedToken, (ParenthesisCloseToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case ParenthesisOpenToken token:
Assert.IsTrue(
ParenthesisOpenToken.AreEqual((ParenthesisOpenToken)expectedToken, (ParenthesisOpenToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case PragmaToken token:
Assert.IsTrue(
PragmaToken.AreEqual((PragmaToken)expectedToken, (PragmaToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case RealLiteralToken token:
Assert.IsTrue(
RealLiteralToken.AreEqual((RealLiteralToken)expectedToken, (RealLiteralToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case StatementEndToken token:
Assert.IsTrue(
StatementEndToken.AreEqual((StatementEndToken)expectedToken, (StatementEndToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case StringLiteralToken token:
Assert.IsTrue(
StringLiteralToken.AreEqual((StringLiteralToken)expectedToken, (StringLiteralToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
case WhitespaceToken token:
Assert.IsTrue(
WhitespaceToken.AreEqual((WhitespaceToken)expectedToken, (WhitespaceToken)actualToken),
LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
);
break;
default:
throw new NotImplementedException($"Cannot compare type '{expectedToken.GetType().Name}'");
}
}
