private void TestReservedWord(string expression, object value, TokenKind expectedTokenKind) {
var lexer = new Tokenizer(expression);
var token1 = lexer.NextToken();
Assert.That(token1.Kind, Is.EqualTo(expectedTokenKind));
Assert.That(token1.Value, Is.EqualTo(value));
var token2 = lexer.NextToken();
Assert.That(token2.Kind, Is.EqualTo(TokenKind.Eof));
}
private void TestStringLiteral(string value, string expected, TokenKind expectedTokenKind) {
var lexer = new Tokenizer(value);
var token1 = lexer.NextToken();
Assert.That(token1.Kind, Is.EqualTo(expectedTokenKind));
Assert.That(token1.Value, Is.EqualTo(expected));
var token2 = lexer.NextToken();
Assert.That(token2.Kind, Is.EqualTo(TokenKind.Eof));
}
private void CheckTokenSequence(string expression, params TokenKind[] tokenKinds) {
var lexer = new Tokenizer(expression);
foreach (var kind in tokenKinds) {
var token = lexer.NextToken();
Assert.That(token.Kind, Is.EqualTo(kind));
}
var token2 = lexer.NextToken();
Assert.That(token2.Kind, Is.EqualTo(TokenKind.Eof));
}
public Lexer(Tokenizer tokenizer) {
_tokenizer = tokenizer;
}