public Parser(Tokenizer tokenizer)
{
Debug.Assert(tokenizer != null);
this.tokenizer = tokenizer;
this.ReadNextToken();
}
public void read_next_token_returns_null_if_input_contains_only_whitespace()
{
using (var stringReader = new StringReader(" \u0009 \u000d "))
using (var tokenizer = new Tokenizer(stringReader))
{
Assert.Null(tokenizer.ReadNextToken());
}
}
public void read_next_token_returns_null_if_stream_has_ended()
{
using (var stringReader = new StringReader(string.Empty))
using (var tokenizer = new Tokenizer(stringReader))
{
Assert.Null(tokenizer.ReadNextToken());
}
}
public void read_next_token_returns_symbol_token_for_symbols()
{
using (var stringReader = new StringReader("{} () + - * / ++ (( <<"))
using (var tokenizer = new Tokenizer(stringReader))
{
var token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("{", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("}", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("(", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal(")", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("+", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("-", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("*", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("/", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("++", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("(", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("(", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("<<", token.Value);
Assert.Null(tokenizer.ReadNextToken());
}
}
public void parse_expression_parses_complex_ternary_conditional_expression()
{
using (var stringReader = new StringReader(@"(1 == 1 + 1) ? ""madness"" : (1 == 2 - 1) ? ""sanity"" : ""madness"""))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal("sanity", expression.Evaluate(NodeEvaluationContext.Empty));
}
}
public void parse_expression_parses_simple_ternary_conditional_expression()
{
using (var stringReader = new StringReader(@"true ? ""yes"" : ""no"""))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal("yes", expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader(@"false ? ""yes"" : ""no"""))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal("no", expression.Evaluate(NodeEvaluationContext.Empty));
}
}
public void parse_expression_parses_complex_null_coalescing_expression()
{
using (var stringReader = new StringReader(@"null ?? null ?? null ?? null ?? null ?? ""finally"""))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal("finally", expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader(@"null ?? ""foo"" ?? null"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal("foo", expression.Evaluate(NodeEvaluationContext.Empty));
}
}
public void read_next_token_throws_if_escape_sequence_is_unrecognized()
{
using (var stringReader = new StringReader(@"""\h"""))
using (var tokenizer = new Tokenizer(stringReader))
{
var ex = Assert.Throws<ParseException>(() => tokenizer.ReadNextToken());
Assert.Equal("Unrecognized escape sequence.", ex.Message);
}
}
public void read_next_token_returns_word_token_for_words()
{
using (var stringReader = new StringReader("a b abc abc123"))
using (var tokenizer = new Tokenizer(stringReader))
{
var token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Word, token.Type);
Assert.Equal("a", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Word, token.Type);
Assert.Equal("b", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Word, token.Type);
Assert.Equal("abc", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Word, token.Type);
Assert.Equal("abc123", token.Value);
Assert.Null(tokenizer.ReadNextToken());
}
}
public void read_next_token_returns_string_token_for_string()
{
using (var stringReader = new StringReader(@"""one"" ""two"" ""three \"" four"""))
using (var tokenizer = new Tokenizer(stringReader))
{
var token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.String, token.Type);
Assert.Equal("one", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.String, token.Type);
Assert.Equal("two", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.String, token.Type);
Assert.Equal("three \" four", token.Value);
}
}
public void parse_expression_parses_conditional_and_expression()
{
using (var stringReader = new StringReader("true && true"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(true, expression.Evaluate(NodeEvaluationContext.Empty));
}
}
public void parse_expression_parses_cast_expression()
{
using (var stringReader = new StringReader("(bool) true"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(true, expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader("(string) \"str\""))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal("str", expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader("(byte) 3"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal((byte)3, expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader("(short) 3"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal((short)3, expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader("(int) 3"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(3, expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader("(long) 3"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(3L, expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader("(float) 3"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(3f, expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader("(double) 3"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(3d, expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader("(decimal) 3"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(3m, expression.Evaluate(NodeEvaluationContext.Empty));
}
}
public void read_next_token_handles_complex_expression()
{
using (var stringReader = new StringReader(@"1+2 * (15/ {0}) + ((int) {1} *{2}) <<3 == ""something"" +""something else"""))
using (var tokenizer = new Tokenizer(stringReader))
{
var token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("1", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("+", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("2", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("*", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("(", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("15", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("/", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("{", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("0", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("}", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal(")", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("+", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("(", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("(", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Word, token.Type);
Assert.Equal("int", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal(")", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("{", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("1", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("}", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("*", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("{", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("2", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("}", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal(")", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("<<", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("3", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("==", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.String, token.Type);
Assert.Equal("something", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Symbol, token.Type);
Assert.Equal("+", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.String, token.Type);
Assert.Equal("something else", token.Value);
Assert.Null(tokenizer.ReadNextToken());
}
}
public void read_next_token_returns_number_token_for_numbers()
{
using (var stringReader = new StringReader("1 2 2398 223.1 13d .8 123abc"))
using (var tokenizer = new Tokenizer(stringReader))
{
var token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("1", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("2", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("2398", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("223.1", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("13d", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal(".8", token.Value);
token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.Number, token.Type);
Assert.Equal("123abc", token.Value);
Assert.Null(tokenizer.ReadNextToken());
}
}
public void parse_expression_throws_if_incorrect_grouping_symbol_is_used()
{
using (var stringReader = new StringReader("3 + [5*2]"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var ex = Assert.Throws<ParseException>(() => parser.ParseExpression());
Assert.Equal("Expected an expression.", ex.Message);
}
}
public void parse_expression_parses_grouping_expression()
{
using (var stringReader = new StringReader("(3)"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(3, expression.Evaluate(NodeEvaluationContext.Empty));
}
}
public void parse_expression_parses_variable_expression()
{
using (var stringReader = new StringReader("{0}"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(3, expression.Evaluate(NodeEvaluationContext.Create(3)));
}
}
public void parse_expression_parses_real_number_expression(string input, object expectedEvaluationValue)
{
using (var stringReader = new StringReader(input))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(expectedEvaluationValue, expression.Evaluate(NodeEvaluationContext.Empty));
}
}
public void read_next_token_handles_recognized_escape_sequences()
{
using (var stringReader = new StringReader(@"""\'\""\\\0\a\b\f\n\r\t\v"""))
using (var tokenizer = new Tokenizer(stringReader))
{
var token = tokenizer.ReadNextToken();
Assert.Equal(TokenType.String, token.Type);
Assert.Equal("\'\"\\\0\a\b\f\n\r\t\v", token.Value);
}
}
public void parse_expression_parses_shift_right_expression()
{
using (var stringReader = new StringReader("4 >> 2"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(1, expression.Evaluate(NodeEvaluationContext.Empty));
}
}
public void parse_expression_parses_greater_than_or_equal_expression()
{
using (var stringReader = new StringReader("10 >= 10"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(true, expression.Evaluate(NodeEvaluationContext.Empty));
}
}
public void parse_expression_throws_if_cast_to_unknown_type_is_specified()
{
using (var stringReader = new StringReader("(fubar) 3"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var ex = Assert.Throws<ParseException>(() => parser.ParseExpression());
Assert.Equal("Cannot cast to type 'fubar'.", ex.Message);
}
}
public void parse_expression_parses_logical_not_expression()
{
using (var stringReader = new StringReader("!false"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(true, expression.Evaluate(NodeEvaluationContext.Empty));
}
}
public void parse_expression_throws_if_unknown_symbol_is_specified()
{
using (var stringReader = new StringReader("1 <<> 3"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var ex = Assert.Throws<ParseException>(() => parser.ParseExpression());
Assert.Equal("Unexpected input '<<>'.", ex.Message);
}
}
public void parse_expression_parses_complex_expressions()
{
using (var stringReader = new StringReader("1 + (8/3d) * (50 >> (3 - 1)) * 1e4"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(320001d, expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader("\"a\"+\"b\" + \"c\" +\"def\" + \"\\nghi\" == {0}"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(true, expression.Evaluate(NodeEvaluationContext.Create("abcdef\nghi")));
Assert.Equal(false, expression.Evaluate(NodeEvaluationContext.Create("abcdefghi")));
}
using (var stringReader = new StringReader("23543L & 3448 | (1024 * 56) ^ 8948 ^ (548395 % 34853)"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(45044L, expression.Evaluate(NodeEvaluationContext.Empty));
}
using (var stringReader = new StringReader("~{0} * -{1} / {2}"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(177m, expression.Evaluate(NodeEvaluationContext.Create(235, 3m, 4)));
Assert.Equal(-279, expression.Evaluate(NodeEvaluationContext.Create(-32, 18, 2)));
}
using (var stringReader = new StringReader(@"{0} > 100 ? ({1} ?? {2} ?? ""default"") : ""small number"""))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal("small number", expression.Evaluate(NodeEvaluationContext.Create(99, "first", "second")));
Assert.Equal("first", expression.Evaluate(NodeEvaluationContext.Create(101, "first", "second")));
Assert.Equal("second", expression.Evaluate(NodeEvaluationContext.Create(101, null, "second")));
Assert.Equal("default", expression.Evaluate(NodeEvaluationContext.Create(101, null, null)));
}
using (var stringReader = new StringReader(@"{0} == {1}"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(true, expression.Evaluate(NodeEvaluationContext.Create(ConsoleKey.A, ConsoleKey.A)));
Assert.Equal(true, expression.Evaluate(NodeEvaluationContext.Create(ConsoleKey.A, (int)ConsoleKey.A)));
Assert.Equal(false, expression.Evaluate(NodeEvaluationContext.Create(ConsoleKey.A, ConsoleKey.B)));
Assert.Equal(true, expression.Evaluate(NodeEvaluationContext.Create(this, this)));
Assert.Equal(false, expression.Evaluate(NodeEvaluationContext.Create(this, expression)));
}
using (var stringReader = new StringReader(@"{0} != null && {1} == {2}"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(true, expression.Evaluate(NodeEvaluationContext.Create(this, ConsoleKey.A, ConsoleKey.A)));
Assert.Equal(false, expression.Evaluate(NodeEvaluationContext.Create(null, ConsoleKey.A, ConsoleKey.A)));
Assert.Equal(false, expression.Evaluate(NodeEvaluationContext.Create(this, ConsoleKey.A, ConsoleKey.B)));
}
using (var stringReader = new StringReader(@"{0} != null && {1} == {2}"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(true, expression.Evaluate(NodeEvaluationContext.Create(this, ConsoleKey.A, ConsoleKey.A)));
Assert.Equal(false, expression.Evaluate(NodeEvaluationContext.Create(null, ConsoleKey.A, ConsoleKey.A)));
Assert.Equal(false, expression.Evaluate(NodeEvaluationContext.Create(this, ConsoleKey.A, ConsoleKey.B)));
}
}
// turn the expression into an AST each time it changes (not each time our Convert methods are called)
private void EnsureExpressionNode()
{
if (this.expressionNode != null || string.IsNullOrEmpty(this.expression))
{
return;
}
using (var stringReader = new StringReader(this.expression))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
this.expressionNode = parser.ParseExpression();
}
}
public void parse_expression_parses_less_than_expression()
{
using (var stringReader = new StringReader("10 < 10"))
using (var tokenizer = new Tokenizer(stringReader))
using (var parser = new Parser(tokenizer))
{
var expression = parser.ParseExpression();
Assert.Equal(false, expression.Evaluate(NodeEvaluationContext.Empty));
}
}
