public void verify_empty_expression_analysis() { var lexer = new Lexer(); var tokens = lexer.Analyze(string.Empty).ToArray(); Assert.Equal(1, tokens.Length); Assert.Equal(string.Empty, tokens[0].Value); Assert.Equal(TokenType.EOF, tokens[0].Type); }
public void verify_analysis_apathy_to_culture_settings() { var current = Thread.CurrentThread.CurrentCulture; var lexer = new Lexer(); foreach (var temp in CultureInfo.GetCultures(CultureTypes.AllCultures)) { Thread.CurrentThread.CurrentCulture = temp; // literals parsing should not vary across diverse cultures var tkn = lexer.Analyze("0.1").First(); // e.g. double literal should be always written using dot in our expressions language, no matter the culture Assert.Equal(TokenType.FLOAT, tkn.Type); Assert.Equal(0.1, tkn.Value); } Thread.CurrentThread.CurrentCulture = current; }
private static void AssertNotToken(string expression, TokenType type) { try { var lexer = new Lexer(); var tokens = lexer.Analyze(expression).ToArray(); var recognized = tokens.Length == 2 && tokens[0].Type == type && tokens[1].Type == TokenType.EOF; Assert.False(recognized); } catch (ParseErrorException e) { Assert.Equal("Invalid token.", e.Message); } }
private static void AssertToken(string expression, object value, TokenType type) { var lexer = new Lexer(); var tokens = lexer.Analyze(expression).ToArray(); Assert.Equal(2, tokens.Length); // tested token + EOF token Assert.Equal(value, tokens[0].Value); Assert.Equal(type, tokens[0].Type); Assert.Equal(TokenType.EOF, tokens[1].Type); }
public void verify_complex_expression_analysis() { const string expression = @"GoAbroad == true && ( (NextCountry != 'european country' && Compare(NextCountry, Country.Name) == 0) || (Age > 24 && Age <= 55.5) &&(1.1+2*2>1-2/2+Array[0].Value=='\'\na\n b\nc\n\'') )"; var lexer = new Lexer(); var tokens = lexer.Analyze(expression).ToArray(); Assert.Equal(49, tokens.Length); Assert.Equal("GoAbroad", tokens[0].Value); Assert.Equal(TokenType.FUNC, tokens[0].Type); Assert.Equal("==", tokens[1].Value); Assert.Equal(TokenType.EQ, tokens[1].Type); Assert.Equal(true, tokens[2].Value); Assert.Equal(TokenType.BOOL, tokens[2].Type); Assert.Equal("&&", tokens[3].Value); Assert.Equal(TokenType.AND, tokens[3].Type); Assert.Equal("(", tokens[4].Value); Assert.Equal(TokenType.LEFT_BRACKET, tokens[4].Type); Assert.Equal("(", tokens[5].Value); Assert.Equal(TokenType.LEFT_BRACKET, tokens[5].Type); Assert.Equal("NextCountry", tokens[6].Value); Assert.Equal(TokenType.FUNC, tokens[6].Type); Assert.Equal("!=", tokens[7].Value); Assert.Equal(TokenType.NEQ, tokens[7].Type); Assert.Equal("european country", tokens[8].Value); Assert.Equal(TokenType.STRING, tokens[8].Type); Assert.Equal("&&", tokens[9].Value); Assert.Equal(TokenType.AND, tokens[9].Type); Assert.Equal("Compare", tokens[10].Value); Assert.Equal(TokenType.FUNC, tokens[10].Type); Assert.Equal("(", tokens[11].Value); Assert.Equal(TokenType.LEFT_BRACKET, tokens[11].Type); Assert.Equal("NextCountry", tokens[12].Value); Assert.Equal(TokenType.FUNC, tokens[12].Type); Assert.Equal(",", tokens[13].Value); Assert.Equal(TokenType.COMMA, tokens[13].Type); Assert.Equal("Country.Name", tokens[14].Value); Assert.Equal(TokenType.FUNC, tokens[14].Type); Assert.Equal(")", tokens[15].Value); Assert.Equal(TokenType.RIGHT_BRACKET, tokens[15].Type); Assert.Equal("==", tokens[16].Value); Assert.Equal(TokenType.EQ, tokens[16].Type); Assert.Equal(0, tokens[17].Value); Assert.Equal(TokenType.INT, tokens[17].Type); Assert.Equal(")", tokens[18].Value); Assert.Equal(TokenType.RIGHT_BRACKET, tokens[18].Type); Assert.Equal("||", tokens[19].Value); Assert.Equal(TokenType.OR, tokens[19].Type); Assert.Equal("(", tokens[20].Value); Assert.Equal(TokenType.LEFT_BRACKET, tokens[20].Type); Assert.Equal("Age", tokens[21].Value); Assert.Equal(TokenType.FUNC, tokens[21].Type); Assert.Equal(">", tokens[22].Value); Assert.Equal(TokenType.GT, tokens[22].Type); Assert.Equal(24, tokens[23].Value); Assert.Equal(TokenType.INT, tokens[23].Type); Assert.Equal("&&", tokens[24].Value); Assert.Equal(TokenType.AND, tokens[24].Type); Assert.Equal("Age", tokens[25].Value); Assert.Equal(TokenType.FUNC, tokens[25].Type); Assert.Equal("<=", tokens[26].Value); Assert.Equal(TokenType.LE, tokens[26].Type); Assert.Equal(55.5d, tokens[27].Value); Assert.Equal(TokenType.FLOAT, tokens[27].Type); Assert.Equal(")", tokens[28].Value); Assert.Equal(TokenType.RIGHT_BRACKET, tokens[28].Type); Assert.Equal("&&", tokens[29].Value); Assert.Equal(TokenType.AND, tokens[29].Type); Assert.Equal("(", tokens[30].Value); Assert.Equal(TokenType.LEFT_BRACKET, tokens[30].Type); Assert.Equal(1.1, tokens[31].Value); Assert.Equal(TokenType.FLOAT, tokens[31].Type); Assert.Equal("+", tokens[32].Value); Assert.Equal(TokenType.ADD, tokens[32].Type); Assert.Equal(2, tokens[33].Value); Assert.Equal(TokenType.INT, tokens[33].Type); Assert.Equal("*", tokens[34].Value); Assert.Equal(TokenType.MUL, tokens[34].Type); Assert.Equal(2, tokens[35].Value); Assert.Equal(TokenType.INT, tokens[35].Type); Assert.Equal(">", tokens[36].Value); Assert.Equal(TokenType.GT, tokens[36].Type); Assert.Equal(1, tokens[37].Value); Assert.Equal(TokenType.INT, tokens[37].Type); Assert.Equal("-", tokens[38].Value); Assert.Equal(TokenType.SUB, tokens[38].Type); Assert.Equal(2, tokens[39].Value); Assert.Equal(TokenType.INT, tokens[39].Type); Assert.Equal("/", tokens[40].Value); Assert.Equal(TokenType.DIV, tokens[40].Type); Assert.Equal(2, tokens[41].Value); Assert.Equal(TokenType.INT, tokens[41].Type); Assert.Equal("+", tokens[42].Value); Assert.Equal(TokenType.ADD, tokens[42].Type); Assert.Equal("Array[0].Value", tokens[43].Value); Assert.Equal(TokenType.FUNC, tokens[43].Type); Assert.Equal("==", tokens[44].Value); Assert.Equal(TokenType.EQ, tokens[44].Type); Assert.Equal("'\r\na\r\n b\r\nc\r\n'", tokens[45].Value); // used alternatively to verbatim string (new line \n in expression has been replaced by windows \r\n) Assert.Equal(TokenType.STRING, tokens[45].Type); Assert.Equal(")", tokens[46].Value); Assert.Equal(TokenType.RIGHT_BRACKET, tokens[46].Type); Assert.Equal(")", tokens[47].Value); Assert.Equal(TokenType.RIGHT_BRACKET, tokens[47].Type); Assert.Equal(string.Empty, tokens[48].Value); Assert.Equal(TokenType.EOF, tokens[48].Type); }
public void verify_invalid_tokens_detection() { var lexer = new Lexer(); var e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("true # false")); Assert.Equal("Invalid token.", e.Message); Assert.Equal(1, e.Location.Line); Assert.Equal(6, e.Location.Column); e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("true\r\n&& ^")); Assert.Equal("Invalid token.", e.Message); Assert.Equal(2, e.Location.Line); Assert.Equal(4, e.Location.Column); e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("'John's cat'")); Assert.Equal("Invalid token.", e.Message); Assert.Equal(1, e.Location.Line); Assert.Equal(12, e.Location.Column); }
public void verify_lexer_analyze_invalid_parameter() { var lexer = new Lexer(); var e = Assert.Throws<ArgumentNullException>(() => lexer.Analyze(null)); Assert.Equal("Expression not provided.\r\nParameter name: expression", e.Message); }
public void verify_valid_tokens_extraction() { const string expression = @"! || && == != < <= > >= + - * / % ~ & ^ | << >> () [] . ? : null true false 123 0.3e-2 0b1010 0xFF '\'\na\n b\nc\n\'' メidメ"; var lexer = new Lexer(); var tokens = lexer.Analyze(expression).ToArray(); Assert.Equal(37, tokens.Length); Assert.Equal("!", tokens[0].Value); Assert.Equal(TokenType.L_NOT, tokens[0].Type); Assert.Equal("||", tokens[1].Value); Assert.Equal(TokenType.L_OR, tokens[1].Type); Assert.Equal("&&", tokens[2].Value); Assert.Equal(TokenType.L_AND, tokens[2].Type); Assert.Equal("==", tokens[3].Value); Assert.Equal(TokenType.EQ, tokens[3].Type); Assert.Equal("!=", tokens[4].Value); Assert.Equal(TokenType.NEQ, tokens[4].Type); Assert.Equal("<", tokens[5].Value); Assert.Equal(TokenType.LT, tokens[5].Type); Assert.Equal("<=", tokens[6].Value); Assert.Equal(TokenType.LE, tokens[6].Type); Assert.Equal(">", tokens[7].Value); Assert.Equal(TokenType.GT, tokens[7].Type); Assert.Equal(">=", tokens[8].Value); Assert.Equal(TokenType.GE, tokens[8].Type); Assert.Equal("+", tokens[9].Value); Assert.Equal(TokenType.ADD, tokens[9].Type); Assert.Equal("-", tokens[10].Value); Assert.Equal(TokenType.SUB, tokens[10].Type); Assert.Equal("*", tokens[11].Value); Assert.Equal(TokenType.MUL, tokens[11].Type); Assert.Equal("/", tokens[12].Value); Assert.Equal(TokenType.DIV, tokens[12].Type); Assert.Equal("%", tokens[13].Value); Assert.Equal(TokenType.MOD, tokens[13].Type); Assert.Equal("~", tokens[14].Value); Assert.Equal(TokenType.B_NOT, tokens[14].Type); Assert.Equal("&", tokens[15].Value); Assert.Equal(TokenType.B_AND, tokens[15].Type); Assert.Equal("^", tokens[16].Value); Assert.Equal(TokenType.XOR, tokens[16].Type); Assert.Equal("|", tokens[17].Value); Assert.Equal(TokenType.B_OR, tokens[17].Type); Assert.Equal("<<", tokens[18].Value); Assert.Equal(TokenType.L_SHIFT, tokens[18].Type); Assert.Equal(">>", tokens[19].Value); Assert.Equal(TokenType.R_SHIFT, tokens[19].Type); Assert.Equal("(", tokens[20].Value); Assert.Equal(TokenType.L_PAR, tokens[20].Type); Assert.Equal(")", tokens[21].Value); Assert.Equal(TokenType.R_PAR, tokens[21].Type); Assert.Equal("[", tokens[22].Value); Assert.Equal(TokenType.L_BRACKET, tokens[22].Type); Assert.Equal("]", tokens[23].Value); Assert.Equal(TokenType.R_BRACKET, tokens[23].Type); Assert.Equal(".", tokens[24].Value); Assert.Equal(TokenType.PERIOD, tokens[24].Type); Assert.Equal("?", tokens[25].Value); Assert.Equal(TokenType.QMARK, tokens[25].Type); Assert.Equal(":", tokens[26].Value); Assert.Equal(TokenType.COLON, tokens[26].Type); Assert.Equal(null, tokens[27].Value); Assert.Equal(TokenType.NULL, tokens[27].Type); Assert.Equal(true, tokens[28].Value); Assert.Equal(TokenType.BOOL, tokens[28].Type); Assert.Equal(false, tokens[29].Value); Assert.Equal(TokenType.BOOL, tokens[29].Type); Assert.Equal(123, tokens[30].Value); Assert.Equal(TokenType.INT, tokens[30].Type); Assert.Equal(0.3e-2, tokens[31].Value); Assert.Equal(TokenType.FLOAT, tokens[31].Type); Assert.Equal(10, tokens[32].Value); Assert.Equal(TokenType.BIN, tokens[32].Type); Assert.Equal(255, tokens[33].Value); Assert.Equal(TokenType.HEX, tokens[33].Type); Assert.Equal("'\r\na\r\n b\r\nc\r\n'", tokens[34].Value); // used alternatively to verbatim string (new line \n in expression string literal has been replaced by windows \r\n) Assert.Equal(TokenType.STRING, tokens[34].Type); Assert.Equal("メidメ", tokens[35].Value); Assert.Equal(TokenType.ID, tokens[35].Type); Assert.Equal(string.Empty, tokens[36].Value); Assert.Equal(TokenType.EOF, tokens[36].Type); }
public void verify_invalid_tokens_detection() { var lexer = new Lexer(); var e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("true # false")); Assert.Equal("Invalid token.", e.Error); Assert.Equal(new Location(1, 6), e.Location, new LocationComparer()); e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("true\r\n&& @")); Assert.Equal("Invalid token.", e.Error); Assert.Equal(new Location(2, 4), e.Location, new LocationComparer()); e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("'John's cat'")); Assert.Equal("Invalid token.", e.Error); Assert.Equal(new Location(1, 12), e.Location, new LocationComparer()); }
private static void AssertNotToken(string expression, TokenType type) { try { var lexer = new Lexer(); var tokens = lexer.Analyze(expression).ToArray(); Assert.IsFalse(tokens.Length == 2 && tokens[0].Type == type); } catch (Exception e) { Assert.IsTrue(e is ParseErrorException); Assert.AreEqual("Invalid token.", e.Message); } }
public void verify_invalid_tokens_detection() { var lexer = new Lexer(); try { lexer.Analyze(null); Assert.Fail(); } catch (Exception e) { Assert.IsTrue(e is ArgumentNullException); Assert.AreEqual("Expression not provided.\r\nParameter name: expression", e.Message); } try { lexer.Analyze("true # false"); Assert.Fail(); } catch (Exception e) { Assert.IsTrue(e is ParseErrorException); Assert.AreEqual("Invalid token.", e.Message); var ctx = ((ParseErrorException) e).Location; Assert.AreEqual(1, ctx.Line); Assert.AreEqual(6, ctx.Column); } try { lexer.Analyze("true\r\n&& ^"); Assert.Fail(); } catch (Exception e) { Assert.IsTrue(e is ParseErrorException); Assert.AreEqual("Invalid token.", e.Message); var ctx = ((ParseErrorException) e).Location; Assert.AreEqual(2, ctx.Line); Assert.AreEqual(4, ctx.Column); } try { lexer.Analyze("'John's cat'"); Assert.Fail(); } catch (Exception e) { Assert.IsTrue(e is ParseErrorException); Assert.AreEqual("Invalid token.", e.Message); var ctx = ((ParseErrorException) e).Location; Assert.AreEqual(1, ctx.Line); Assert.AreEqual(12, ctx.Column); } }
public void verify_complex_expression_analysis() { const string expression = @"GoAbroad == true && ( (NextCountry != 'european country' && Compare(NextCountry, Country.Name) == 0) || (Age > 24 && Age <= 55.5) && (1+2*2>1-2/2) )"; var lexer = new Lexer(); var tokens = lexer.Analyze(expression).ToArray(); Assert.AreEqual(45, tokens.Length); Assert.AreEqual("GoAbroad", tokens[0].Value); Assert.AreEqual(TokenType.FUNC, tokens[0].Type); Assert.AreEqual("==", tokens[1].Value); Assert.AreEqual(TokenType.EQ, tokens[1].Type); Assert.AreEqual(true, tokens[2].Value); Assert.AreEqual(TokenType.BOOL, tokens[2].Type); Assert.AreEqual("&&", tokens[3].Value); Assert.AreEqual(TokenType.AND, tokens[3].Type); Assert.AreEqual("(", tokens[4].Value); Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[4].Type); Assert.AreEqual("(", tokens[5].Value); Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[5].Type); Assert.AreEqual("NextCountry", tokens[6].Value); Assert.AreEqual(TokenType.FUNC, tokens[6].Type); Assert.AreEqual("!=", tokens[7].Value); Assert.AreEqual(TokenType.NEQ, tokens[7].Type); Assert.AreEqual("european country", tokens[8].Value); Assert.AreEqual(TokenType.STRING, tokens[8].Type); Assert.AreEqual("&&", tokens[9].Value); Assert.AreEqual(TokenType.AND, tokens[9].Type); Assert.AreEqual("Compare", tokens[10].Value); Assert.AreEqual(TokenType.FUNC, tokens[10].Type); Assert.AreEqual("(", tokens[11].Value); Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[11].Type); Assert.AreEqual("NextCountry", tokens[12].Value); Assert.AreEqual(TokenType.FUNC, tokens[12].Type); Assert.AreEqual(",", tokens[13].Value); Assert.AreEqual(TokenType.COMMA, tokens[13].Type); Assert.AreEqual("Country.Name", tokens[14].Value); Assert.AreEqual(TokenType.FUNC, tokens[14].Type); Assert.AreEqual(")", tokens[15].Value); Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[15].Type); Assert.AreEqual("==", tokens[16].Value); Assert.AreEqual(TokenType.EQ, tokens[16].Type); Assert.AreEqual(0, tokens[17].Value); Assert.AreEqual(TokenType.INT, tokens[17].Type); Assert.AreEqual(")", tokens[18].Value); Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[18].Type); Assert.AreEqual("||", tokens[19].Value); Assert.AreEqual(TokenType.OR, tokens[19].Type); Assert.AreEqual("(", tokens[20].Value); Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[20].Type); Assert.AreEqual("Age", tokens[21].Value); Assert.AreEqual(TokenType.FUNC, tokens[21].Type); Assert.AreEqual(">", tokens[22].Value); Assert.AreEqual(TokenType.GT, tokens[22].Type); Assert.AreEqual(24, tokens[23].Value); Assert.AreEqual(TokenType.INT, tokens[23].Type); Assert.AreEqual("&&", tokens[24].Value); Assert.AreEqual(TokenType.AND, tokens[24].Type); Assert.AreEqual("Age", tokens[25].Value); Assert.AreEqual(TokenType.FUNC, tokens[25].Type); Assert.AreEqual("<=", tokens[26].Value); Assert.AreEqual(TokenType.LE, tokens[26].Type); Assert.AreEqual(55.5d, tokens[27].Value); Assert.AreEqual(TokenType.FLOAT, tokens[27].Type); Assert.AreEqual(")", tokens[28].Value); Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[28].Type); Assert.AreEqual("&&", tokens[29].Value); Assert.AreEqual(TokenType.AND, tokens[29].Type); Assert.AreEqual("(", tokens[30].Value); Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[30].Type); Assert.AreEqual(1, tokens[31].Value); Assert.AreEqual(TokenType.INT, tokens[31].Type); Assert.AreEqual("+", tokens[32].Value); Assert.AreEqual(TokenType.ADD, tokens[32].Type); Assert.AreEqual(2, tokens[33].Value); Assert.AreEqual(TokenType.INT, tokens[33].Type); Assert.AreEqual("*", tokens[34].Value); Assert.AreEqual(TokenType.MUL, tokens[34].Type); Assert.AreEqual(2, tokens[35].Value); Assert.AreEqual(TokenType.INT, tokens[35].Type); Assert.AreEqual(">", tokens[36].Value); Assert.AreEqual(TokenType.GT, tokens[36].Type); Assert.AreEqual(1, tokens[37].Value); Assert.AreEqual(TokenType.INT, tokens[37].Type); Assert.AreEqual("-", tokens[38].Value); Assert.AreEqual(TokenType.SUB, tokens[38].Type); Assert.AreEqual(2, tokens[39].Value); Assert.AreEqual(TokenType.INT, tokens[39].Type); Assert.AreEqual("/", tokens[40].Value); Assert.AreEqual(TokenType.DIV, tokens[40].Type); Assert.AreEqual(2, tokens[41].Value); Assert.AreEqual(TokenType.INT, tokens[41].Type); Assert.AreEqual(")", tokens[42].Value); Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[42].Type); Assert.AreEqual(")", tokens[43].Value); Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[43].Type); Assert.AreEqual(string.Empty, tokens[44].Value); Assert.AreEqual(TokenType.EOF, tokens[44].Type); }