Analyze() public method

Analyzes a specified logical expression and extracts a sequence of tokens.
expression is null
public Analyze ( string expression ) : IEnumerable
expression string The logical expression.
return IEnumerable
 public void verify_empty_expression_analysis()
 {
     var lexer = new Lexer();
     var tokens = lexer.Analyze(string.Empty).ToArray();
     Assert.Equal(1, tokens.Length);
     Assert.Equal(string.Empty, tokens[0].Value);
     Assert.Equal(TokenType.EOF, tokens[0].Type);
 }
        public void verify_analysis_apathy_to_culture_settings()
        {
            var current = Thread.CurrentThread.CurrentCulture;

            var lexer = new Lexer();
            foreach (var temp in CultureInfo.GetCultures(CultureTypes.AllCultures))
            {
                Thread.CurrentThread.CurrentCulture = temp;

                // literals parsing should not vary across diverse cultures
                var tkn = lexer.Analyze("0.1").First(); // e.g. double literal should be always written using dot in our expressions language, no matter the culture
                Assert.Equal(TokenType.FLOAT, tkn.Type);
                Assert.Equal(0.1, tkn.Value);
            }

            Thread.CurrentThread.CurrentCulture = current;
        }
        private static void AssertNotToken(string expression, TokenType type)
        {
            try
            {
                var lexer = new Lexer();
                var tokens = lexer.Analyze(expression).ToArray();

                var recognized = tokens.Length == 2 && tokens[0].Type == type && tokens[1].Type == TokenType.EOF;
                Assert.False(recognized);
            }
            catch (ParseErrorException e)
            {
                Assert.Equal("Invalid token.", e.Message);
            }
        }
        private static void AssertToken(string expression, object value, TokenType type)
        {
            var lexer = new Lexer();
            var tokens = lexer.Analyze(expression).ToArray();

            Assert.Equal(2, tokens.Length); // tested token + EOF token
            Assert.Equal(value, tokens[0].Value);
            Assert.Equal(type, tokens[0].Type);
            Assert.Equal(TokenType.EOF, tokens[1].Type);
        }
        public void verify_complex_expression_analysis()
        {
            const string expression =
                @"GoAbroad == true
                      && (
                             (NextCountry != 'european country' && Compare(NextCountry, Country.Name) == 0)
                             || (Age > 24 && Age <= 55.5)

                             &&(1.1+2*2>1-2/2+Array[0].Value=='\'\na\n b\nc\n\'')
                         )";

            var lexer = new Lexer();
            var tokens = lexer.Analyze(expression).ToArray();
            Assert.Equal(49, tokens.Length);
            Assert.Equal("GoAbroad", tokens[0].Value);
            Assert.Equal(TokenType.FUNC, tokens[0].Type);
            Assert.Equal("==", tokens[1].Value);
            Assert.Equal(TokenType.EQ, tokens[1].Type);
            Assert.Equal(true, tokens[2].Value);
            Assert.Equal(TokenType.BOOL, tokens[2].Type);
            Assert.Equal("&&", tokens[3].Value);
            Assert.Equal(TokenType.AND, tokens[3].Type);
            Assert.Equal("(", tokens[4].Value);
            Assert.Equal(TokenType.LEFT_BRACKET, tokens[4].Type);
            Assert.Equal("(", tokens[5].Value);
            Assert.Equal(TokenType.LEFT_BRACKET, tokens[5].Type);
            Assert.Equal("NextCountry", tokens[6].Value);
            Assert.Equal(TokenType.FUNC, tokens[6].Type);
            Assert.Equal("!=", tokens[7].Value);
            Assert.Equal(TokenType.NEQ, tokens[7].Type);
            Assert.Equal("european country", tokens[8].Value);
            Assert.Equal(TokenType.STRING, tokens[8].Type);
            Assert.Equal("&&", tokens[9].Value);
            Assert.Equal(TokenType.AND, tokens[9].Type);
            Assert.Equal("Compare", tokens[10].Value);
            Assert.Equal(TokenType.FUNC, tokens[10].Type);
            Assert.Equal("(", tokens[11].Value);
            Assert.Equal(TokenType.LEFT_BRACKET, tokens[11].Type);
            Assert.Equal("NextCountry", tokens[12].Value);
            Assert.Equal(TokenType.FUNC, tokens[12].Type);
            Assert.Equal(",", tokens[13].Value);
            Assert.Equal(TokenType.COMMA, tokens[13].Type);
            Assert.Equal("Country.Name", tokens[14].Value);
            Assert.Equal(TokenType.FUNC, tokens[14].Type);
            Assert.Equal(")", tokens[15].Value);
            Assert.Equal(TokenType.RIGHT_BRACKET, tokens[15].Type);
            Assert.Equal("==", tokens[16].Value);
            Assert.Equal(TokenType.EQ, tokens[16].Type);
            Assert.Equal(0, tokens[17].Value);
            Assert.Equal(TokenType.INT, tokens[17].Type);
            Assert.Equal(")", tokens[18].Value);
            Assert.Equal(TokenType.RIGHT_BRACKET, tokens[18].Type);
            Assert.Equal("||", tokens[19].Value);
            Assert.Equal(TokenType.OR, tokens[19].Type);
            Assert.Equal("(", tokens[20].Value);
            Assert.Equal(TokenType.LEFT_BRACKET, tokens[20].Type);
            Assert.Equal("Age", tokens[21].Value);
            Assert.Equal(TokenType.FUNC, tokens[21].Type);
            Assert.Equal(">", tokens[22].Value);
            Assert.Equal(TokenType.GT, tokens[22].Type);
            Assert.Equal(24, tokens[23].Value);
            Assert.Equal(TokenType.INT, tokens[23].Type);
            Assert.Equal("&&", tokens[24].Value);
            Assert.Equal(TokenType.AND, tokens[24].Type);
            Assert.Equal("Age", tokens[25].Value);
            Assert.Equal(TokenType.FUNC, tokens[25].Type);
            Assert.Equal("<=", tokens[26].Value);
            Assert.Equal(TokenType.LE, tokens[26].Type);
            Assert.Equal(55.5d, tokens[27].Value);
            Assert.Equal(TokenType.FLOAT, tokens[27].Type);
            Assert.Equal(")", tokens[28].Value);
            Assert.Equal(TokenType.RIGHT_BRACKET, tokens[28].Type);
            Assert.Equal("&&", tokens[29].Value);
            Assert.Equal(TokenType.AND, tokens[29].Type);
            Assert.Equal("(", tokens[30].Value);
            Assert.Equal(TokenType.LEFT_BRACKET, tokens[30].Type);
            Assert.Equal(1.1, tokens[31].Value);
            Assert.Equal(TokenType.FLOAT, tokens[31].Type);
            Assert.Equal("+", tokens[32].Value);
            Assert.Equal(TokenType.ADD, tokens[32].Type);
            Assert.Equal(2, tokens[33].Value);
            Assert.Equal(TokenType.INT, tokens[33].Type);
            Assert.Equal("*", tokens[34].Value);
            Assert.Equal(TokenType.MUL, tokens[34].Type);
            Assert.Equal(2, tokens[35].Value);
            Assert.Equal(TokenType.INT, tokens[35].Type);
            Assert.Equal(">", tokens[36].Value);
            Assert.Equal(TokenType.GT, tokens[36].Type);
            Assert.Equal(1, tokens[37].Value);
            Assert.Equal(TokenType.INT, tokens[37].Type);
            Assert.Equal("-", tokens[38].Value);
            Assert.Equal(TokenType.SUB, tokens[38].Type);
            Assert.Equal(2, tokens[39].Value);
            Assert.Equal(TokenType.INT, tokens[39].Type);
            Assert.Equal("/", tokens[40].Value);
            Assert.Equal(TokenType.DIV, tokens[40].Type);
            Assert.Equal(2, tokens[41].Value);
            Assert.Equal(TokenType.INT, tokens[41].Type);
            Assert.Equal("+", tokens[42].Value);
            Assert.Equal(TokenType.ADD, tokens[42].Type);
            Assert.Equal("Array[0].Value", tokens[43].Value);
            Assert.Equal(TokenType.FUNC, tokens[43].Type);
            Assert.Equal("==", tokens[44].Value);
            Assert.Equal(TokenType.EQ, tokens[44].Type);
            Assert.Equal("'\r\na\r\n b\r\nc\r\n'", tokens[45].Value); // used alternatively to verbatim string (new line \n in expression has been replaced by windows \r\n)
            Assert.Equal(TokenType.STRING, tokens[45].Type);
            Assert.Equal(")", tokens[46].Value);
            Assert.Equal(TokenType.RIGHT_BRACKET, tokens[46].Type);
            Assert.Equal(")", tokens[47].Value);
            Assert.Equal(TokenType.RIGHT_BRACKET, tokens[47].Type);
            Assert.Equal(string.Empty, tokens[48].Value);
            Assert.Equal(TokenType.EOF, tokens[48].Type);
        }
        public void verify_invalid_tokens_detection()
        {
            var lexer = new Lexer();

            var e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("true # false"));
            Assert.Equal("Invalid token.", e.Message);
            Assert.Equal(1, e.Location.Line);
            Assert.Equal(6, e.Location.Column);

            e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("true\r\n&& ^"));
            Assert.Equal("Invalid token.", e.Message);
            Assert.Equal(2, e.Location.Line);
            Assert.Equal(4, e.Location.Column);

            e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("'John's cat'"));
            Assert.Equal("Invalid token.", e.Message);
            Assert.Equal(1, e.Location.Line);
            Assert.Equal(12, e.Location.Column);
        }
 public void verify_lexer_analyze_invalid_parameter()
 {
     var lexer = new Lexer();
     var e = Assert.Throws<ArgumentNullException>(() => lexer.Analyze(null));
     Assert.Equal("Expression not provided.\r\nParameter name: expression", e.Message);
 }
        public void verify_valid_tokens_extraction()
        {
            const string expression =
                @"! || && == != < <= > >= + - * / % ~ & ^ | << >> () [] . ? :
                  null true false 123 0.3e-2 0b1010 0xFF '\'\na\n b\nc\n\'' メidメ";

            var lexer = new Lexer();
            var tokens = lexer.Analyze(expression).ToArray();
            Assert.Equal(37, tokens.Length);
            Assert.Equal("!", tokens[0].Value);
            Assert.Equal(TokenType.L_NOT, tokens[0].Type);
            Assert.Equal("||", tokens[1].Value);
            Assert.Equal(TokenType.L_OR, tokens[1].Type);
            Assert.Equal("&&", tokens[2].Value);
            Assert.Equal(TokenType.L_AND, tokens[2].Type);
            Assert.Equal("==", tokens[3].Value);
            Assert.Equal(TokenType.EQ, tokens[3].Type);
            Assert.Equal("!=", tokens[4].Value);
            Assert.Equal(TokenType.NEQ, tokens[4].Type);
            Assert.Equal("<", tokens[5].Value);
            Assert.Equal(TokenType.LT, tokens[5].Type);
            Assert.Equal("<=", tokens[6].Value);
            Assert.Equal(TokenType.LE, tokens[6].Type);
            Assert.Equal(">", tokens[7].Value);
            Assert.Equal(TokenType.GT, tokens[7].Type);
            Assert.Equal(">=", tokens[8].Value);
            Assert.Equal(TokenType.GE, tokens[8].Type);
            Assert.Equal("+", tokens[9].Value);
            Assert.Equal(TokenType.ADD, tokens[9].Type);
            Assert.Equal("-", tokens[10].Value);
            Assert.Equal(TokenType.SUB, tokens[10].Type);
            Assert.Equal("*", tokens[11].Value);
            Assert.Equal(TokenType.MUL, tokens[11].Type);
            Assert.Equal("/", tokens[12].Value);
            Assert.Equal(TokenType.DIV, tokens[12].Type);
            Assert.Equal("%", tokens[13].Value);
            Assert.Equal(TokenType.MOD, tokens[13].Type);
            Assert.Equal("~", tokens[14].Value);
            Assert.Equal(TokenType.B_NOT, tokens[14].Type);
            Assert.Equal("&", tokens[15].Value);
            Assert.Equal(TokenType.B_AND, tokens[15].Type);
            Assert.Equal("^", tokens[16].Value);
            Assert.Equal(TokenType.XOR, tokens[16].Type);
            Assert.Equal("|", tokens[17].Value);
            Assert.Equal(TokenType.B_OR, tokens[17].Type);
            Assert.Equal("<<", tokens[18].Value);
            Assert.Equal(TokenType.L_SHIFT, tokens[18].Type);
            Assert.Equal(">>", tokens[19].Value);
            Assert.Equal(TokenType.R_SHIFT, tokens[19].Type);
            Assert.Equal("(", tokens[20].Value);
            Assert.Equal(TokenType.L_PAR, tokens[20].Type);
            Assert.Equal(")", tokens[21].Value);
            Assert.Equal(TokenType.R_PAR, tokens[21].Type);
            Assert.Equal("[", tokens[22].Value);
            Assert.Equal(TokenType.L_BRACKET, tokens[22].Type);
            Assert.Equal("]", tokens[23].Value);
            Assert.Equal(TokenType.R_BRACKET, tokens[23].Type);
            Assert.Equal(".", tokens[24].Value);
            Assert.Equal(TokenType.PERIOD, tokens[24].Type);
            Assert.Equal("?", tokens[25].Value);
            Assert.Equal(TokenType.QMARK, tokens[25].Type);
            Assert.Equal(":", tokens[26].Value);
            Assert.Equal(TokenType.COLON, tokens[26].Type);
            Assert.Equal(null, tokens[27].Value);
            Assert.Equal(TokenType.NULL, tokens[27].Type);
            Assert.Equal(true, tokens[28].Value);
            Assert.Equal(TokenType.BOOL, tokens[28].Type);
            Assert.Equal(false, tokens[29].Value);
            Assert.Equal(TokenType.BOOL, tokens[29].Type);
            Assert.Equal(123, tokens[30].Value);
            Assert.Equal(TokenType.INT, tokens[30].Type);
            Assert.Equal(0.3e-2, tokens[31].Value);
            Assert.Equal(TokenType.FLOAT, tokens[31].Type);
            Assert.Equal(10, tokens[32].Value);
            Assert.Equal(TokenType.BIN, tokens[32].Type);
            Assert.Equal(255, tokens[33].Value);
            Assert.Equal(TokenType.HEX, tokens[33].Type);
            Assert.Equal("'\r\na\r\n b\r\nc\r\n'", tokens[34].Value); // used alternatively to verbatim string (new line \n in expression string literal has been replaced by windows \r\n)
            Assert.Equal(TokenType.STRING, tokens[34].Type);
            Assert.Equal("メidメ", tokens[35].Value);
            Assert.Equal(TokenType.ID, tokens[35].Type);
            Assert.Equal(string.Empty, tokens[36].Value);
            Assert.Equal(TokenType.EOF, tokens[36].Type);
        }
        public void verify_invalid_tokens_detection()
        {
            var lexer = new Lexer();

            var e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("true # false"));
            Assert.Equal("Invalid token.", e.Error);
            Assert.Equal(new Location(1, 6), e.Location, new LocationComparer());

            e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("true\r\n&& @"));
            Assert.Equal("Invalid token.", e.Error);
            Assert.Equal(new Location(2, 4), e.Location, new LocationComparer());

            e = Assert.Throws<ParseErrorException>(() => lexer.Analyze("'John's cat'"));
            Assert.Equal("Invalid token.", e.Error);
            Assert.Equal(new Location(1, 12), e.Location, new LocationComparer());
        }
        private static void AssertNotToken(string expression, TokenType type)
        {
            try
            {
                var lexer = new Lexer();
                var tokens = lexer.Analyze(expression).ToArray();

                Assert.IsFalse(tokens.Length == 2 && tokens[0].Type == type);
            }
            catch (Exception e)
            {
                Assert.IsTrue(e is ParseErrorException);
                Assert.AreEqual("Invalid token.", e.Message);
            }
        }
        public void verify_invalid_tokens_detection()
        {
            var lexer = new Lexer();

            try
            {
                lexer.Analyze(null);
                Assert.Fail();
            }
            catch (Exception e)
            {
                Assert.IsTrue(e is ArgumentNullException);
                Assert.AreEqual("Expression not provided.\r\nParameter name: expression", e.Message);
            }

            try
            {
                lexer.Analyze("true # false");
                Assert.Fail();
            }
            catch (Exception e)
            {
                Assert.IsTrue(e is ParseErrorException);
                Assert.AreEqual("Invalid token.", e.Message);

                var ctx = ((ParseErrorException) e).Location;
                Assert.AreEqual(1, ctx.Line);
                Assert.AreEqual(6, ctx.Column);
            }

            try
            {
                lexer.Analyze("true\r\n&& ^");
                Assert.Fail();
            }
            catch (Exception e)
            {
                Assert.IsTrue(e is ParseErrorException);
                Assert.AreEqual("Invalid token.", e.Message);

                var ctx = ((ParseErrorException) e).Location;
                Assert.AreEqual(2, ctx.Line);
                Assert.AreEqual(4, ctx.Column);
            }

            try
            {
                lexer.Analyze("'John's cat'");
                Assert.Fail();
            }
            catch (Exception e)
            {
                Assert.IsTrue(e is ParseErrorException);
                Assert.AreEqual("Invalid token.", e.Message);

                var ctx = ((ParseErrorException) e).Location;
                Assert.AreEqual(1, ctx.Line);
                Assert.AreEqual(12, ctx.Column);
            }
        }
        public void verify_complex_expression_analysis()
        {
            const string expression =
                @"GoAbroad == true
                      && (
                             (NextCountry != 'european country' && Compare(NextCountry, Country.Name) == 0)
                             || (Age > 24 && Age <= 55.5)
                             && (1+2*2>1-2/2)
                         )";

            var lexer = new Lexer();
            var tokens = lexer.Analyze(expression).ToArray();
            Assert.AreEqual(45, tokens.Length);
            Assert.AreEqual("GoAbroad", tokens[0].Value);
            Assert.AreEqual(TokenType.FUNC, tokens[0].Type);
            Assert.AreEqual("==", tokens[1].Value);
            Assert.AreEqual(TokenType.EQ, tokens[1].Type);
            Assert.AreEqual(true, tokens[2].Value);
            Assert.AreEqual(TokenType.BOOL, tokens[2].Type);
            Assert.AreEqual("&&", tokens[3].Value);
            Assert.AreEqual(TokenType.AND, tokens[3].Type);
            Assert.AreEqual("(", tokens[4].Value);
            Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[4].Type);
            Assert.AreEqual("(", tokens[5].Value);
            Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[5].Type);
            Assert.AreEqual("NextCountry", tokens[6].Value);
            Assert.AreEqual(TokenType.FUNC, tokens[6].Type);
            Assert.AreEqual("!=", tokens[7].Value);
            Assert.AreEqual(TokenType.NEQ, tokens[7].Type);
            Assert.AreEqual("european country", tokens[8].Value);
            Assert.AreEqual(TokenType.STRING, tokens[8].Type);
            Assert.AreEqual("&&", tokens[9].Value);
            Assert.AreEqual(TokenType.AND, tokens[9].Type);
            Assert.AreEqual("Compare", tokens[10].Value);
            Assert.AreEqual(TokenType.FUNC, tokens[10].Type);
            Assert.AreEqual("(", tokens[11].Value);
            Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[11].Type);
            Assert.AreEqual("NextCountry", tokens[12].Value);
            Assert.AreEqual(TokenType.FUNC, tokens[12].Type);
            Assert.AreEqual(",", tokens[13].Value);
            Assert.AreEqual(TokenType.COMMA, tokens[13].Type);
            Assert.AreEqual("Country.Name", tokens[14].Value);
            Assert.AreEqual(TokenType.FUNC, tokens[14].Type);
            Assert.AreEqual(")", tokens[15].Value);
            Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[15].Type);
            Assert.AreEqual("==", tokens[16].Value);
            Assert.AreEqual(TokenType.EQ, tokens[16].Type);
            Assert.AreEqual(0, tokens[17].Value);
            Assert.AreEqual(TokenType.INT, tokens[17].Type);
            Assert.AreEqual(")", tokens[18].Value);
            Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[18].Type);
            Assert.AreEqual("||", tokens[19].Value);
            Assert.AreEqual(TokenType.OR, tokens[19].Type);
            Assert.AreEqual("(", tokens[20].Value);
            Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[20].Type);
            Assert.AreEqual("Age", tokens[21].Value);
            Assert.AreEqual(TokenType.FUNC, tokens[21].Type);
            Assert.AreEqual(">", tokens[22].Value);
            Assert.AreEqual(TokenType.GT, tokens[22].Type);
            Assert.AreEqual(24, tokens[23].Value);
            Assert.AreEqual(TokenType.INT, tokens[23].Type);
            Assert.AreEqual("&&", tokens[24].Value);
            Assert.AreEqual(TokenType.AND, tokens[24].Type);
            Assert.AreEqual("Age", tokens[25].Value);
            Assert.AreEqual(TokenType.FUNC, tokens[25].Type);
            Assert.AreEqual("<=", tokens[26].Value);
            Assert.AreEqual(TokenType.LE, tokens[26].Type);
            Assert.AreEqual(55.5d, tokens[27].Value);
            Assert.AreEqual(TokenType.FLOAT, tokens[27].Type);
            Assert.AreEqual(")", tokens[28].Value);
            Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[28].Type);
            Assert.AreEqual("&&", tokens[29].Value);
            Assert.AreEqual(TokenType.AND, tokens[29].Type);
            Assert.AreEqual("(", tokens[30].Value);
            Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[30].Type);
            Assert.AreEqual(1, tokens[31].Value);
            Assert.AreEqual(TokenType.INT, tokens[31].Type);
            Assert.AreEqual("+", tokens[32].Value);
            Assert.AreEqual(TokenType.ADD, tokens[32].Type);
            Assert.AreEqual(2, tokens[33].Value);
            Assert.AreEqual(TokenType.INT, tokens[33].Type);
            Assert.AreEqual("*", tokens[34].Value);
            Assert.AreEqual(TokenType.MUL, tokens[34].Type);
            Assert.AreEqual(2, tokens[35].Value);
            Assert.AreEqual(TokenType.INT, tokens[35].Type);
            Assert.AreEqual(">", tokens[36].Value);
            Assert.AreEqual(TokenType.GT, tokens[36].Type);
            Assert.AreEqual(1, tokens[37].Value);
            Assert.AreEqual(TokenType.INT, tokens[37].Type);
            Assert.AreEqual("-", tokens[38].Value);
            Assert.AreEqual(TokenType.SUB, tokens[38].Type);
            Assert.AreEqual(2, tokens[39].Value);
            Assert.AreEqual(TokenType.INT, tokens[39].Type);
            Assert.AreEqual("/", tokens[40].Value);
            Assert.AreEqual(TokenType.DIV, tokens[40].Type);
            Assert.AreEqual(2, tokens[41].Value);
            Assert.AreEqual(TokenType.INT, tokens[41].Type);
            Assert.AreEqual(")", tokens[42].Value);
            Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[42].Type);
            Assert.AreEqual(")", tokens[43].Value);
            Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[43].Type);
            Assert.AreEqual(string.Empty, tokens[44].Value);
            Assert.AreEqual(TokenType.EOF, tokens[44].Type);
        }
        private void InitTokenizer(string expression)
        {
            var lexer = new Lexer();

            Tokens = new Stack <Token>(lexer.Analyze(expression).Reverse());
        }