Example #1
0
        static void Main(string[] args)
        {
            var text = "foo 123 bar";

            using (var sr = new StreamReader(@"..\..\Program.cs"))
                text = sr.ReadToEnd();

            Console.WriteLine("Lex: " + text);

            var tokenizer = new TestTokenizer(text);             // generated from Example.lx

            Console.WriteLine("Disassembly:");
            Console.WriteLine(Lex.Disassemble(TestTokenizer.Program));
            Console.WriteLine();

            foreach (var tok in tokenizer)
            {
                // we don't want errors or whitespace but we don't know the symbol
                // id for whitespace because you can switch tokenizers around
                // so we check tok.Value instead
                if (-1 != tok.SymbolId && !string.IsNullOrWhiteSpace(tok.Value))
                {
                    Console.WriteLine("{0}: {1}", tok.SymbolId, tok.Value);
                }
            }

            Stopwatch sw   = new Stopwatch();
            const int ITER = 1000;

            for (var i = 0; i < ITER; ++i)
            {
                var lc = LexContext.Create(text);
                while (LexContext.EndOfInput != lc.Current)
                {
                    lc.ClearCapture();
                    sw.Start();
                    var acc = Lex.Run(TestTokenizer.Program, lc);
                    sw.Stop();
                }
            }

            Console.WriteLine("Lexed in " + sw.ElapsedMilliseconds / (float)ITER + " msec");
        }
Example #2
0
        public void TestTokenizerParsing()
        {
            TestTokenizer tokenizer = new TestTokenizer();
            {
                string           input  = "abc def123.456ghi7...__xx;";
                List <CodeToken> tokens = tokenizer.Tokenize(input.ToCharArray());

                Assert.AreEqual(TestTokenizer.Id, tokens[0].Id);
                Assert.AreEqual(new TextPosition(0, 0), tokens[0].Start);
                Assert.AreEqual(new TextPosition(0, 3), tokens[0].End);
                Assert.AreEqual("abc", tokens[0].Value);

                Assert.AreEqual(-1, tokens[1].Id);
                Assert.AreEqual(new TextPosition(0, 3), tokens[1].Start);
                Assert.AreEqual(new TextPosition(0, 4), tokens[1].End);
                Assert.AreEqual(" ", tokens[1].Value);

                Assert.AreEqual(TestTokenizer.Id, tokens[2].Id);
                Assert.AreEqual(new TextPosition(0, 4), tokens[2].Start);
                Assert.AreEqual(new TextPosition(0, 10), tokens[2].End);
                Assert.AreEqual("def123", tokens[2].Value);

                Assert.AreEqual(-1, tokens[3].Id);
                Assert.AreEqual(new TextPosition(0, 10), tokens[3].Start);
                Assert.AreEqual(new TextPosition(0, 11), tokens[3].End);
                Assert.AreEqual(".", tokens[3].Value);

                Assert.AreEqual(TestTokenizer.Integer, tokens[4].Id);
                Assert.AreEqual(new TextPosition(0, 11), tokens[4].Start);
                Assert.AreEqual(new TextPosition(0, 14), tokens[4].End);
                Assert.AreEqual("456", tokens[4].Value);

                Assert.AreEqual(TestTokenizer.Id, tokens[5].Id);
                Assert.AreEqual(new TextPosition(0, 14), tokens[5].Start);
                Assert.AreEqual(new TextPosition(0, 18), tokens[5].End);
                Assert.AreEqual("ghi7", tokens[5].Value);

                Assert.AreEqual(-1, tokens[6].Id);
                Assert.AreEqual(new TextPosition(0, 18), tokens[6].Start);
                Assert.AreEqual(new TextPosition(0, 19), tokens[6].End);
                Assert.AreEqual(".", tokens[6].Value);

                Assert.AreEqual(-1, tokens[7].Id);
                Assert.AreEqual(new TextPosition(0, 19), tokens[7].Start);
                Assert.AreEqual(new TextPosition(0, 20), tokens[7].End);
                Assert.AreEqual(".", tokens[7].Value);

                Assert.AreEqual(-1, tokens[8].Id);
                Assert.AreEqual(new TextPosition(0, 20), tokens[8].Start);
                Assert.AreEqual(new TextPosition(0, 21), tokens[8].End);
                Assert.AreEqual(".", tokens[8].Value);

                Assert.AreEqual(TestTokenizer.Id, tokens[9].Id);
                Assert.AreEqual(new TextPosition(0, 21), tokens[9].Start);
                Assert.AreEqual(new TextPosition(0, 25), tokens[9].End);
                Assert.AreEqual("__xx", tokens[9].Value);

                Assert.AreEqual(-1, tokens[10].Id);
                Assert.AreEqual(new TextPosition(0, 25), tokens[10].Start);
                Assert.AreEqual(new TextPosition(0, 26), tokens[10].End);
                Assert.AreEqual(";", tokens[10].Value);
            }
            {
                string           input  = "abc def;123.456gh?7...__xx;";
                List <CodeToken> tokens = tokenizer.Tokenize(input.ToCharArray());

                Assert.AreEqual(TestTokenizer.Id, tokens[0].Id);
                Assert.AreEqual(new TextPosition(0, 0), tokens[0].Start);
                Assert.AreEqual(new TextPosition(0, 3), tokens[0].End);
                Assert.AreEqual("abc", tokens[0].Value);

                Assert.AreEqual(-1, tokens[1].Id);
                Assert.AreEqual(new TextPosition(0, 3), tokens[1].Start);
                Assert.AreEqual(new TextPosition(0, 4), tokens[1].End);
                Assert.AreEqual(" ", tokens[1].Value);

                Assert.AreEqual(TestTokenizer.Id, tokens[2].Id);
                Assert.AreEqual(new TextPosition(0, 4), tokens[2].Start);
                Assert.AreEqual(new TextPosition(0, 7), tokens[2].End);
                Assert.AreEqual("def", tokens[2].Value);

                Assert.AreEqual(-1, tokens[3].Id);
                Assert.AreEqual(new TextPosition(0, 7), tokens[3].Start);
                Assert.AreEqual(new TextPosition(0, 8), tokens[3].End);
                Assert.AreEqual(";", tokens[3].Value);

                Assert.AreEqual(TestTokenizer.Double, tokens[4].Id);
                Assert.AreEqual(new TextPosition(0, 8), tokens[4].Start);
                Assert.AreEqual(new TextPosition(0, 15), tokens[4].End);
                Assert.AreEqual("123.456", tokens[4].Value);

                Assert.AreEqual(TestTokenizer.Id, tokens[5].Id);
                Assert.AreEqual(new TextPosition(0, 15), tokens[5].Start);
                Assert.AreEqual(new TextPosition(0, 17), tokens[5].End);
                Assert.AreEqual("gh", tokens[5].Value);

                Assert.AreEqual(-1, tokens[6].Id);
                Assert.AreEqual(new TextPosition(0, 17), tokens[6].Start);
                Assert.AreEqual(new TextPosition(0, 18), tokens[6].End);
                Assert.AreEqual("?", tokens[6].Value);

                Assert.AreEqual(TestTokenizer.Integer, tokens[7].Id);
                Assert.AreEqual(new TextPosition(0, 18), tokens[7].Start);
                Assert.AreEqual(new TextPosition(0, 19), tokens[7].End);
                Assert.AreEqual("7", tokens[7].Value);

                Assert.AreEqual(-1, tokens[8].Id);
                Assert.AreEqual(new TextPosition(0, 19), tokens[8].Start);
                Assert.AreEqual(new TextPosition(0, 20), tokens[8].End);
                Assert.AreEqual(".", tokens[8].Value);

                Assert.AreEqual(-1, tokens[9].Id);
                Assert.AreEqual(new TextPosition(0, 20), tokens[9].Start);
                Assert.AreEqual(new TextPosition(0, 21), tokens[9].End);
                Assert.AreEqual(".", tokens[9].Value);

                Assert.AreEqual(-1, tokens[10].Id);
                Assert.AreEqual(new TextPosition(0, 21), tokens[10].Start);
                Assert.AreEqual(new TextPosition(0, 22), tokens[10].End);
                Assert.AreEqual(".", tokens[10].Value);

                Assert.AreEqual(TestTokenizer.Id, tokens[11].Id);
                Assert.AreEqual(new TextPosition(0, 22), tokens[11].Start);
                Assert.AreEqual(new TextPosition(0, 26), tokens[11].End);
                Assert.AreEqual("__xx", tokens[11].Value);

                Assert.AreEqual(-1, tokens[12].Id);
                Assert.AreEqual(new TextPosition(0, 26), tokens[12].Start);
                Assert.AreEqual(new TextPosition(0, 27), tokens[12].End);
                Assert.AreEqual(";", tokens[12].Value);
            }
        }