예제 #1
0
        static void _RunLexer()
        {
            // compile a lexer
            var prog = Lex.CompileLexerRegex(true,
                                             @"[A-Z_a-z][A-Z_a-z0-9]*", // id
                                             @"0|(\-?[1-9][0-9]*)",     // int
                                             @"( |\t|\r|\n|\v|\f)"      // space
                                             );

            // dump the program to the console
            Console.WriteLine(Lex.Disassemble(prog));

            // our test data - 14 tokens. 29 length
            var text = "fubar bar 123 1foo bar -243 0";

            Console.WriteLine("Lex: " + text);

            // spin up a lexer context
            // see: https://www.codeproject.com/Articles/5256794/LexContext-A-streamlined-cursor-over-a-text-input
            var lc = LexContext.Create(text);

            // while more input to be read
            while (LexContext.EndOfInput != lc.Current)
            {
                // clear any current captured data
                lc.ClearCapture();
                // lex our next input and dump it
                Console.WriteLine("{0}: \"{1}\"", Lex.Run(prog, lc), lc.GetCapture());
            }
            var       sw   = new Stopwatch();
            const int ITER = 1000;

            for (var i = 0; i < ITER; ++i)
            {
                lc = LexContext.Create(text);
                while (LexContext.EndOfInput != lc.Current)
                {
                    lc.ClearCapture();
                    sw.Start();
                    var acc = Lex.Run(prog, lc);
                    sw.Stop();
                }
            }
            Console.WriteLine("Lexed in " + sw.ElapsedMilliseconds / (float)ITER + " msec");
        }
예제 #2
0
        static void Main2()
        {
            var id    = @"[A-Z_a-z][A-Z_a-z0-9]*";
            var @int  = @"0|(\-?[1-9][0-9]*)";
            var space = @"( |\t|\r|\n|\v|\f)";

            Lex.RenderOptimizedExecutionGraph(id, @"..\..\id_nfa.jpg");
            Lex.RenderOptimizedExecutionGraph(@int, @"..\..\int_nfa.jpg");
            Lex.RenderOptimizedExecutionGraph(@space, @"..\..\space_nfa.jpg");
            var prog = Lex.CompileLexerRegex(true,
                                             id,   // id
                                             @int, // int
                                             space // space
                                             );

            prog = Lex.CompileRegexPart(@int);
            prog = Lex.FinalizePart(prog);
            Console.WriteLine(Lex.Disassemble(prog));
            Console.WriteLine(Lex.RunWithLogging(prog, LexContext.Create("123"), Console.Out));
        }
예제 #3
0
        static void _RunLexer(int[][] prog)
        {
            // compile a lexer
            if (null == prog)
            {
                prog = Lex.CompileLexerRegex(true,
                                             @"[A-Z_a-z][A-Z_a-z0-9]*", // id
                                             @"0|(\-?[1-9][0-9]*)",     // int
                                             @"( |\t|\r|\n|\v|\f)"      // space
                                             );
            }

            // dump the program to the console
            //Console.WriteLine(Lex.Disassemble(prog));

            // our test data - 14 tokens. 29 length
            var text = "#@*($ fubar bar 123 1foo bar -243 0";

            Console.WriteLine("Lex: " + text);

            // spin up a lexer context
            // see: https://www.codeproject.com/Articles/5256794/LexContext-A-streamlined-cursor-over-a-text-input
            var lc = LexContext.Create(text);

            // while more input to be read
            while (LexContext.EndOfInput != lc.Current)
            {
                // clear any current captured data
                lc.ClearCapture();
                int acc;
                var stat = Lex.RunWithLoggingAndStatistics(prog, lc,
#if VERBOSE
                                                           Console.Error
#else
                                                           TextWriter.Null
#endif
                                                           , out acc);
                // lex our next input and dump it
                Console.WriteLine("{0}: \"{1}\"", acc, lc.GetCapture());
            }
        }
예제 #4
0
        static void Main()
        {
            var test = "fubar bar 123 1foo bar -243 0";

            Console.WriteLine("Lex: " + test);
            var prog = Lex.CompileLexerRegex(false,
                                             @"[A-Z_a-z][A-Z_a-z0-9]*", // id
                                             @"0|(\-?[1-9][0-9]*)",     // int
                                             @"( |\t|\r|\n|\v|\f)"      // space
                                             );

            Console.WriteLine("Unoptimized dump:");
            Console.WriteLine(Lex.Disassemble(prog));
            Console.WriteLine();
            var progOpt = Lex.CompileLexerRegex(true,
                                                @"[A-Z_a-z][A-Z_a-z0-9]*", // id
                                                @"0|(\-?[1-9][0-9]*)",     // int
                                                @"( |\t|\r|\n|\v|\f)"      // space
                                                );

            Console.WriteLine("Optimized dump:");
            Console.WriteLine(Lex.Disassemble(progOpt));
            Console.WriteLine();
            var progDfa = Lex.AssembleFrom(@"..\..\dfa.lasm");

            Console.WriteLine("DFA dump:");
            Console.WriteLine(Lex.Disassemble(progDfa));
            Console.WriteLine();
            for (var i = 0; i < 10; ++i)
            {
                Console.WriteLine("Pass #" + (i + 1));
                Console.Write("NFA: ");
                Perf(prog, test);
                Console.Write("NFA+DFA (optimized): ");
                Perf(progOpt, test);

                Console.Write("DFA: ");
                Perf(progDfa, test);
            }
        }
예제 #5
0
        static void Main()
        {
            var test = "fubar bar 123 1foo bar -243 0 baz 83";

            Console.WriteLine("Lex: " + test);
            var prog = Lex.CompileLexerRegex(false,
                                             @"[A-Z_a-z][A-Z_a-z0-9]*", // id
                                             @"0|(\-?[1-9][0-9]*)",     // int
                                             @"( |\t|\r|\n|\v|\f)"      // space
                                             );

            Console.WriteLine("Unoptimized dump:");
            Console.WriteLine(Lex.Disassemble(prog));
            Console.WriteLine();

            var progOpt = Lex.CompileLexerRegex(true,
                                                @"[A-Z_a-z][A-Z_a-z0-9]*", // id
                                                @"0|(\-?[1-9][0-9]*)",     // int
                                                @"( |\t|\r|\n|\v|\f)"      // space
                                                );

            Console.WriteLine("Optimized dump:");
            Console.WriteLine(Lex.Disassemble(progOpt));
            Console.WriteLine();
            var progDfa = Lex.AssembleFrom(@"..\..\dfa.lasm");

            Console.WriteLine("DFA dump:");
            Console.WriteLine(Lex.Disassemble(progDfa));
            Console.WriteLine();

            var nfa = FA.ToLexer(new FA[]
            {
                FA.Parse(@"[A-Z_a-z][A-Z_a-z0-9]*", 0),
                FA.Parse(@"0|(\-?[1-9][0-9]*)", 1),
                FA.Parse(@"( |\t|\r|\n|\v|\f)", 2)
            });
            var dfa = nfa.ToDfa();

            dfa.TrimDuplicates();
            var dfaTable = dfa.ToDfaStateTable();

            var        result        = -1;
            var        count         = 0f;
            var        maxFiberCount = 0;
            var        avgCharPasses = 0f;
            LexContext lc            = LexContext.Create(test);

            while (LexContext.EndOfInput != lc.Current)
            {
                var stats = Lex.RunWithLoggingAndStatistics(prog, lc, TextWriter.Null, out result);
                maxFiberCount = stats.MaxFiberCount;
                if (stats.AverageCharacterPasses > avgCharPasses)
                {
                    avgCharPasses = stats.AverageCharacterPasses;
                }

                ++count;
            }
            Console.WriteLine("NFA ran with " + maxFiberCount + " max fibers and " + avgCharPasses + " average char passes");
            count         = 0f;
            maxFiberCount = 0;
            avgCharPasses = 0f;
            count         = 0;
            lc            = LexContext.Create(test);
            while (LexContext.EndOfInput != lc.Current)
            {
                var stats = Lex.RunWithLoggingAndStatistics(progOpt, lc, TextWriter.Null, out result);
                maxFiberCount = stats.MaxFiberCount;
                if (stats.AverageCharacterPasses > avgCharPasses)
                {
                    avgCharPasses = stats.AverageCharacterPasses;
                }

                ++count;
            }
            Console.WriteLine("NFA+DFA (optimized) ran with " + maxFiberCount + " max fibers and " + avgCharPasses + " average char passes");
            count         = 0;
            maxFiberCount = 0;
            avgCharPasses = 0f;
            lc            = LexContext.Create(test);
            while (LexContext.EndOfInput != lc.Current)
            {
                var stats = Lex.RunWithLoggingAndStatistics(progDfa, lc, TextWriter.Null, out result);
                maxFiberCount = stats.MaxFiberCount;
                if (stats.AverageCharacterPasses > avgCharPasses)
                {
                    avgCharPasses = stats.AverageCharacterPasses;
                }

                ++count;
            }
            Console.WriteLine("DFA ran with " + maxFiberCount + " max fibers and " + avgCharPasses + " average char passes");
#if RUNPERF
            for (var i = 0; i < 5; ++i)
            {
                test = string.Concat(test, test);
            }

            for (var i = 0; i < 10; ++i)
            {
                Console.WriteLine("Pass #" + (i + 1));
                Console.Write("NFA: ");
                _Perf(prog, test);
                Console.WriteLine();
                Console.Write("NFA+DFA (optimized): ");
                _Perf(progOpt, test);
                Console.WriteLine();
                Console.Write("DFA: ");
                _Perf(progDfa, test);
                Console.WriteLine();
                Console.Write("NFA (raw): ");
                _Perf(nfa, test);
                Console.WriteLine();
                Console.Write("DFA (raw): ");
                _Perf(dfaTable, test);
                Console.WriteLine();
                Console.WriteLine();
            }
            Console.WriteLine();
#endif
            _RunLexer(progOpt);
            Console.Error.WriteLine();
        }