static void _RunLexer() { // compile a lexer var prog = Lex.CompileLexerRegex(true, @"[A-Z_a-z][A-Z_a-z0-9]*", // id @"0|(\-?[1-9][0-9]*)", // int @"( |\t|\r|\n|\v|\f)" // space ); // dump the program to the console Console.WriteLine(Lex.Disassemble(prog)); // our test data - 14 tokens. 29 length var text = "fubar bar 123 1foo bar -243 0"; Console.WriteLine("Lex: " + text); // spin up a lexer context // see: https://www.codeproject.com/Articles/5256794/LexContext-A-streamlined-cursor-over-a-text-input var lc = LexContext.Create(text); // while more input to be read while (LexContext.EndOfInput != lc.Current) { // clear any current captured data lc.ClearCapture(); // lex our next input and dump it Console.WriteLine("{0}: \"{1}\"", Lex.Run(prog, lc), lc.GetCapture()); } var sw = new Stopwatch(); const int ITER = 1000; for (var i = 0; i < ITER; ++i) { lc = LexContext.Create(text); while (LexContext.EndOfInput != lc.Current) { lc.ClearCapture(); sw.Start(); var acc = Lex.Run(prog, lc); sw.Stop(); } } Console.WriteLine("Lexed in " + sw.ElapsedMilliseconds / (float)ITER + " msec"); }
static void Main2() { var id = @"[A-Z_a-z][A-Z_a-z0-9]*"; var @int = @"0|(\-?[1-9][0-9]*)"; var space = @"( |\t|\r|\n|\v|\f)"; Lex.RenderOptimizedExecutionGraph(id, @"..\..\id_nfa.jpg"); Lex.RenderOptimizedExecutionGraph(@int, @"..\..\int_nfa.jpg"); Lex.RenderOptimizedExecutionGraph(@space, @"..\..\space_nfa.jpg"); var prog = Lex.CompileLexerRegex(true, id, // id @int, // int space // space ); prog = Lex.CompileRegexPart(@int); prog = Lex.FinalizePart(prog); Console.WriteLine(Lex.Disassemble(prog)); Console.WriteLine(Lex.RunWithLogging(prog, LexContext.Create("123"), Console.Out)); }
static void _RunLexer(int[][] prog) { // compile a lexer if (null == prog) { prog = Lex.CompileLexerRegex(true, @"[A-Z_a-z][A-Z_a-z0-9]*", // id @"0|(\-?[1-9][0-9]*)", // int @"( |\t|\r|\n|\v|\f)" // space ); } // dump the program to the console //Console.WriteLine(Lex.Disassemble(prog)); // our test data - 14 tokens. 29 length var text = "#@*($ fubar bar 123 1foo bar -243 0"; Console.WriteLine("Lex: " + text); // spin up a lexer context // see: https://www.codeproject.com/Articles/5256794/LexContext-A-streamlined-cursor-over-a-text-input var lc = LexContext.Create(text); // while more input to be read while (LexContext.EndOfInput != lc.Current) { // clear any current captured data lc.ClearCapture(); int acc; var stat = Lex.RunWithLoggingAndStatistics(prog, lc, #if VERBOSE Console.Error #else TextWriter.Null #endif , out acc); // lex our next input and dump it Console.WriteLine("{0}: \"{1}\"", acc, lc.GetCapture()); } }
static void Main() { var test = "fubar bar 123 1foo bar -243 0"; Console.WriteLine("Lex: " + test); var prog = Lex.CompileLexerRegex(false, @"[A-Z_a-z][A-Z_a-z0-9]*", // id @"0|(\-?[1-9][0-9]*)", // int @"( |\t|\r|\n|\v|\f)" // space ); Console.WriteLine("Unoptimized dump:"); Console.WriteLine(Lex.Disassemble(prog)); Console.WriteLine(); var progOpt = Lex.CompileLexerRegex(true, @"[A-Z_a-z][A-Z_a-z0-9]*", // id @"0|(\-?[1-9][0-9]*)", // int @"( |\t|\r|\n|\v|\f)" // space ); Console.WriteLine("Optimized dump:"); Console.WriteLine(Lex.Disassemble(progOpt)); Console.WriteLine(); var progDfa = Lex.AssembleFrom(@"..\..\dfa.lasm"); Console.WriteLine("DFA dump:"); Console.WriteLine(Lex.Disassemble(progDfa)); Console.WriteLine(); for (var i = 0; i < 10; ++i) { Console.WriteLine("Pass #" + (i + 1)); Console.Write("NFA: "); Perf(prog, test); Console.Write("NFA+DFA (optimized): "); Perf(progOpt, test); Console.Write("DFA: "); Perf(progDfa, test); } }
static void Main() { var test = "fubar bar 123 1foo bar -243 0 baz 83"; Console.WriteLine("Lex: " + test); var prog = Lex.CompileLexerRegex(false, @"[A-Z_a-z][A-Z_a-z0-9]*", // id @"0|(\-?[1-9][0-9]*)", // int @"( |\t|\r|\n|\v|\f)" // space ); Console.WriteLine("Unoptimized dump:"); Console.WriteLine(Lex.Disassemble(prog)); Console.WriteLine(); var progOpt = Lex.CompileLexerRegex(true, @"[A-Z_a-z][A-Z_a-z0-9]*", // id @"0|(\-?[1-9][0-9]*)", // int @"( |\t|\r|\n|\v|\f)" // space ); Console.WriteLine("Optimized dump:"); Console.WriteLine(Lex.Disassemble(progOpt)); Console.WriteLine(); var progDfa = Lex.AssembleFrom(@"..\..\dfa.lasm"); Console.WriteLine("DFA dump:"); Console.WriteLine(Lex.Disassemble(progDfa)); Console.WriteLine(); var nfa = FA.ToLexer(new FA[] { FA.Parse(@"[A-Z_a-z][A-Z_a-z0-9]*", 0), FA.Parse(@"0|(\-?[1-9][0-9]*)", 1), FA.Parse(@"( |\t|\r|\n|\v|\f)", 2) }); var dfa = nfa.ToDfa(); dfa.TrimDuplicates(); var dfaTable = dfa.ToDfaStateTable(); var result = -1; var count = 0f; var maxFiberCount = 0; var avgCharPasses = 0f; LexContext lc = LexContext.Create(test); while (LexContext.EndOfInput != lc.Current) { var stats = Lex.RunWithLoggingAndStatistics(prog, lc, TextWriter.Null, out result); maxFiberCount = stats.MaxFiberCount; if (stats.AverageCharacterPasses > avgCharPasses) { avgCharPasses = stats.AverageCharacterPasses; } ++count; } Console.WriteLine("NFA ran with " + maxFiberCount + " max fibers and " + avgCharPasses + " average char passes"); count = 0f; maxFiberCount = 0; avgCharPasses = 0f; count = 0; lc = LexContext.Create(test); while (LexContext.EndOfInput != lc.Current) { var stats = Lex.RunWithLoggingAndStatistics(progOpt, lc, TextWriter.Null, out result); maxFiberCount = stats.MaxFiberCount; if (stats.AverageCharacterPasses > avgCharPasses) { avgCharPasses = stats.AverageCharacterPasses; } ++count; } Console.WriteLine("NFA+DFA (optimized) ran with " + maxFiberCount + " max fibers and " + avgCharPasses + " average char passes"); count = 0; maxFiberCount = 0; avgCharPasses = 0f; lc = LexContext.Create(test); while (LexContext.EndOfInput != lc.Current) { var stats = Lex.RunWithLoggingAndStatistics(progDfa, lc, TextWriter.Null, out result); maxFiberCount = stats.MaxFiberCount; if (stats.AverageCharacterPasses > avgCharPasses) { avgCharPasses = stats.AverageCharacterPasses; } ++count; } Console.WriteLine("DFA ran with " + maxFiberCount + " max fibers and " + avgCharPasses + " average char passes"); #if RUNPERF for (var i = 0; i < 5; ++i) { test = string.Concat(test, test); } for (var i = 0; i < 10; ++i) { Console.WriteLine("Pass #" + (i + 1)); Console.Write("NFA: "); _Perf(prog, test); Console.WriteLine(); Console.Write("NFA+DFA (optimized): "); _Perf(progOpt, test); Console.WriteLine(); Console.Write("DFA: "); _Perf(progDfa, test); Console.WriteLine(); Console.Write("NFA (raw): "); _Perf(nfa, test); Console.WriteLine(); Console.Write("DFA (raw): "); _Perf(dfaTable, test); Console.WriteLine(); Console.WriteLine(); } Console.WriteLine(); #endif _RunLexer(progOpt); Console.Error.WriteLine(); }