static void _RunLexer()
{
// compile a lexer
var prog = Lex.CompileLexerRegex(true,
@"[A-Z_a-z][A-Z_a-z0-9]*", // id
@"0|(\-?[1-9][0-9]*)", // int
@"( |\t|\r|\n|\v|\f)" // space
);
// dump the program to the console
Console.WriteLine(Lex.Disassemble(prog));
// our test data - 14 tokens. 29 length
var text = "fubar bar 123 1foo bar -243 0";
Console.WriteLine("Lex: " + text);
// spin up a lexer context
// see: https://www.codeproject.com/Articles/5256794/LexContext-A-streamlined-cursor-over-a-text-input
var lc = LexContext.Create(text);
// while more input to be read
while (LexContext.EndOfInput != lc.Current)
{
// clear any current captured data
lc.ClearCapture();
// lex our next input and dump it
Console.WriteLine("{0}: \"{1}\"", Lex.Run(prog, lc), lc.GetCapture());
}
var sw = new Stopwatch();
const int ITER = 1000;
for (var i = 0; i < ITER; ++i)
{
lc = LexContext.Create(text);
while (LexContext.EndOfInput != lc.Current)
{
lc.ClearCapture();
sw.Start();
var acc = Lex.Run(prog, lc);
sw.Stop();
}
}
Console.WriteLine("Lexed in " + sw.ElapsedMilliseconds / (float)ITER + " msec");
}
static void Main2()
{
var id = @"[A-Z_a-z][A-Z_a-z0-9]*";
var @int = @"0|(\-?[1-9][0-9]*)";
var space = @"( |\t|\r|\n|\v|\f)";
Lex.RenderOptimizedExecutionGraph(id, @"..\..\id_nfa.jpg");
Lex.RenderOptimizedExecutionGraph(@int, @"..\..\int_nfa.jpg");
Lex.RenderOptimizedExecutionGraph(@space, @"..\..\space_nfa.jpg");
var prog = Lex.CompileLexerRegex(true,
id, // id
@int, // int
space // space
);
prog = Lex.CompileRegexPart(@int);
prog = Lex.FinalizePart(prog);
Console.WriteLine(Lex.Disassemble(prog));
Console.WriteLine(Lex.RunWithLogging(prog, LexContext.Create("123"), Console.Out));
}
static void _RunLexer(int[][] prog)
{
// compile a lexer
if (null == prog)
{
prog = Lex.CompileLexerRegex(true,
@"[A-Z_a-z][A-Z_a-z0-9]*", // id
@"0|(\-?[1-9][0-9]*)", // int
@"( |\t|\r|\n|\v|\f)" // space
);
}
// dump the program to the console
//Console.WriteLine(Lex.Disassemble(prog));
// our test data - 14 tokens. 29 length
var text = "#@*($ fubar bar 123 1foo bar -243 0";
Console.WriteLine("Lex: " + text);
// spin up a lexer context
// see: https://www.codeproject.com/Articles/5256794/LexContext-A-streamlined-cursor-over-a-text-input
var lc = LexContext.Create(text);
// while more input to be read
while (LexContext.EndOfInput != lc.Current)
{
// clear any current captured data
lc.ClearCapture();
int acc;
var stat = Lex.RunWithLoggingAndStatistics(prog, lc,
#if VERBOSE
Console.Error
#else
TextWriter.Null
#endif
, out acc);
// lex our next input and dump it
Console.WriteLine("{0}: \"{1}\"", acc, lc.GetCapture());
}
}
static void Main()
{
var test = "fubar bar 123 1foo bar -243 0";
Console.WriteLine("Lex: " + test);
var prog = Lex.CompileLexerRegex(false,
@"[A-Z_a-z][A-Z_a-z0-9]*", // id
@"0|(\-?[1-9][0-9]*)", // int
@"( |\t|\r|\n|\v|\f)" // space
);
Console.WriteLine("Unoptimized dump:");
Console.WriteLine(Lex.Disassemble(prog));
Console.WriteLine();
var progOpt = Lex.CompileLexerRegex(true,
@"[A-Z_a-z][A-Z_a-z0-9]*", // id
@"0|(\-?[1-9][0-9]*)", // int
@"( |\t|\r|\n|\v|\f)" // space
);
Console.WriteLine("Optimized dump:");
Console.WriteLine(Lex.Disassemble(progOpt));
Console.WriteLine();
var progDfa = Lex.AssembleFrom(@"..\..\dfa.lasm");
Console.WriteLine("DFA dump:");
Console.WriteLine(Lex.Disassemble(progDfa));
Console.WriteLine();
for (var i = 0; i < 10; ++i)
{
Console.WriteLine("Pass #" + (i + 1));
Console.Write("NFA: ");
Perf(prog, test);
Console.Write("NFA+DFA (optimized): ");
Perf(progOpt, test);
Console.Write("DFA: ");
Perf(progDfa, test);
}
}
static void Main()
{
var test = "fubar bar 123 1foo bar -243 0 baz 83";
Console.WriteLine("Lex: " + test);
var prog = Lex.CompileLexerRegex(false,
@"[A-Z_a-z][A-Z_a-z0-9]*", // id
@"0|(\-?[1-9][0-9]*)", // int
@"( |\t|\r|\n|\v|\f)" // space
);
Console.WriteLine("Unoptimized dump:");
Console.WriteLine(Lex.Disassemble(prog));
Console.WriteLine();
var progOpt = Lex.CompileLexerRegex(true,
@"[A-Z_a-z][A-Z_a-z0-9]*", // id
@"0|(\-?[1-9][0-9]*)", // int
@"( |\t|\r|\n|\v|\f)" // space
);
Console.WriteLine("Optimized dump:");
Console.WriteLine(Lex.Disassemble(progOpt));
Console.WriteLine();
var progDfa = Lex.AssembleFrom(@"..\..\dfa.lasm");
Console.WriteLine("DFA dump:");
Console.WriteLine(Lex.Disassemble(progDfa));
Console.WriteLine();
var nfa = FA.ToLexer(new FA[]
{
FA.Parse(@"[A-Z_a-z][A-Z_a-z0-9]*", 0),
FA.Parse(@"0|(\-?[1-9][0-9]*)", 1),
FA.Parse(@"( |\t|\r|\n|\v|\f)", 2)
});
var dfa = nfa.ToDfa();
dfa.TrimDuplicates();
var dfaTable = dfa.ToDfaStateTable();
var result = -1;
var count = 0f;
var maxFiberCount = 0;
var avgCharPasses = 0f;
LexContext lc = LexContext.Create(test);
while (LexContext.EndOfInput != lc.Current)
{
var stats = Lex.RunWithLoggingAndStatistics(prog, lc, TextWriter.Null, out result);
maxFiberCount = stats.MaxFiberCount;
if (stats.AverageCharacterPasses > avgCharPasses)
{
avgCharPasses = stats.AverageCharacterPasses;
}
++count;
}
Console.WriteLine("NFA ran with " + maxFiberCount + " max fibers and " + avgCharPasses + " average char passes");
count = 0f;
maxFiberCount = 0;
avgCharPasses = 0f;
count = 0;
lc = LexContext.Create(test);
while (LexContext.EndOfInput != lc.Current)
{
var stats = Lex.RunWithLoggingAndStatistics(progOpt, lc, TextWriter.Null, out result);
maxFiberCount = stats.MaxFiberCount;
if (stats.AverageCharacterPasses > avgCharPasses)
{
avgCharPasses = stats.AverageCharacterPasses;
}
++count;
}
Console.WriteLine("NFA+DFA (optimized) ran with " + maxFiberCount + " max fibers and " + avgCharPasses + " average char passes");
count = 0;
maxFiberCount = 0;
avgCharPasses = 0f;
lc = LexContext.Create(test);
while (LexContext.EndOfInput != lc.Current)
{
var stats = Lex.RunWithLoggingAndStatistics(progDfa, lc, TextWriter.Null, out result);
maxFiberCount = stats.MaxFiberCount;
if (stats.AverageCharacterPasses > avgCharPasses)
{
avgCharPasses = stats.AverageCharacterPasses;
}
++count;
}
Console.WriteLine("DFA ran with " + maxFiberCount + " max fibers and " + avgCharPasses + " average char passes");
#if RUNPERF
for (var i = 0; i < 5; ++i)
{
test = string.Concat(test, test);
}
for (var i = 0; i < 10; ++i)
{
Console.WriteLine("Pass #" + (i + 1));
Console.Write("NFA: ");
_Perf(prog, test);
Console.WriteLine();
Console.Write("NFA+DFA (optimized): ");
_Perf(progOpt, test);
Console.WriteLine();
Console.Write("DFA: ");
_Perf(progDfa, test);
Console.WriteLine();
Console.Write("NFA (raw): ");
_Perf(nfa, test);
Console.WriteLine();
Console.Write("DFA (raw): ");
_Perf(dfaTable, test);
Console.WriteLine();
Console.WriteLine();
}
Console.WriteLine();
#endif
_RunLexer(progOpt);
Console.Error.WriteLine();
}