static void _RunLexer()
{
var digits = CharFA <string> .Repeat(
CharFA <string> .Set("0123456789"),
1, -1
, "Digits");
var word = CharFA <string> .Repeat(
CharFA <string> .Set(new CharRange[] { new CharRange('A', 'Z'), new CharRange('a', 'z') }),
1, -1
, "Word");
var whitespace = CharFA <string> .Repeat(
CharFA <string> .Set(" \t\r\n\v\f"),
1, -1
, "Whitespace");
var lexer = CharFA <string> .ToLexer(digits, word, whitespace);
var lexerDfa = lexer.ToDfa();
lexerDfa.TrimDuplicates();
// we use a symbol table with the DFA state table to map ids back to strings
var symbolTable = new string[] { "Digits", "Word", "Whitespace", "#ERROR" };
// make sure to pass the symbol table if you're using one
var dfaTable = lexer.ToDfaStateTable(symbolTable);
var test = "foo123_ _bar";
Console.WriteLine("Lex using the NFA");
// create a parse context over our test string
var pc = ParseContext.Create(test);
// while not end of input
while (-1 != pc.Current)
{
// clear the capture so that we don't keep appending the token data
pc.ClearCapture();
// lex the next token
var acc = lexer.Lex(pc, "#ERROR");
// write the result
Console.WriteLine("{0}: {1}", acc, pc.GetCapture());
}
Console.WriteLine();
Console.WriteLine("Lex using the DFA");
// create a new parse context over our test string
// because our old parse context is now past the end
pc = ParseContext.Create(test);
while (-1 != pc.Current)
{
pc.ClearCapture();
// lex using the DFA. This works exactly like
// the previous Lex method except that it's
// optimized for DFA traversal.
// DO NOT use this with an NFA. It won't work
// but won't error (can't check for perf reasons)
var acc = lexerDfa.LexDfa(pc, "#ERROR");
// write the result
Console.WriteLine("{0}: {1}", acc, pc.GetCapture());
}
Console.WriteLine();
Console.WriteLine("Lex using the DFA state table");
pc = ParseContext.Create(test);
while (-1 != pc.Current)
{
pc.ClearCapture();
// Lex using our DFA table. This is a little different
// because it's a static method that takes CharDfaEntry[]
// as its first parameter. It also uses symbol ids instead
// of the actual symbol. You must map them back using the
// symbol table you created earlier.
var acc = CharFA <string> .LexDfa(dfaTable, pc, 3);
// when we write this, we map our symbol id back to the
// symbol using our symbol table
Console.WriteLine("{0}: {1}", symbolTable[acc], pc.GetCapture());
}
Console.WriteLine();
Console.WriteLine("Lex using our compiled lex method");
pc = ParseContext.Create(test);
while (-1 != pc.Current)
{
pc.ClearCapture();
// Lex using our compiledDFA. Like the table driven lex
// this also uses symbol ids instead of the actual symbol.
var acc = Lex(pc);
// when we write this, we map our symbol id back to the
// symbol using our symbol table
Console.WriteLine("{0}: {1}", symbolTable[acc], pc.GetCapture());
}
Console.WriteLine();
}
