static void _RunLexer() { var digits = CharFA <string> .Repeat( CharFA <string> .Set("0123456789"), 1, -1 , "Digits"); var word = CharFA <string> .Repeat( CharFA <string> .Set(new CharRange[] { new CharRange('A', 'Z'), new CharRange('a', 'z') }), 1, -1 , "Word"); var whitespace = CharFA <string> .Repeat( CharFA <string> .Set(" \t\r\n\v\f"), 1, -1 , "Whitespace"); var lexer = CharFA <string> .ToLexer(digits, word, whitespace); var lexerDfa = lexer.ToDfa(); lexerDfa.TrimDuplicates(); // we use a symbol table with the DFA state table to map ids back to strings var symbolTable = new string[] { "Digits", "Word", "Whitespace", "#ERROR" }; // make sure to pass the symbol table if you're using one var dfaTable = lexer.ToDfaStateTable(symbolTable); var test = "foo123_ _bar"; Console.WriteLine("Lex using the NFA"); // create a parse context over our test string var pc = ParseContext.Create(test); // while not end of input while (-1 != pc.Current) { // clear the capture so that we don't keep appending the token data pc.ClearCapture(); // lex the next token var acc = lexer.Lex(pc, "#ERROR"); // write the result Console.WriteLine("{0}: {1}", acc, pc.GetCapture()); } Console.WriteLine(); Console.WriteLine("Lex using the DFA"); // create a new parse context over our test string // because our old parse context is now past the end pc = ParseContext.Create(test); while (-1 != pc.Current) { pc.ClearCapture(); // lex using the DFA. This works exactly like // the previous Lex method except that it's // optimized for DFA traversal. // DO NOT use this with an NFA. It won't work // but won't error (can't check for perf reasons) var acc = lexerDfa.LexDfa(pc, "#ERROR"); // write the result Console.WriteLine("{0}: {1}", acc, pc.GetCapture()); } Console.WriteLine(); Console.WriteLine("Lex using the DFA state table"); pc = ParseContext.Create(test); while (-1 != pc.Current) { pc.ClearCapture(); // Lex using our DFA table. This is a little different // because it's a static method that takes CharDfaEntry[] // as its first parameter. It also uses symbol ids instead // of the actual symbol. You must map them back using the // symbol table you created earlier. var acc = CharFA <string> .LexDfa(dfaTable, pc, 3); // when we write this, we map our symbol id back to the // symbol using our symbol table Console.WriteLine("{0}: {1}", symbolTable[acc], pc.GetCapture()); } Console.WriteLine(); Console.WriteLine("Lex using our compiled lex method"); pc = ParseContext.Create(test); while (-1 != pc.Current) { pc.ClearCapture(); // Lex using our compiledDFA. Like the table driven lex // this also uses symbol ids instead of the actual symbol. var acc = Lex(pc); // when we write this, we map our symbol id back to the // symbol using our symbol table Console.WriteLine("{0}: {1}", symbolTable[acc], pc.GetCapture()); } Console.WriteLine(); }