static void Dump(SimpleTokenReader tokenReader) { Console.WriteLine("text\ttype"); Token token = null; while ((token = tokenReader.Read()) != null) { Console.WriteLine(token.GetText() + "\t\t" + token.GetType()); } }
private static void TestLexer() { SimpleLexer lexer = new SimpleLexer(); string script = "int age = 45;"; SimpleTokenReader tokenReader = lexer.Tokenize(script); Dump(tokenReader); script = "inta age = 45;"; tokenReader = lexer.Tokenize(script); Dump(tokenReader); Console.ReadKey(); }