public static TokenList DefaultTokenList(string input) { Tokenizer tokenizer = new Tokenizer(new StringReader(input)); tokenizer.AddRegexRule(new Regex("^\'([^\"\n])*\'"), m => new StringToken(m.Value.Substring(1, m.Value.Length - 2))); tokenizer.AddRegexRule(new Regex("^\"([^\"\n])*\""), m => new StringToken(m.Value.Substring(1, m.Value.Length - 2))); tokenizer.AddDefaultRuleForInteger(); tokenizer.AddDefaultRuleForFloat(); tokenizer.AddDefaultRuleForIdentifier(); tokenizer.AddSymbolRule("=="); tokenizer.AddSpecialRule(c => { if (char.IsSymbol(c) || char.IsPunctuation(c)) { return(new SymbolToken(c)); } return(null); }); tokenizer.AddRegexRule(new Regex(@"^\/\/.*\n"), m => new CommentToken()); tokenizer.ignore = c => char.IsWhiteSpace(c); TokenList list = new TokenList(tokenizer); return(list); }