public static TokenList DefaultTokenList(string input)
{
Tokenizer tokenizer = new Tokenizer(new StringReader(input));
tokenizer.AddRegexRule(new Regex("^\'([^\"\n])*\'"), m => new StringToken(m.Value.Substring(1, m.Value.Length - 2)));
tokenizer.AddRegexRule(new Regex("^\"([^\"\n])*\""), m => new StringToken(m.Value.Substring(1, m.Value.Length - 2)));
tokenizer.AddDefaultRuleForInteger();
tokenizer.AddDefaultRuleForFloat();
tokenizer.AddDefaultRuleForIdentifier();
tokenizer.AddSymbolRule("==");
tokenizer.AddSpecialRule(c =>
{
if (char.IsSymbol(c) || char.IsPunctuation(c))
{
return(new SymbolToken(c));
}
return(null);
});
tokenizer.AddRegexRule(new Regex(@"^\/\/.*\n"), m => new CommentToken());
tokenizer.ignore = c => char.IsWhiteSpace(c);
TokenList list = new TokenList(tokenizer);
return(list);
}
public static TokenList TokensFor(string input)
{
Tokenizer tokenizer = new Tokenizer(new StringReader(input));
tokenizer.AddDefaultRuleForInteger();
tokenizer.AddDefaultRuleForIdentifier();
tokenizer.AddSpecialRule(c =>
{
if (char.IsSymbol(c) || char.IsPunctuation(c))
{
return(new SymbolToken(c));
}
return(null);
});
tokenizer.ignore = c => char.IsWhiteSpace(c);
return(new TokenList(tokenizer));
}