public static TokenList DefaultTokenList(string input)
{
Tokenizer tokenizer = new Tokenizer(new StringReader(input));
tokenizer.AddRegexRule(new Regex("^\'([^\"\n])*\'"), m => new StringToken(m.Value.Substring(1, m.Value.Length - 2)));
tokenizer.AddRegexRule(new Regex("^\"([^\"\n])*\""), m => new StringToken(m.Value.Substring(1, m.Value.Length - 2)));
tokenizer.AddDefaultRuleForInteger();
tokenizer.AddDefaultRuleForFloat();
tokenizer.AddDefaultRuleForIdentifier();
tokenizer.AddSymbolRule("==");
tokenizer.AddSpecialRule(c =>
{
if (char.IsSymbol(c) || char.IsPunctuation(c))
{
return(new SymbolToken(c));
}
return(null);
});
tokenizer.AddRegexRule(new Regex(@"^\/\/.*\n"), m => new CommentToken());
tokenizer.ignore = c => char.IsWhiteSpace(c);
TokenList list = new TokenList(tokenizer);
return(list);
}
public static TokenList TokensFor(string input)
{
Tokenizer tokenizer = new Tokenizer(new StringReader(input));
tokenizer.AddDefaultRuleForInteger();
tokenizer.AddDefaultRuleForIdentifier();
tokenizer.AddSpecialRule(c =>
{
if (char.IsSymbol(c) || char.IsPunctuation(c))
{
return(new SymbolToken(c));
}
return(null);
});
tokenizer.ignore = c => char.IsWhiteSpace(c);
return(new TokenList(tokenizer));
}
public static TokenList DefaultTokenList(string input)
{
Tokenizer tokenizer = new Tokenizer(new StringReader(input));
tokenizer.rules.Add(new Regex(@"^([0-9]+)"), m => new IntToken(int.Parse(m.Value)));
tokenizer.rules.Add(new Regex(@"^([0-9]+(?:\.[0-9]+)?)"), m => new FloatToken(float.Parse(m.Value.Replace('.', ','))));
tokenizer.AddSpecialRule(c =>
{
if (char.IsSymbol(c) || char.IsPunctuation(c))
{
return(new SymbolToken(c));
}
return(null);
});
tokenizer.ignore = c => char.IsWhiteSpace(c);
return(new TokenList(tokenizer));
}