public abstract Token tokenize(Tokenizer tokenizer);
public override bool tokenizable(Tokenizer t)
{
return(t.input.peek() == '"');
}
public abstract bool tokenizable(Tokenizer tokenizer);
public override bool tokenizable(Tokenizer t)
{
return(isLetter(t.input));
}
public override bool tokenizable(Tokenizer t)
{
return(this.symbol == t.input.peek());
}
public override bool tokenizable(Tokenizer t)
{
return(String.Concat(t.input.peek(), t.input.peek(2)) == Environment.NewLine || t.input.peek().ToString() == Environment.NewLine);
}
public override Token tokenize(Tokenizer t)
{
return(new Token(t.input.Position, t.input.LineNumber,
"whitespace", t.input.loop(isWhiteSpace)));
}
public override bool tokenizable(Tokenizer t)
{
return(Char.IsWhiteSpace(t.input.peek()));
}
