public List <Token> Tokenize(Action <int, string> onError)
{
List <Token> tokens = new List <Token>();
do
{
Token next = NextToken(onError);
if (next.Type != TokenType.Skip)
{
tokens.Add(next);
}
} while (!stream.IsEndOfStream());
tokens.Add(new Token(TokenType.EOF));
return(tokens);
}