public TokenStream Tokenize(String input)
{
input = PreProcessInput(input);
var stream = new TokenStream();
var tokens = lexer.Tokenize(input) as List<Token>;
//Ignore all whitespace
tokens.Where(t => t.Type != TokenType.Whitespace).ToList()
.ForEach(t => stream.Add(t));
return stream;
}
public Parser(TokenStream tokens)
{
tokenStream = tokens;
}