示例#1
0
        public TokenStream Tokenize(String input)
        {
            input = PreProcessInput(input);

            var stream = new TokenStream();

            var tokens = lexer.Tokenize(input) as List<Token>;
            //Ignore all whitespace
            tokens.Where(t => t.Type != TokenType.Whitespace).ToList()
                .ForEach(t => stream.Add(t));

            return stream;
        }
示例#2
0
 public Parser(TokenStream tokens)
 {
     tokenStream = tokens;
 }