C# (CSharp) Migraine.Core TokenStream Examples

Example #1

File: MigraineLexer.cs Project: marco-fiset/migraine

        public TokenStream Tokenize(String input)
        {
            input = PreProcessInput(input);

            var stream = new TokenStream();

            var tokens = lexer.Tokenize(input) as List<Token>;
            //Ignore all whitespace
            tokens.Where(t => t.Type != TokenType.Whitespace).ToList()
                .ForEach(t => stream.Add(t));

            return stream;
        }

Example #2

File: Parser.cs Project: marco-fiset/migraine

 public Parser(TokenStream tokens)
 {
     tokenStream = tokens;
 }