private void MatchInside(Tokenizer tokenizer) { var list = tokenizer.Source.Elements(); foreach(var node in list) { var t = new Tokenizer(node, tokenizer.Results, tokenizer.Matchers.ToArray()); t.Parse(); } }
public IEnumerable<Token> Lex() { List < IMatcher > matchers = InitializeMatchList(); List<Token> results = new List<Token>(); Tokenizer tokenizer = new Tokenizer(_source, results, matchers.ToArray()); tokenizer.Parse(); return results; }
public override void Match(Tokenizer tokenizer) { tokenizer.Results.Add(new Token(_pattern.TokenType, tokenizer.Source)); MatchInside(tokenizer); tokenizer.Results.Add(new Token(TokenType.EOF)); }