private void MatchInside(Tokenizer tokenizer)
{
var list = tokenizer.Source.Elements();
foreach(var node in list) {
var t = new Tokenizer(node, tokenizer.Results, tokenizer.Matchers.ToArray());
t.Parse();
}
}
public IEnumerable<Token> Lex()
{
List < IMatcher > matchers = InitializeMatchList();
List<Token> results = new List<Token>();
Tokenizer tokenizer = new Tokenizer(_source, results, matchers.ToArray());
tokenizer.Parse();
return results;
}
public override void Match(Tokenizer tokenizer)
{
tokenizer.Results.Add(new Token(_pattern.TokenType, tokenizer.Source));
MatchInside(tokenizer);
tokenizer.Results.Add(new Token(TokenType.EOF));
}
