public IEnumerable<Token> Tokenize(string input) { var state = new LexerState(input); var grammar = new HtmlGrammar(state); state.Put(new Token(TokenCategory.StartDocument, 0, 0)); grammar.Document(); state.Put(new Token(TokenCategory.EndDocument, input.Length, input.Length)); return state.Tokens; }
internal CommonGrammar(LexerState lexerState) { _lexerState = lexerState; }
public HandlebarsGrammar(LexerState lexerState) { _lexerState = lexerState; _commonGrammar = new CommonGrammar(lexerState); }