public IEnumerable<Token> Tokenize(string input)
{
var state = new LexerState(input);
var grammar = new HtmlGrammar(state);
state.Put(new Token(TokenCategory.StartDocument, 0, 0));
grammar.Document();
state.Put(new Token(TokenCategory.EndDocument, input.Length, input.Length));
return state.Tokens;
}
internal CommonGrammar(LexerState lexerState)
{
_lexerState = lexerState;
}
public HandlebarsGrammar(LexerState lexerState)
{
_lexerState = lexerState;
_commonGrammar = new CommonGrammar(lexerState);
}
