protected TokenizerBackedParser(LanguageCharacteristics <TTokenizer> language, ParserContext context)
: base(context)
{
Language = language;
LanguageTokenizeString = Language.TokenizeString;
var languageTokenizer = Language.CreateTokenizer(Context.Source);
_tokenizer = new TokenizerView <TTokenizer>(languageTokenizer);
SpanContext = new SpanContextBuilder();
}
private TokenizerView <TTokenizer, TSymbol, TSymbolType> InitTokenizer()
{
return(_tokenizer = new TokenizerView <TTokenizer, TSymbol, TSymbolType>(Language.CreateTokenizer(Context.Source)));
}
