protected TokenizerBackedParser(LanguageCharacteristics <TTokenizer> language, ParserContext context)
: base(context)
{
Language = language;
var languageTokenizer = Language.CreateTokenizer(Context.Source);
_tokenizer = new TokenizerView <TTokenizer>(languageTokenizer);
SpanContext = new SpanContextBuilder();
}
protected void InitializeContext(SpanContextBuilder spanContext)
{
SpanContextConfig?.Invoke(spanContext);
}
private void CommentSpanContextConfig(SpanContextBuilder spanContext)
{
spanContext.ChunkGenerator = SpanChunkGenerator.Null;
spanContext.EditHandler = SpanEditHandler.CreateDefault(Language.TokenizeString);
}
