private async Task<TokenList> TokenizeAsync(ITextProvider text, IParsingExecutionContext context)
{
var stream = new TextStream(text);
var tokens = await Lexer.TokenizeAsync(stream, context);
LastTokenizationDuration = Lexer.LastTokenizationDuration;
return tokens;
}
private TokenList Tokenize(IParsingRequest request)
{
var stream = new TextStream(request.Text);
return Lexer.Tokenize(stream, new ParsingRequestLexingContext(request));
}