private async Task <CrawledPage> MakeOneRequestAsync(Uri uri)
{
var perfWatch = Stopwatch.StartNew();
await lastCalls.WaitForCallAsync(uri)
.ConfigureAwait(false);
using (var contentExtractor = new WebContentExtractor())
{
using (var pageRequester = new PageRequester(new CrawlConfiguration(), contentExtractor))
{
var page = await pageRequester.MakeRequestAsync(uri).ConfigureAwait(false);
if (page.HttpRequestException != null)
{
throw new InvalidOperationException("HTTP error.", page.HttpRequestException);
}
logger.LogDebug("Request to {Url} took {Elapsed}", uri, perfWatch.Elapsed);
return(page);
}
}
}
public void Setup()
{
_uut = new WebContentExtractor();
}
