public async Task CrawlAsync(SitePage sitePage)
{
try
{
OnPageDownloadBegin?.Invoke(this, sitePage);
await Downloader.Execute(sitePage);
OnPageDownloadComplete?.Invoke(this, sitePage);
SitePageTemplateExtractor.FindAll(sitePage.Uri, sitePage.Contents, _filters)
.ForEach(async i =>
{
Interlocked.Increment(ref _numberOfLinksLeft);
await CrawlAsync(i);
});
}
catch (Exception exc)
{
OnError?.Invoke(this, exc);
}
if (Interlocked.Decrement(ref _numberOfLinksLeft) == 0)
{
OnDownloadCompleted?.Invoke(this, EventArgs.Empty);
}
}
public static async Task Execute(SitePage page)
{
page.Contents = await Download(page.Uri);
if (!page.MediaUrl.IsEmpty)
{
await Task.WhenAll(page.MediaUrl.Select(async x =>
{
var content = await Download(x.Key);
page.MediaUrl.AddOrUpdate(x.Key, content, (k, v) => content);
}));
}
}
