public async Task CrawlAsync(SitePage sitePage) { try { OnPageDownloadBegin?.Invoke(this, sitePage); await Downloader.Execute(sitePage); OnPageDownloadComplete?.Invoke(this, sitePage); SitePageTemplateExtractor.FindAll(sitePage.Uri, sitePage.Contents, _filters) .ForEach(async i => { Interlocked.Increment(ref _numberOfLinksLeft); await CrawlAsync(i); }); } catch (Exception exc) { OnError?.Invoke(this, exc); } if (Interlocked.Decrement(ref _numberOfLinksLeft) == 0) { OnDownloadCompleted?.Invoke(this, EventArgs.Empty); } }
public static async Task Execute(SitePage page) { page.Contents = await Download(page.Uri); if (!page.MediaUrl.IsEmpty) { await Task.WhenAll(page.MediaUrl.Select(async x => { var content = await Download(x.Key); page.MediaUrl.AddOrUpdate(x.Key, content, (k, v) => content); })); } }