private async Task <CrawlResult> AsyncCrawlUrls(string[] urlsToCrawl) { Dictionary <string, CrawlResult> crawledUrls = new Dictionary <string, CrawlResult>(); foreach (string url in urlsToCrawl) { string[] nestedUrls = await LoadPageAndExtractUniqueUrls(url); if (nestedUrls != null) { CrawlResult nestedResult = await CrawlNestedUrls(0, nestedUrls); crawledUrls.Add(url, nestedResult); } } return(new CrawlResult(crawledUrls)); }
private async Task <CrawlResult> CrawlNestedUrls(int currentDepth, string[] urlsToCrawl) { Dictionary <string, CrawlResult> crawledUrls = null; if (currentDepth <= _crawlDepth) { crawledUrls = new Dictionary <string, CrawlResult>(); foreach (string url in urlsToCrawl) { string[] nestedUrls = await LoadPageAndExtractUniqueUrls(url); if (nestedUrls != null) { CrawlResult nestedResult = await CrawlNestedUrls(currentDepth + 1, nestedUrls); crawledUrls.Add(url, nestedResult); } } } return(new CrawlResult(crawledUrls)); }