private async Task <CrawlResult> AsyncCrawlUrls(string[] urlsToCrawl)
{
Dictionary <string, CrawlResult> crawledUrls = new Dictionary <string, CrawlResult>();
foreach (string url in urlsToCrawl)
{
string[] nestedUrls = await LoadPageAndExtractUniqueUrls(url);
if (nestedUrls != null)
{
CrawlResult nestedResult = await CrawlNestedUrls(0, nestedUrls);
crawledUrls.Add(url, nestedResult);
}
}
return(new CrawlResult(crawledUrls));
}
private async Task <CrawlResult> CrawlNestedUrls(int currentDepth, string[] urlsToCrawl)
{
Dictionary <string, CrawlResult> crawledUrls = null;
if (currentDepth <= _crawlDepth)
{
crawledUrls = new Dictionary <string, CrawlResult>();
foreach (string url in urlsToCrawl)
{
string[] nestedUrls = await LoadPageAndExtractUniqueUrls(url);
if (nestedUrls != null)
{
CrawlResult nestedResult = await CrawlNestedUrls(currentDepth + 1, nestedUrls);
crawledUrls.Add(url, nestedResult);
}
}
}
return(new CrawlResult(crawledUrls));
}