public async Task <CrawlResult> StartCrawlingAsync(List <string> urls, int currentDepth)
{
Dictionary <string, CrawlResult> crawlResultList = new Dictionary <string, CrawlResult>();
foreach (string url in urls)
{
Console.WriteLine(writeWithTabs(currentDepth, url));
CrawlResult currentUrlCrawlResult = null;
if (currentDepth < maxDepth)
{
List <string> parsedUrls = HTMLParser.GetHref(url);
if (parsedUrls != null)
{
currentUrlCrawlResult = await StartCrawlingAsync(parsedUrls, currentDepth + 1);
}
if ((currentUrlCrawlResult != null) && (!crawlResultList.ContainsKey(url)))
{
crawlResultList.Add(url, currentUrlCrawlResult);
}
;
}
}
CrawlResult crawlResult = new CrawlResult();
crawlResult.InnerUrls = crawlResultList;
return(crawlResult);
}
private async static void nuGo()
{
try
{
List <string> listhref = HTMLParser.GetHref("http://www.tut.by");
Console.WriteLine(listhref.Count);
WebCrawler wc = new WebCrawler(2);
CrawlResult cr = new CrawlResult();
cr = await wc.StartCrawlingAsync(listhref, 1);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
