private async Task CraulRootAsync(CraulResult result, int level)
{
if (level > depth)
{
return;
}
foreach (var rootUrl in result.Childs)
{
string htmlPage = "";
IEnumerable <string> newRootUrls = new List <string>();
WebClient webClient = new WebClient();
try
{
htmlPage = await webClient.DownloadStringTaskAsync(rootUrl.Url);
newRootUrls = await GetRootUrlsAsync(htmlPage);
AddChildsToResult(rootUrl, newRootUrls);
}
catch (Exception e)
{
errorList.Add(e);
}
await CraulRootAsync(rootUrl, level + 1);
}
}
private void AddChildsToResult(CraulResult result, IEnumerable <string> rootUrls)
{
foreach (string url in rootUrls)
{
if (url.StartsWith("http"))
{
result.Childs.Add(new CraulResult(url));
}
}
}
public Task <CraulResult> PerformCraulingAsync(IEnumerable <string> rootUrls)
{
return(Task <CraulResult> .Run(async() =>
{
CraulResult result = new CraulResult("root");
AddChildsToResult(result, rootUrls);
await CraulRootAsync(result, 0);
if (errorList.Count != 0)
{
EncounteredErrors = new AggregateException(errorList);
}
return result;
}
));
}