private async Task Crawl(Uri uri)
{
downloading.Add(uri);
var resource = await browser.DownloadPage(uri);
downloading.Remove(uri);
completed.Add(uri);
if (resource is IPage page)
{
IEnumerable <Uri> newUris;
lock (visitedAddLock)
{
newUris = page.Uris
.Where(IsValidUri)
.Select(AbsoluteUri)
.Except(visited)
.Distinct()
.ToList();
foreach (var u in newUris)
{
visited.Add(u);
}
}
var tasks = newUris
.Select(u => Crawl(AbsoluteUri(u)))
.Append(storage.Save(startUri.MakeRelativeUri(page.Uri), page.Content));
await Task.WhenAll(tasks);
}
else if (resource is IFile file)
{
await storage.Save(startUri.MakeRelativeUri(file.Uri), file.Content);
}
}
