private async Task Crawl(Uri uri) { downloading.Add(uri); var resource = await browser.DownloadPage(uri); downloading.Remove(uri); completed.Add(uri); if (resource is IPage page) { IEnumerable <Uri> newUris; lock (visitedAddLock) { newUris = page.Uris .Where(IsValidUri) .Select(AbsoluteUri) .Except(visited) .Distinct() .ToList(); foreach (var u in newUris) { visited.Add(u); } } var tasks = newUris .Select(u => Crawl(AbsoluteUri(u))) .Append(storage.Save(startUri.MakeRelativeUri(page.Uri), page.Content)); await Task.WhenAll(tasks); } else if (resource is IFile file) { await storage.Save(startUri.MakeRelativeUri(file.Uri), file.Content); } }