//TODO Fix Me
private static async Task <IEnumerable <DownloadResult> > GetDownloadables(string url, string extension, int levels)
{
var downloadResults = new ConcurrentBag <DownloadResult>();
var results = GetDownloadables(url, extension).Result.ToList();
downloadResults.AddRange(results);
var tempResults = new ConcurrentBag <DownloadResult>();
tempResults.AddRange(results);
for (int i = 0; i < levels; i++)
{
Parallel.ForEach(results.Where(r => r.IsDirectory),
//new ParallelOptions() { MaxDegreeOfParallelism = 1 },
async directory =>
{
tempResults.AddRange(await GetDownloadables(directory.Url, extension));
});
Console.WriteLine("Level {0}, {1} urls added", i, tempResults.Count);
downloadResults.AddRange(tempResults);
Console.WriteLine("Level {0}, {1} urls total", i, downloadResults.Count);
results = tempResults.Clone();
tempResults = new ConcurrentBag <DownloadResult>();
}
return(downloadResults.Distinct().OrderBy(x => x.Url));
}
