private async Task DownloadTitles(List <Uri> AllTitlesUrls) { var urlsForTitles = AllTitlesUrls.Take(TAKE_COUNT); int skipTitles = TAKE_COUNT; var downloadTasks = urlsForTitles.Select(link => WebPageDownloader.Download(link)).ToList(); while (downloadTasks.Any()) { Task <WebPage> _task = null; try { _task = await Task.WhenAny(downloadTasks); } catch (Exception ex) { Console.WriteLine($"{DateTime.Now} ||Error Message:: {ex.Message}"); await Task.Delay(DELAY_IF_EXCEPTION); _task.Start(); } finally { Titles.Add(TitleBuilder.Get(_task.Result.Html, _task.Result.Url)); Console.WriteLine($"{DateTime.Now} ||Добавлен новый тайтл. ВСЕГО:{Titles.Count}"); downloadTasks.Remove(_task); await Task.Delay(DELAY); var newTask = AllTitlesUrls.Skip(skipTitles).Take(1).Select(link => WebPageDownloader.Download(link)); skipTitles += 1; downloadTasks.AddRange(newTask); } } }
private const int DELAY_IF_EXCEPTION = 10000; // 10 seconds private async Task <List <Uri> > GetTitlesUrlsFromCatPages(List <Uri> catUrls) { List <Uri> urlsForTitles = new List <Uri>(); var catPagesUrls = catUrls.Take(TAKE_COUNT); int skipTitles = TAKE_COUNT; var downloadTasks = catPagesUrls.Select(link => WebPageDownloader.Download(link)).ToList(); while (downloadTasks.Any()) { Task <WebPage> _task = null; try { _task = await Task.WhenAny(downloadTasks); } catch (Exception ex) { Console.WriteLine($"{DateTime.Now} ||Error Message:: {ex.Message}"); await Task.Delay(DELAY_IF_EXCEPTION); _task.Start(); } finally { urlsForTitles.AddRange(GetTitlesUrl(_task.Result)); Console.WriteLine($"{DateTime.Now} ||CatPages {_task.Result.Id} Downloaded"); downloadTasks.Remove(_task); await Task.Delay(DELAY); var newTask = catUrls.Skip(skipTitles).Take(1).Select(link => WebPageDownloader.Download(link)); skipTitles += 1; downloadTasks.AddRange(newTask); } } return(urlsForTitles); }