private async Task DownloadTitles(List <Uri> AllTitlesUrls)
{
var urlsForTitles = AllTitlesUrls.Take(TAKE_COUNT);
int skipTitles = TAKE_COUNT;
var downloadTasks = urlsForTitles.Select(link => WebPageDownloader.Download(link)).ToList();
while (downloadTasks.Any())
{
Task <WebPage> _task = null;
try
{
_task = await Task.WhenAny(downloadTasks);
}
catch (Exception ex)
{
Console.WriteLine($"{DateTime.Now} ||Error Message:: {ex.Message}");
await Task.Delay(DELAY_IF_EXCEPTION);
_task.Start();
}
finally
{
Titles.Add(TitleBuilder.Get(_task.Result.Html, _task.Result.Url));
Console.WriteLine($"{DateTime.Now} ||Добавлен новый тайтл. ВСЕГО:{Titles.Count}");
downloadTasks.Remove(_task);
await Task.Delay(DELAY);
var newTask = AllTitlesUrls.Skip(skipTitles).Take(1).Select(link => WebPageDownloader.Download(link));
skipTitles += 1;
downloadTasks.AddRange(newTask);
}
}
}
private const int DELAY_IF_EXCEPTION = 10000; // 10 seconds
private async Task <List <Uri> > GetTitlesUrlsFromCatPages(List <Uri> catUrls)
{
List <Uri> urlsForTitles = new List <Uri>();
var catPagesUrls = catUrls.Take(TAKE_COUNT);
int skipTitles = TAKE_COUNT;
var downloadTasks = catPagesUrls.Select(link => WebPageDownloader.Download(link)).ToList();
while (downloadTasks.Any())
{
Task <WebPage> _task = null;
try
{
_task = await Task.WhenAny(downloadTasks);
}
catch (Exception ex)
{
Console.WriteLine($"{DateTime.Now} ||Error Message:: {ex.Message}");
await Task.Delay(DELAY_IF_EXCEPTION);
_task.Start();
}
finally
{
urlsForTitles.AddRange(GetTitlesUrl(_task.Result));
Console.WriteLine($"{DateTime.Now} ||CatPages {_task.Result.Id} Downloaded");
downloadTasks.Remove(_task);
await Task.Delay(DELAY);
var newTask = catUrls.Skip(skipTitles).Take(1).Select(link => WebPageDownloader.Download(link));
skipTitles += 1;
downloadTasks.AddRange(newTask);
}
}
return(urlsForTitles);
}
