Ejemplo n.º 1
0
        private async Task DownloadTitles(List <Uri> AllTitlesUrls)
        {
            var urlsForTitles = AllTitlesUrls.Take(TAKE_COUNT);
            int skipTitles    = TAKE_COUNT;

            var downloadTasks = urlsForTitles.Select(link => WebPageDownloader.Download(link)).ToList();

            while (downloadTasks.Any())
            {
                Task <WebPage> _task = null;
                try
                {
                    _task = await Task.WhenAny(downloadTasks);
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"{DateTime.Now} ||Error Message:: {ex.Message}");
                    await Task.Delay(DELAY_IF_EXCEPTION);

                    _task.Start();
                }
                finally
                {
                    Titles.Add(TitleBuilder.Get(_task.Result.Html, _task.Result.Url));
                    Console.WriteLine($"{DateTime.Now} ||Добавлен новый тайтл. ВСЕГО:{Titles.Count}");
                    downloadTasks.Remove(_task);

                    await Task.Delay(DELAY);

                    var newTask = AllTitlesUrls.Skip(skipTitles).Take(1).Select(link => WebPageDownloader.Download(link));
                    skipTitles += 1;
                    downloadTasks.AddRange(newTask);
                }
            }
        }
Ejemplo n.º 2
0
        private const int DELAY_IF_EXCEPTION = 10000; // 10 seconds
        private async Task <List <Uri> > GetTitlesUrlsFromCatPages(List <Uri> catUrls)
        {
            List <Uri> urlsForTitles = new List <Uri>();
            var        catPagesUrls  = catUrls.Take(TAKE_COUNT);
            int        skipTitles    = TAKE_COUNT;

            var downloadTasks = catPagesUrls.Select(link => WebPageDownloader.Download(link)).ToList();

            while (downloadTasks.Any())
            {
                Task <WebPage> _task = null;
                try
                {
                    _task = await Task.WhenAny(downloadTasks);
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"{DateTime.Now} ||Error Message:: {ex.Message}");
                    await Task.Delay(DELAY_IF_EXCEPTION);

                    _task.Start();
                }
                finally
                {
                    urlsForTitles.AddRange(GetTitlesUrl(_task.Result));
                    Console.WriteLine($"{DateTime.Now} ||CatPages {_task.Result.Id} Downloaded");
                    downloadTasks.Remove(_task);

                    await Task.Delay(DELAY);

                    var newTask = catUrls.Skip(skipTitles).Take(1).Select(link => WebPageDownloader.Download(link));
                    skipTitles += 1;
                    downloadTasks.AddRange(newTask);
                }
            }

            return(urlsForTitles);
        }