private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct) { while (true) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } Monitor.Enter(_lockObject); if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count) { IEnumerable <QueueListItem> queueList = QueueManager.Items.Except(_crawlerService.ActiveItems); QueueListItem nextQueueItem = queueList.First(); IBlog blog = nextQueueItem.Blog; ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct); crawler.IsBlogOnlineAsync().Wait(4000); crawler.Dispose(); if (_crawlerService.ActiveItems.Any(item => item.Blog.Name.Equals(nextQueueItem.Blog.Name) && item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType))) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } if (!nextQueueItem.Blog.Online) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } _crawlerService.AddActiveItems(nextQueueItem); Monitor.Exit(_lockObject); await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct); } else { Monitor.Exit(_lockObject); await Task.Delay(4000, ct); } } }
private async Task RunCrawlerTasks(CancellationToken ct, PauseToken pt) { while (true) { ct.ThrowIfCancellationRequested(); if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(ct); } Monitor.Enter(lockObject); if (crawlerService.ActiveItems.Count() < QueueManager.Items.Count()) { IEnumerable <QueueListItem> queueList = QueueManager.Items.Except(crawlerService.ActiveItems); QueueListItem nextQueueItem = queueList.First(); IBlog blog = nextQueueItem.Blog; IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog); downloader.IsBlogOnlineAsync().Wait(4000); if (crawlerService.ActiveItems.Any(item => item.Blog.Name.Contains(nextQueueItem.Blog.Name))) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(lockObject); continue; } if (!nextQueueItem.Blog.Online) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(lockObject); continue; } crawlerService.AddActiveItems(nextQueueItem); Monitor.Exit(lockObject); await StartSiteSpecificDownloader(nextQueueItem, ct, pt); } else { Monitor.Exit(lockObject); await Task.Delay(4000, ct); } } }
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct) { while (true) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } bool lockTaken = false; Monitor.Enter(_lockObject, ref lockTaken); try { if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count) { QueueListItem nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First(); IBlog blog = nextQueueItem.Blog; ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct); crawler.IsBlogOnlineAsync().Wait(4000); crawler.Dispose(); if (_crawlerService.ActiveItems.Any(item => item.Blog.Name.Equals(nextQueueItem.Blog.Name) && item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType))) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } if (!nextQueueItem.Blog.Online) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } _crawlerService.AddActiveItems(nextQueueItem); Monitor.Exit(_lockObject); lockTaken = false; await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct); } else { Monitor.Exit(_lockObject); lockTaken = false; await Task.Delay(4000, ct); } } catch (Exception e) { Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e); _shellService.ShowError(e, "Error starting the next item in the queue."); if (lockTaken) { Monitor.Exit(_lockObject); } } } }
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct) { while (true) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } bool lockTaken = false; Monitor.Enter(_lockObject, ref lockTaken); try { if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count) { QueueListItem nextQueueItem; try { nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First(); } catch (InvalidOperationException) { Monitor.Exit(_lockObject); continue; } IBlog blog = nextQueueItem.Blog; var privacyConsentNeeded = false; ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct); try { crawler.IsBlogOnlineAsync().Wait(4000); } catch (AggregateException ex) { if (ex.InnerExceptions.Any(x => x.Message == "Acceptance of privacy consent needed!")) { privacyConsentNeeded = true; } } finally { crawler.Dispose(); } if (privacyConsentNeeded || (_crawlerService.ActiveItems.Any(item => item.Blog.Name.Equals(nextQueueItem.Blog.Name) && item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType))) || (!nextQueueItem.Blog.Online)) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } _crawlerService.AddActiveItems(nextQueueItem); Monitor.Exit(_lockObject); lockTaken = false; await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct); } else { Monitor.Exit(_lockObject); lockTaken = false; await Task.Delay(4000, ct); } } catch (Exception e) { if (!ct.IsCancellationRequested) { Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e); } if (lockTaken) { Monitor.Exit(_lockObject); } } } }