private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
{
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
Monitor.Enter(_lockObject);
if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count)
{
IEnumerable <QueueListItem> queueList = QueueManager.Items.Except(_crawlerService.ActiveItems);
QueueListItem nextQueueItem = queueList.First();
IBlog blog = nextQueueItem.Blog;
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct);
crawler.IsBlogOnlineAsync().Wait(4000);
crawler.Dispose();
if (_crawlerService.ActiveItems.Any(item =>
item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType)))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
if (!nextQueueItem.Blog.Online)
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
_crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(_lockObject);
await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct);
}
else
{
Monitor.Exit(_lockObject);
await Task.Delay(4000, ct);
}
}
}
private async Task RunCrawlerTasks(CancellationToken ct, PauseToken pt)
{
while (true)
{
ct.ThrowIfCancellationRequested();
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait(ct);
}
Monitor.Enter(lockObject);
if (crawlerService.ActiveItems.Count() < QueueManager.Items.Count())
{
IEnumerable <QueueListItem> queueList = QueueManager.Items.Except(crawlerService.ActiveItems);
QueueListItem nextQueueItem = queueList.First();
IBlog blog = nextQueueItem.Blog;
IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog);
downloader.IsBlogOnlineAsync().Wait(4000);
if (crawlerService.ActiveItems.Any(item => item.Blog.Name.Contains(nextQueueItem.Blog.Name)))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(lockObject);
continue;
}
if (!nextQueueItem.Blog.Online)
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(lockObject);
continue;
}
crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(lockObject);
await StartSiteSpecificDownloader(nextQueueItem, ct, pt);
}
else
{
Monitor.Exit(lockObject);
await Task.Delay(4000, ct);
}
}
}
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
{
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
bool lockTaken = false;
Monitor.Enter(_lockObject, ref lockTaken);
try
{
if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count)
{
QueueListItem nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First();
IBlog blog = nextQueueItem.Blog;
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct);
crawler.IsBlogOnlineAsync().Wait(4000);
crawler.Dispose();
if (_crawlerService.ActiveItems.Any(item =>
item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType)))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
if (!nextQueueItem.Blog.Online)
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
_crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(_lockObject);
lockTaken = false;
await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct);
}
else
{
Monitor.Exit(_lockObject);
lockTaken = false;
await Task.Delay(4000, ct);
}
}
catch (Exception e)
{
Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e);
_shellService.ShowError(e, "Error starting the next item in the queue.");
if (lockTaken)
{
Monitor.Exit(_lockObject);
}
}
}
}
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
{
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
bool lockTaken = false;
Monitor.Enter(_lockObject, ref lockTaken);
try
{
if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count)
{
QueueListItem nextQueueItem;
try
{
nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First();
}
catch (InvalidOperationException)
{
Monitor.Exit(_lockObject);
continue;
}
IBlog blog = nextQueueItem.Blog;
var privacyConsentNeeded = false;
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct);
try
{
crawler.IsBlogOnlineAsync().Wait(4000);
}
catch (AggregateException ex)
{
if (ex.InnerExceptions.Any(x => x.Message == "Acceptance of privacy consent needed!"))
{
privacyConsentNeeded = true;
}
}
finally
{
crawler.Dispose();
}
if (privacyConsentNeeded ||
(_crawlerService.ActiveItems.Any(item =>
item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType))) ||
(!nextQueueItem.Blog.Online))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
_crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(_lockObject);
lockTaken = false;
await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct);
}
else
{
Monitor.Exit(_lockObject);
lockTaken = false;
await Task.Delay(4000, ct);
}
}
catch (Exception e)
{
if (!ct.IsCancellationRequested)
{
Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e);
}
if (lockTaken)
{
Monitor.Exit(_lockObject);
}
}
}
}