private async Task StartSiteSpecificDownloaderAsync(QueueListItem queueListItem, PauseToken pt, CancellationToken ct)
{
IBlog blog = queueListItem.Blog;
blog.Dirty = true;
ProgressThrottler <DownloadProgress> progress = SetupThrottledQueueListProgress(queueListItem);
ICrawler crawler = null;
try
{
crawler = _crawlerFactory.GetCrawler(blog, progress, pt, ct);
await crawler.CrawlAsync();
blog.UpdateProgress(true);
}
finally
{
crawler?.Dispose();
}
Monitor.Enter(_lockObject);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => _crawlerService.RemoveActiveItem(queueListItem));
Monitor.Exit(_lockObject);
if (!ct.IsCancellationRequested)
{
Monitor.Enter(_lockObject);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(queueListItem));
Monitor.Exit(_lockObject);
}
}
private async Task UpdateMetaInformationAsync(IBlog blog)
{
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), new PauseToken(),
new CancellationToken());
await crawler.UpdateMetaInformationAsync();
crawler.Dispose();
}
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
{
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
Monitor.Enter(_lockObject);
if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count)
{
IEnumerable <QueueListItem> queueList = QueueManager.Items.Except(_crawlerService.ActiveItems);
QueueListItem nextQueueItem = queueList.First();
IBlog blog = nextQueueItem.Blog;
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct);
crawler.IsBlogOnlineAsync().Wait(4000);
crawler.Dispose();
if (_crawlerService.ActiveItems.Any(item =>
item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType)))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
if (!nextQueueItem.Blog.Online)
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
_crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(_lockObject);
await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct);
}
else
{
Monitor.Exit(_lockObject);
await Task.Delay(4000, ct);
}
}
}
private async Task CheckStatusOfBlogsAsync(SemaphoreSlim semaphoreSlim, IBlog blog)
{
await semaphoreSlim.WaitAsync();
try
{
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), new PauseToken(),
new CancellationToken());
await crawler.IsBlogOnlineAsync();
crawler.Dispose();
}
finally
{
semaphoreSlim.Release();
}
}
private async Task StartSiteSpecificDownloaderAsync(QueueListItem queueListItem, PauseToken pt, CancellationToken ct)
{
IBlog blog = queueListItem.Blog;
blog.Dirty = true;
ProgressThrottler <DownloadProgress> progress = SetupThrottledQueueListProgress(queueListItem);
ICrawler crawler = null;
try
{
crawler = _crawlerFactory.GetCrawler(blog, progress, pt, ct);
queueListItem.InterruptionRequested += crawler.InterruptionRequestedEventHandler;
await crawler.CrawlAsync();
blog.UpdateProgress(false);
}
catch (Exception e)
{
if (!ct.IsCancellationRequested)
{
Logger.Error("CrawlerController.StartSiteSpecificDownloaderAsync: {0}", e);
}
}
finally
{
if (crawler != null)
{
queueListItem.InterruptionRequested -= crawler.InterruptionRequestedEventHandler;
}
crawler?.Dispose();
}
Monitor.Enter(_lockObject);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => _crawlerService.RemoveActiveItem(queueListItem));
Monitor.Exit(_lockObject);
if (!ct.IsCancellationRequested)
{
Monitor.Enter(_lockObject);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(queueListItem));
Monitor.Exit(_lockObject);
}
}
private async Task CheckStatusOfBlogsAsync(SemaphoreSlim semaphoreSlim, IBlog blog)
{
await semaphoreSlim.WaitAsync();
ICrawler crawler = null;
try
{
bool isHiddenTumblrBlog = false;
if (blog.BlogType == BlogTypes.tumblr)
{
isHiddenTumblrBlog = await _tumblrBlogDetector.IsHiddenTumblrBlogAsync(blog.Url);
}
if (isHiddenTumblrBlog)
{
blog.BlogType = BlogTypes.tmblrpriv;
}
crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), new PauseToken(), new CancellationToken());
await crawler.IsBlogOnlineAsync();
}
catch (Exception ex)
{
Logger.Error("ManagerController.CheckStatusOfBlogsAsync: {0}", ex);
_shellService.ShowError(ex, $"Online check for '{blog.Name}' failed: {ex.Message}");
blog.Online = false;
}
finally
{
crawler?.Dispose();
try
{
semaphoreSlim.Release();
}
catch (ObjectDisposedException)
{ }
}
}
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
{
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
bool lockTaken = false;
Monitor.Enter(_lockObject, ref lockTaken);
try
{
if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count)
{
QueueListItem nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First();
IBlog blog = nextQueueItem.Blog;
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct);
crawler.IsBlogOnlineAsync().Wait(4000);
crawler.Dispose();
if (_crawlerService.ActiveItems.Any(item =>
item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType)))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
if (!nextQueueItem.Blog.Online)
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
_crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(_lockObject);
lockTaken = false;
await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct);
}
else
{
Monitor.Exit(_lockObject);
lockTaken = false;
await Task.Delay(4000, ct);
}
}
catch (Exception e)
{
Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e);
_shellService.ShowError(e, "Error starting the next item in the queue.");
if (lockTaken)
{
Monitor.Exit(_lockObject);
}
}
}
}
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
{
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
bool lockTaken = false;
Monitor.Enter(_lockObject, ref lockTaken);
try
{
if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count)
{
QueueListItem nextQueueItem;
try
{
nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First();
}
catch (InvalidOperationException)
{
Monitor.Exit(_lockObject);
continue;
}
IBlog blog = nextQueueItem.Blog;
var privacyConsentNeeded = false;
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct);
try
{
crawler.IsBlogOnlineAsync().Wait(4000);
}
catch (AggregateException ex)
{
if (ex.InnerExceptions.Any(x => x.Message == "Acceptance of privacy consent needed!"))
{
privacyConsentNeeded = true;
}
}
finally
{
crawler.Dispose();
}
if (privacyConsentNeeded ||
(_crawlerService.ActiveItems.Any(item =>
item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType))) ||
(!nextQueueItem.Blog.Online))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
_crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(_lockObject);
lockTaken = false;
await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct);
}
else
{
Monitor.Exit(_lockObject);
lockTaken = false;
await Task.Delay(4000, ct);
}
}
catch (Exception e)
{
if (!ct.IsCancellationRequested)
{
Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e);
}
if (lockTaken)
{
Monitor.Exit(_lockObject);
}
}
}
}