private async Task StartSiteSpecificDownloaderAsync(QueueListItem queueListItem, PauseToken pt, CancellationToken ct)
{
IBlog blog = queueListItem.Blog;
blog.Dirty = true;
ProgressThrottler <DownloadProgress> progress = SetupThrottledQueueListProgress(queueListItem);
ICrawler crawler = null;
try
{
crawler = _crawlerFactory.GetCrawler(blog, progress, pt, ct);
await crawler.CrawlAsync();
blog.UpdateProgress(true);
}
finally
{
crawler?.Dispose();
}
Monitor.Enter(_lockObject);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => _crawlerService.RemoveActiveItem(queueListItem));
Monitor.Exit(_lockObject);
if (!ct.IsCancellationRequested)
{
Monitor.Enter(_lockObject);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(queueListItem));
Monitor.Exit(_lockObject);
}
}
public ShellViewModel(IShellView view, IShellService shellService, ICrawlerService crawlerService, ExportFactory <SettingsViewModel> settingsViewModelFactory, ExportFactory <AboutViewModel> aboutViewModelFactory)
: base(view)
{
ShellService = shellService;
CrawlerService = crawlerService;
_settings = shellService.Settings;
_settingsViewModelFactory = settingsViewModelFactory;
_aboutViewModelFactory = aboutViewModelFactory;
_errors = new ObservableCollection <Tuple <Exception, string> >();
_exitCommand = new DelegateCommand(Close);
_closeErrorCommand = new DelegateCommand(CloseError);
_garbageCollectorCommand = new DelegateCommand(GC.Collect);
_showSettingsCommand = new DelegateCommand(ShowSettingsView);
_showAboutCommand = new DelegateCommand(ShowAboutView);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => BindingOperations.EnableCollectionSynchronization(Errors, _errorsLock));
_errors.CollectionChanged += ErrorsCollectionChanged;
view.Closed += ViewClosed;
// Restore the window size when the values are valid.
if (_settings.Left >= 0 && _settings.Top >= 0 && _settings.Width > 0 && _settings.Height > 0 &&
_settings.Left + _settings.Width <= view.VirtualScreenWidth &&
_settings.Top + _settings.Height <= view.VirtualScreenHeight)
{
view.Left = _settings.Left;
view.Top = _settings.Top;
view.Height = _settings.Height;
view.Width = _settings.Width;
view.GridSplitterPosition = _settings.GridSplitterPosition;
}
view.IsMaximized = _settings.IsMaximized;
}
private async Task AddBlogAsync(string blogUrl)
{
if (string.IsNullOrEmpty(blogUrl))
{
blogUrl = crawlerService.NewBlogUrl;
}
// FIXME: Dependency
var blog = new TumblrBlog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index"), BlogTypes.tumblr);
TransferGlobalSettingsToBlog(blog);
IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog);
await downloader.IsBlogOnlineAsync();
await downloader.UpdateMetaInformationAsync();
lock (lockObject)
{
if (managerService.BlogFiles.Select(blogs => blogs.Name).ToList().Contains(blog.Name))
{
shellService.ShowError(null, Resources.BlogAlreadyExist, blog.Name);
return;
}
if (blog.Save())
{
QueueOnDispatcher.CheckBeginInvokeOnUI((Action)(() => managerService.BlogFiles.Add(blog)));
}
}
}
private void OnTimedEvent()
{
if (CrawlerService.AutoDownloadCommand.CanExecute(null))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => CrawlerService.AutoDownloadCommand.Execute(null));
}
CrawlerService.Timer.Change(new TimeSpan(24, 00, 00), Timeout.InfiniteTimeSpan);
}
private void AddToManager(IBlog blog)
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => _managerService.BlogFiles.Add(blog));
if (_shellService.Settings.LoadAllDatabases)
{
_managerService.AddDatabase(new Files().Load(blog.ChildId));
}
}
public void RequestShutdown()
{
Task.Run(() => QueueOnDispatcher.CheckBeginInvokeOnUI(() =>
{
DoShutdown();
Shutdown();
})).GetAwaiter().GetResult();
}
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
{
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
Monitor.Enter(_lockObject);
if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count)
{
IEnumerable <QueueListItem> queueList = QueueManager.Items.Except(_crawlerService.ActiveItems);
QueueListItem nextQueueItem = queueList.First();
IBlog blog = nextQueueItem.Blog;
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct);
crawler.IsBlogOnlineAsync().Wait(4000);
crawler.Dispose();
if (_crawlerService.ActiveItems.Any(item =>
item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType)))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
if (!nextQueueItem.Blog.Online)
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
_crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(_lockObject);
await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct);
}
else
{
Monitor.Exit(_lockObject);
await Task.Delay(4000, ct);
}
}
}
protected override void OnStartup(StartupEventArgs e)
{
base.OnStartup(e);
//InitializeCultures();
ServicePointManager.DefaultConnectionLimit = 400;
// Trust all SSL hosts since tumblr.com messed up their ssl cert on amazon s3.
ServicePointManager.ServerCertificateValidationCallback += new RemoteCertificateValidationCallback(ValidateCertificate);
catalog = new AggregateCatalog();
// Add the WpfApplicationFramework assembly to the catalog
catalog.Catalogs.Add(new AssemblyCatalog(typeof(WafConfiguration).Assembly));
// Add the TumblThree.Applications assembly
catalog.Catalogs.Add(new AssemblyCatalog(typeof(ShellViewModel).Assembly));
// Add the TumblThree.Domain assembly
catalog.Catalogs.Add(new AssemblyCatalog(typeof(IBlog).Assembly));
// Add the TumblThree.Presentation assembly
catalog.Catalogs.Add(new AssemblyCatalog(typeof(App).Assembly));
container = new CompositionContainer(catalog, CompositionOptions.DisableSilentRejection);
var batch = new CompositionBatch();
batch.AddExportedValue(container);
container.Compose(batch);
// Initialize all presentation services
IEnumerable <IPresentationService> presentationServices = container.GetExportedValues <IPresentationService>();
foreach (IPresentationService presentationService in presentationServices)
{
presentationService.Initialize();
}
// Initialize and run all module controllers
moduleControllers = container.GetExportedValues <IModuleController>();
foreach (IModuleController moduleController in moduleControllers)
{
moduleController.Initialize();
}
var shellService = container.GetExportedValue <IShellService>();
InitializeCultures(shellService);
foreach (IModuleController moduleController in moduleControllers)
{
moduleController.Run();
}
QueueOnDispatcher.Initialize();
}
private async Task AddBlogAsync(string blogUrl)
{
if (string.IsNullOrEmpty(blogUrl))
{
blogUrl = crawlerService.NewBlogUrl;
}
IBlog blog;
// TODO: Dependency, not SOLID!
if (Validator.IsValidTumblrUrl(blogUrl))
{
blog = new Blog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index"), BlogTypes.tumblr);
}
else if (Validator.IsValidTumblrLikedByUrl(blogUrl))
{
blog = new TumblrLikeByBlog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index"), BlogTypes.tlb);
}
//else if (Validator.IsValidTumblrSearchUrl(blogUrl))
// blog = new TumblrSearchBlog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index"), BlogTypes.ts);
else
{
return;
}
TransferGlobalSettingsToBlog(blog);
IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog);
await downloader.IsBlogOnlineAsync();
if (CheckIfTumblrPrivateBlog(blog))
{
blog = PromoteTumblrBlogToPrivateBlog(blogUrl);
downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog);
}
await downloader.UpdateMetaInformationAsync();
lock (lockObject)
{
if (managerService.BlogFiles.Any(blogs => blogs.Name.Equals(blog.Name) && blogs.BlogType.Equals(blog.BlogType)))
{
shellService.ShowError(null, Resources.BlogAlreadyExist, blog.Name);
return;
}
if (blog.Save())
{
QueueOnDispatcher.CheckBeginInvokeOnUI((Action)(() => managerService.BlogFiles.Add(blog)));
}
}
}
private async Task RunCrawlerTasks(CancellationToken ct, PauseToken pt)
{
while (true)
{
ct.ThrowIfCancellationRequested();
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait(ct);
}
Monitor.Enter(lockObject);
if (crawlerService.ActiveItems.Count() < QueueManager.Items.Count())
{
IEnumerable <QueueListItem> queueList = QueueManager.Items.Except(crawlerService.ActiveItems);
QueueListItem nextQueueItem = queueList.First();
IBlog blog = nextQueueItem.Blog;
IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog);
downloader.IsBlogOnlineAsync().Wait(4000);
if (crawlerService.ActiveItems.Any(item => item.Blog.Name.Contains(nextQueueItem.Blog.Name)))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(lockObject);
continue;
}
if (!nextQueueItem.Blog.Online)
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(lockObject);
continue;
}
crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(lockObject);
await StartSiteSpecificDownloader(nextQueueItem, ct, pt);
}
else
{
Monitor.Exit(lockObject);
await Task.Delay(4000, ct);
}
}
}
private async Task AddBlogAsync(string blogUrl)
{
if (string.IsNullOrEmpty(blogUrl))
{
blogUrl = crawlerService.NewBlogUrl;
}
// TODO: Dependency, not SOLID!
IBlog blog;
try
{
blog = BlogFactory.GetBlog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index"));
}
catch (ArgumentException)
{
return;
}
blog = settingsService.TransferGlobalSettingsToBlog(blog);
ICrawler crawler = CrawlerFactory.GetCrawler(blog.BlogType, new CancellationToken(), new PauseToken(), new Progress <DownloadProgress>(), shellService, crawlerService, blog);
await crawler.IsBlogOnlineAsync();
if (CheckIfTumblrPrivateBlog(blog))
{
blog = PromoteTumblrBlogToPrivateBlog(blog);
crawler = CrawlerFactory.GetCrawler(blog.BlogType, new CancellationToken(), new PauseToken(), new Progress <DownloadProgress>(), shellService, crawlerService, blog);
}
await crawler.UpdateMetaInformationAsync();
lock (lockObject)
{
if (managerService.BlogFiles.Any(blogs => blogs.Name.Equals(blog.Name) && blogs.BlogType.Equals(blog.BlogType)))
{
shellService.ShowError(null, Resources.BlogAlreadyExist, blog.Name);
return;
}
if (blog.Save())
{
QueueOnDispatcher.CheckBeginInvokeOnUI((Action)(() => managerService.BlogFiles.Add(blog)));
}
}
}
private async Task StartSiteSpecificDownloaderAsync(QueueListItem queueListItem, PauseToken pt, CancellationToken ct)
{
IBlog blog = queueListItem.Blog;
blog.Dirty = true;
ProgressThrottler <DownloadProgress> progress = SetupThrottledQueueListProgress(queueListItem);
ICrawler crawler = null;
try
{
crawler = _crawlerFactory.GetCrawler(blog, progress, pt, ct);
queueListItem.InterruptionRequested += crawler.InterruptionRequestedEventHandler;
await crawler.CrawlAsync();
blog.UpdateProgress(false);
}
catch (Exception e)
{
if (!ct.IsCancellationRequested)
{
Logger.Error("CrawlerController.StartSiteSpecificDownloaderAsync: {0}", e);
}
}
finally
{
if (crawler != null)
{
queueListItem.InterruptionRequested -= crawler.InterruptionRequestedEventHandler;
}
crawler?.Dispose();
}
Monitor.Enter(_lockObject);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => _crawlerService.RemoveActiveItem(queueListItem));
Monitor.Exit(_lockObject);
if (!ct.IsCancellationRequested)
{
Monitor.Enter(_lockObject);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(queueListItem));
Monitor.Exit(_lockObject);
}
}
private async Task AddBlogBatchedAsync(IEnumerable <string> urls, bool fromClipboard)
{
var semaphoreSlim = new SemaphoreSlim(25);
await _addBlogSemaphoreSlim.WaitAsync();
QueueOnDispatcher.CheckBeginInvokeOnUI(() => Mouse.OverrideCursor = System.Windows.Input.Cursors.Wait);
try
{
IEnumerable <Task> tasks = urls.Select(async url => await AddBlogsAsync(semaphoreSlim, url, fromClipboard));
await Task.WhenAll(tasks);
}
finally
{
_addBlogSemaphoreSlim.Release();
semaphoreSlim.Dispose();
QueueOnDispatcher.CheckBeginInvokeOnUI(() => Mouse.OverrideCursor = null);
}
}
private async Task StartSiteSpecificDownloader(QueueListItem queueListItem, CancellationToken ct, PauseToken pt)
{
IBlog blog = queueListItem.Blog;
blog.Dirty = true;
ProgressThrottler <DownloadProgress> progress = SetupThrottledQueueListProgress(queueListItem);
ICrawler crawler = CrawlerFactory.GetCrawler(blog.BlogType, ct, pt, progress, shellService, crawlerService, blog);
await crawler.Crawl();
if (ct.IsCancellationRequested)
{
Monitor.Enter(lockObject);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => crawlerService.RemoveActiveItem(queueListItem));
Monitor.Exit(lockObject);
}
else
{
Monitor.Enter(lockObject);
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(queueListItem));
QueueOnDispatcher.CheckBeginInvokeOnUI(() => crawlerService.RemoveActiveItem(queueListItem));
Monitor.Exit(lockObject);
}
}
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
{
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
bool lockTaken = false;
Monitor.Enter(_lockObject, ref lockTaken);
try
{
if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count)
{
QueueListItem nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First();
IBlog blog = nextQueueItem.Blog;
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct);
crawler.IsBlogOnlineAsync().Wait(4000);
crawler.Dispose();
if (_crawlerService.ActiveItems.Any(item =>
item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType)))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
if (!nextQueueItem.Blog.Online)
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
_crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(_lockObject);
lockTaken = false;
await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct);
}
else
{
Monitor.Exit(_lockObject);
lockTaken = false;
await Task.Delay(4000, ct);
}
}
catch (Exception e)
{
Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e);
_shellService.ShowError(e, "Error starting the next item in the queue.");
if (lockTaken)
{
Monitor.Exit(_lockObject);
}
}
}
}
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
{
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
bool lockTaken = false;
Monitor.Enter(_lockObject, ref lockTaken);
try
{
if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count)
{
QueueListItem nextQueueItem;
try
{
nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First();
}
catch (InvalidOperationException)
{
Monitor.Exit(_lockObject);
continue;
}
IBlog blog = nextQueueItem.Blog;
var privacyConsentNeeded = false;
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct);
try
{
crawler.IsBlogOnlineAsync().Wait(4000);
}
catch (AggregateException ex)
{
if (ex.InnerExceptions.Any(x => x.Message == "Acceptance of privacy consent needed!"))
{
privacyConsentNeeded = true;
}
}
finally
{
crawler.Dispose();
}
if (privacyConsentNeeded ||
(_crawlerService.ActiveItems.Any(item =>
item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType))) ||
(!nextQueueItem.Blog.Online))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
_crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(_lockObject);
lockTaken = false;
await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct);
}
else
{
Monitor.Exit(_lockObject);
lockTaken = false;
await Task.Delay(4000, ct);
}
}
catch (Exception e)
{
if (!ct.IsCancellationRequested)
{
Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e);
}
if (lockTaken)
{
Monitor.Exit(_lockObject);
}
}
}
}
