private async Task StartSiteSpecificDownloaderAsync(QueueListItem queueListItem, PauseToken pt, CancellationToken ct) { IBlog blog = queueListItem.Blog; blog.Dirty = true; ProgressThrottler <DownloadProgress> progress = SetupThrottledQueueListProgress(queueListItem); ICrawler crawler = null; try { crawler = _crawlerFactory.GetCrawler(blog, progress, pt, ct); await crawler.CrawlAsync(); blog.UpdateProgress(true); } finally { crawler?.Dispose(); } Monitor.Enter(_lockObject); QueueOnDispatcher.CheckBeginInvokeOnUI(() => _crawlerService.RemoveActiveItem(queueListItem)); Monitor.Exit(_lockObject); if (!ct.IsCancellationRequested) { Monitor.Enter(_lockObject); QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(queueListItem)); Monitor.Exit(_lockObject); } }
public ShellViewModel(IShellView view, IShellService shellService, ICrawlerService crawlerService, ExportFactory <SettingsViewModel> settingsViewModelFactory, ExportFactory <AboutViewModel> aboutViewModelFactory) : base(view) { ShellService = shellService; CrawlerService = crawlerService; _settings = shellService.Settings; _settingsViewModelFactory = settingsViewModelFactory; _aboutViewModelFactory = aboutViewModelFactory; _errors = new ObservableCollection <Tuple <Exception, string> >(); _exitCommand = new DelegateCommand(Close); _closeErrorCommand = new DelegateCommand(CloseError); _garbageCollectorCommand = new DelegateCommand(GC.Collect); _showSettingsCommand = new DelegateCommand(ShowSettingsView); _showAboutCommand = new DelegateCommand(ShowAboutView); QueueOnDispatcher.CheckBeginInvokeOnUI(() => BindingOperations.EnableCollectionSynchronization(Errors, _errorsLock)); _errors.CollectionChanged += ErrorsCollectionChanged; view.Closed += ViewClosed; // Restore the window size when the values are valid. if (_settings.Left >= 0 && _settings.Top >= 0 && _settings.Width > 0 && _settings.Height > 0 && _settings.Left + _settings.Width <= view.VirtualScreenWidth && _settings.Top + _settings.Height <= view.VirtualScreenHeight) { view.Left = _settings.Left; view.Top = _settings.Top; view.Height = _settings.Height; view.Width = _settings.Width; view.GridSplitterPosition = _settings.GridSplitterPosition; } view.IsMaximized = _settings.IsMaximized; }
private async Task AddBlogAsync(string blogUrl) { if (string.IsNullOrEmpty(blogUrl)) { blogUrl = crawlerService.NewBlogUrl; } // FIXME: Dependency var blog = new TumblrBlog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index"), BlogTypes.tumblr); TransferGlobalSettingsToBlog(blog); IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog); await downloader.IsBlogOnlineAsync(); await downloader.UpdateMetaInformationAsync(); lock (lockObject) { if (managerService.BlogFiles.Select(blogs => blogs.Name).ToList().Contains(blog.Name)) { shellService.ShowError(null, Resources.BlogAlreadyExist, blog.Name); return; } if (blog.Save()) { QueueOnDispatcher.CheckBeginInvokeOnUI((Action)(() => managerService.BlogFiles.Add(blog))); } } }
private void OnTimedEvent() { if (CrawlerService.AutoDownloadCommand.CanExecute(null)) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => CrawlerService.AutoDownloadCommand.Execute(null)); } CrawlerService.Timer.Change(new TimeSpan(24, 00, 00), Timeout.InfiniteTimeSpan); }
private void AddToManager(IBlog blog) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => _managerService.BlogFiles.Add(blog)); if (_shellService.Settings.LoadAllDatabases) { _managerService.AddDatabase(new Files().Load(blog.ChildId)); } }
public void RequestShutdown() { Task.Run(() => QueueOnDispatcher.CheckBeginInvokeOnUI(() => { DoShutdown(); Shutdown(); })).GetAwaiter().GetResult(); }
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct) { while (true) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } Monitor.Enter(_lockObject); if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count) { IEnumerable <QueueListItem> queueList = QueueManager.Items.Except(_crawlerService.ActiveItems); QueueListItem nextQueueItem = queueList.First(); IBlog blog = nextQueueItem.Blog; ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct); crawler.IsBlogOnlineAsync().Wait(4000); crawler.Dispose(); if (_crawlerService.ActiveItems.Any(item => item.Blog.Name.Equals(nextQueueItem.Blog.Name) && item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType))) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } if (!nextQueueItem.Blog.Online) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } _crawlerService.AddActiveItems(nextQueueItem); Monitor.Exit(_lockObject); await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct); } else { Monitor.Exit(_lockObject); await Task.Delay(4000, ct); } } }
protected override void OnStartup(StartupEventArgs e) { base.OnStartup(e); //InitializeCultures(); ServicePointManager.DefaultConnectionLimit = 400; // Trust all SSL hosts since tumblr.com messed up their ssl cert on amazon s3. ServicePointManager.ServerCertificateValidationCallback += new RemoteCertificateValidationCallback(ValidateCertificate); catalog = new AggregateCatalog(); // Add the WpfApplicationFramework assembly to the catalog catalog.Catalogs.Add(new AssemblyCatalog(typeof(WafConfiguration).Assembly)); // Add the TumblThree.Applications assembly catalog.Catalogs.Add(new AssemblyCatalog(typeof(ShellViewModel).Assembly)); // Add the TumblThree.Domain assembly catalog.Catalogs.Add(new AssemblyCatalog(typeof(IBlog).Assembly)); // Add the TumblThree.Presentation assembly catalog.Catalogs.Add(new AssemblyCatalog(typeof(App).Assembly)); container = new CompositionContainer(catalog, CompositionOptions.DisableSilentRejection); var batch = new CompositionBatch(); batch.AddExportedValue(container); container.Compose(batch); // Initialize all presentation services IEnumerable <IPresentationService> presentationServices = container.GetExportedValues <IPresentationService>(); foreach (IPresentationService presentationService in presentationServices) { presentationService.Initialize(); } // Initialize and run all module controllers moduleControllers = container.GetExportedValues <IModuleController>(); foreach (IModuleController moduleController in moduleControllers) { moduleController.Initialize(); } var shellService = container.GetExportedValue <IShellService>(); InitializeCultures(shellService); foreach (IModuleController moduleController in moduleControllers) { moduleController.Run(); } QueueOnDispatcher.Initialize(); }
private async Task AddBlogAsync(string blogUrl) { if (string.IsNullOrEmpty(blogUrl)) { blogUrl = crawlerService.NewBlogUrl; } IBlog blog; // TODO: Dependency, not SOLID! if (Validator.IsValidTumblrUrl(blogUrl)) { blog = new Blog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index"), BlogTypes.tumblr); } else if (Validator.IsValidTumblrLikedByUrl(blogUrl)) { blog = new TumblrLikeByBlog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index"), BlogTypes.tlb); } //else if (Validator.IsValidTumblrSearchUrl(blogUrl)) // blog = new TumblrSearchBlog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index"), BlogTypes.ts); else { return; } TransferGlobalSettingsToBlog(blog); IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog); await downloader.IsBlogOnlineAsync(); if (CheckIfTumblrPrivateBlog(blog)) { blog = PromoteTumblrBlogToPrivateBlog(blogUrl); downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog); } await downloader.UpdateMetaInformationAsync(); lock (lockObject) { if (managerService.BlogFiles.Any(blogs => blogs.Name.Equals(blog.Name) && blogs.BlogType.Equals(blog.BlogType))) { shellService.ShowError(null, Resources.BlogAlreadyExist, blog.Name); return; } if (blog.Save()) { QueueOnDispatcher.CheckBeginInvokeOnUI((Action)(() => managerService.BlogFiles.Add(blog))); } } }
private async Task RunCrawlerTasks(CancellationToken ct, PauseToken pt) { while (true) { ct.ThrowIfCancellationRequested(); if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(ct); } Monitor.Enter(lockObject); if (crawlerService.ActiveItems.Count() < QueueManager.Items.Count()) { IEnumerable <QueueListItem> queueList = QueueManager.Items.Except(crawlerService.ActiveItems); QueueListItem nextQueueItem = queueList.First(); IBlog blog = nextQueueItem.Blog; IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog); downloader.IsBlogOnlineAsync().Wait(4000); if (crawlerService.ActiveItems.Any(item => item.Blog.Name.Contains(nextQueueItem.Blog.Name))) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(lockObject); continue; } if (!nextQueueItem.Blog.Online) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(lockObject); continue; } crawlerService.AddActiveItems(nextQueueItem); Monitor.Exit(lockObject); await StartSiteSpecificDownloader(nextQueueItem, ct, pt); } else { Monitor.Exit(lockObject); await Task.Delay(4000, ct); } } }
private async Task AddBlogAsync(string blogUrl) { if (string.IsNullOrEmpty(blogUrl)) { blogUrl = crawlerService.NewBlogUrl; } // TODO: Dependency, not SOLID! IBlog blog; try { blog = BlogFactory.GetBlog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index")); } catch (ArgumentException) { return; } blog = settingsService.TransferGlobalSettingsToBlog(blog); ICrawler crawler = CrawlerFactory.GetCrawler(blog.BlogType, new CancellationToken(), new PauseToken(), new Progress <DownloadProgress>(), shellService, crawlerService, blog); await crawler.IsBlogOnlineAsync(); if (CheckIfTumblrPrivateBlog(blog)) { blog = PromoteTumblrBlogToPrivateBlog(blog); crawler = CrawlerFactory.GetCrawler(blog.BlogType, new CancellationToken(), new PauseToken(), new Progress <DownloadProgress>(), shellService, crawlerService, blog); } await crawler.UpdateMetaInformationAsync(); lock (lockObject) { if (managerService.BlogFiles.Any(blogs => blogs.Name.Equals(blog.Name) && blogs.BlogType.Equals(blog.BlogType))) { shellService.ShowError(null, Resources.BlogAlreadyExist, blog.Name); return; } if (blog.Save()) { QueueOnDispatcher.CheckBeginInvokeOnUI((Action)(() => managerService.BlogFiles.Add(blog))); } } }
private async Task StartSiteSpecificDownloaderAsync(QueueListItem queueListItem, PauseToken pt, CancellationToken ct) { IBlog blog = queueListItem.Blog; blog.Dirty = true; ProgressThrottler <DownloadProgress> progress = SetupThrottledQueueListProgress(queueListItem); ICrawler crawler = null; try { crawler = _crawlerFactory.GetCrawler(blog, progress, pt, ct); queueListItem.InterruptionRequested += crawler.InterruptionRequestedEventHandler; await crawler.CrawlAsync(); blog.UpdateProgress(false); } catch (Exception e) { if (!ct.IsCancellationRequested) { Logger.Error("CrawlerController.StartSiteSpecificDownloaderAsync: {0}", e); } } finally { if (crawler != null) { queueListItem.InterruptionRequested -= crawler.InterruptionRequestedEventHandler; } crawler?.Dispose(); } Monitor.Enter(_lockObject); QueueOnDispatcher.CheckBeginInvokeOnUI(() => _crawlerService.RemoveActiveItem(queueListItem)); Monitor.Exit(_lockObject); if (!ct.IsCancellationRequested) { Monitor.Enter(_lockObject); QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(queueListItem)); Monitor.Exit(_lockObject); } }
private async Task AddBlogBatchedAsync(IEnumerable <string> urls, bool fromClipboard) { var semaphoreSlim = new SemaphoreSlim(25); await _addBlogSemaphoreSlim.WaitAsync(); QueueOnDispatcher.CheckBeginInvokeOnUI(() => Mouse.OverrideCursor = System.Windows.Input.Cursors.Wait); try { IEnumerable <Task> tasks = urls.Select(async url => await AddBlogsAsync(semaphoreSlim, url, fromClipboard)); await Task.WhenAll(tasks); } finally { _addBlogSemaphoreSlim.Release(); semaphoreSlim.Dispose(); QueueOnDispatcher.CheckBeginInvokeOnUI(() => Mouse.OverrideCursor = null); } }
private async Task StartSiteSpecificDownloader(QueueListItem queueListItem, CancellationToken ct, PauseToken pt) { IBlog blog = queueListItem.Blog; blog.Dirty = true; ProgressThrottler <DownloadProgress> progress = SetupThrottledQueueListProgress(queueListItem); ICrawler crawler = CrawlerFactory.GetCrawler(blog.BlogType, ct, pt, progress, shellService, crawlerService, blog); await crawler.Crawl(); if (ct.IsCancellationRequested) { Monitor.Enter(lockObject); QueueOnDispatcher.CheckBeginInvokeOnUI(() => crawlerService.RemoveActiveItem(queueListItem)); Monitor.Exit(lockObject); } else { Monitor.Enter(lockObject); QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(queueListItem)); QueueOnDispatcher.CheckBeginInvokeOnUI(() => crawlerService.RemoveActiveItem(queueListItem)); Monitor.Exit(lockObject); } }
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct) { while (true) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } bool lockTaken = false; Monitor.Enter(_lockObject, ref lockTaken); try { if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count) { QueueListItem nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First(); IBlog blog = nextQueueItem.Blog; ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct); crawler.IsBlogOnlineAsync().Wait(4000); crawler.Dispose(); if (_crawlerService.ActiveItems.Any(item => item.Blog.Name.Equals(nextQueueItem.Blog.Name) && item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType))) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } if (!nextQueueItem.Blog.Online) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } _crawlerService.AddActiveItems(nextQueueItem); Monitor.Exit(_lockObject); lockTaken = false; await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct); } else { Monitor.Exit(_lockObject); lockTaken = false; await Task.Delay(4000, ct); } } catch (Exception e) { Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e); _shellService.ShowError(e, "Error starting the next item in the queue."); if (lockTaken) { Monitor.Exit(_lockObject); } } } }
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct) { while (true) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } bool lockTaken = false; Monitor.Enter(_lockObject, ref lockTaken); try { if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count) { QueueListItem nextQueueItem; try { nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First(); } catch (InvalidOperationException) { Monitor.Exit(_lockObject); continue; } IBlog blog = nextQueueItem.Blog; var privacyConsentNeeded = false; ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct); try { crawler.IsBlogOnlineAsync().Wait(4000); } catch (AggregateException ex) { if (ex.InnerExceptions.Any(x => x.Message == "Acceptance of privacy consent needed!")) { privacyConsentNeeded = true; } } finally { crawler.Dispose(); } if (privacyConsentNeeded || (_crawlerService.ActiveItems.Any(item => item.Blog.Name.Equals(nextQueueItem.Blog.Name) && item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType))) || (!nextQueueItem.Blog.Online)) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } _crawlerService.AddActiveItems(nextQueueItem); Monitor.Exit(_lockObject); lockTaken = false; await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct); } else { Monitor.Exit(_lockObject); lockTaken = false; await Task.Delay(4000, ct); } } catch (Exception e) { if (!ct.IsCancellationRequested) { Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e); } if (lockTaken) { Monitor.Exit(_lockObject); } } } }