public virtual async Task <bool> DownloadBlogAsync() { concurrentConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentConnections / crawlerService.ActiveItems.Count); concurrentVideoConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentVideoConnections / crawlerService.ActiveItems.Count); var trackedTasks = new List <Task>(); var completeDownload = true; blog.CreateDataFolder(); foreach (TumblrPost downloadItem in postQueue.GetConsumingEnumerable()) { if (downloadItem.GetType() == typeof(VideoPost)) { await concurrentVideoConnectionsSemaphore.WaitAsync(); } await concurrentConnectionsSemaphore.WaitAsync(); if (CheckifShouldStop()) { break; } CheckIfShouldPause(); trackedTasks.Add(DownloadPostAsync(downloadItem)); } // TODO: Is this even right? try { await Task.WhenAll(trackedTasks); } catch { completeDownload = false; } blog.LastDownloadedPhoto = null; blog.LastDownloadedVideo = null; files.Save(); return(completeDownload); }
public virtual async Task <bool> DownloadBlogAsync() { var concurrentConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentConnections / crawlerService.ActiveItems.Count); var concurrentVideoConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentVideoConnections / crawlerService.ActiveItems.Count); var trackedTasks = new List <Task>(); var completeDownload = true; blog.CreateDataFolder(); foreach (TumblrPost downloadItem in producerConsumerCollection.GetConsumingEnumerable()) { if (downloadItem.PostType == PostTypes.Video) { await concurrentVideoConnectionsSemaphore.WaitAsync(); } await concurrentConnectionsSemaphore.WaitAsync(); if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } trackedTasks.Add(new Func <Task>(async() => { try { await DownloadPostAsync(downloadItem); } catch {} finally { concurrentConnectionsSemaphore.Release(); concurrentVideoConnectionsSemaphore.Release(); } })()); } try { await Task.WhenAll(trackedTasks); } catch { completeDownload = false; } blog.LastDownloadedPhoto = null; blog.LastDownloadedVideo = null; files.Save(); return(completeDownload); }
public virtual async Task DownloadCrawlerDataAsync() { var trackedTasks = new List <Task>(); blog.CreateDataFolder(); foreach (TumblrCrawlerData <T> downloadItem in jsonQueue.GetConsumingEnumerable()) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } trackedTasks.Add(new Func <Task>(async() => { try { await DownloadTextPost(downloadItem); } catch { } })()); } try { await Task.WhenAll(trackedTasks); } catch { } }
public virtual async Task DownloadCrawlerDataAsync() { var trackedTasks = new List <Task>(); blog.CreateDataFolder(); foreach (TumblrCrawlerData <XDocument> downloadItem in xmlQueue.GetConsumingEnumerable()) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } trackedTasks.Add(DownloadPostAsync(downloadItem)); } await Task.WhenAll(trackedTasks); }