public virtual async Task DownloadCrawlerDataAsync() { var trackedTasks = new List <Task>(); blog.CreateDataFolder(); try { foreach (TumblrCrawlerData <T> downloadItem in jsonQueue.GetConsumingEnumerable(ct)) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } trackedTasks.Add(DownloadPostAsync(downloadItem)); } } catch (OperationCanceledException e) { System.Diagnostics.Debug.WriteLine(e.ToString()); } await Task.WhenAll(trackedTasks); }
public virtual async Task DownloadCrawlerDataAsync() { var trackedTasks = new List <Task>(); blog.CreateDataFolder(); foreach (TumblrCrawlerData <XDocument> downloadItem in xmlQueue.GetConsumingEnumerable()) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } trackedTasks.Add(new Func <Task>(async() => { try { await DownloadTextPost(downloadItem); } catch { } })()); } try { await Task.WhenAll(trackedTasks); } catch { } }
public virtual async Task <bool> DownloadBlogAsync() { concurrentConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentConnections / crawlerService.ActiveItems.Count); concurrentVideoConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentVideoConnections / crawlerService.ActiveItems.Count); var trackedTasks = new List <Task>(); var completeDownload = true; blog.CreateDataFolder(); try { foreach (TumblrPost downloadItem in postQueue.GetConsumingEnumerable(ct)) { if (downloadItem.GetType() == typeof(VideoPost)) { await concurrentVideoConnectionsSemaphore.WaitAsync(); } await concurrentConnectionsSemaphore.WaitAsync(); if (CheckIfShouldStop()) { break; } CheckIfShouldPause(); trackedTasks.Add(DownloadPostAsync(downloadItem)); } } catch (OperationCanceledException e) { System.Diagnostics.Debug.WriteLine(e.ToString()); } // TODO: Is this even right? try { await Task.WhenAll(trackedTasks); } catch { completeDownload = false; } blog.LastDownloadedPhoto = null; blog.LastDownloadedVideo = null; files.Save(); return(completeDownload); }
public virtual async Task <bool> DownloadBlogAsync() { var concurrentConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentConnections / crawlerService.ActiveItems.Count); var concurrentVideoConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentVideoConnections / crawlerService.ActiveItems.Count); var trackedTasks = new List <Task>(); var completeDownload = true; blog.CreateDataFolder(); foreach (TumblrPost downloadItem in postQueue.GetConsumingEnumerable()) { if (downloadItem.GetType() == typeof(VideoPost)) { await concurrentVideoConnectionsSemaphore.WaitAsync(); } await concurrentConnectionsSemaphore.WaitAsync(); if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } trackedTasks.Add(new Func <Task>(async() => { try { await DownloadPostAsync(downloadItem); } catch {} finally { concurrentConnectionsSemaphore.Release(); if (downloadItem.GetType() == typeof(VideoPost)) { concurrentVideoConnectionsSemaphore.Release(); } } })()); } try { await Task.WhenAll(trackedTasks); } catch { completeDownload = false; } blog.LastDownloadedPhoto = null; blog.LastDownloadedVideo = null; files.Save(); return(completeDownload); }
public virtual async Task DownloadCrawlerDataAsync() { var trackedTasks = new List <Task>(); blog.CreateDataFolder(); foreach (TumblrCrawlerData <T> downloadItem in jsonQueue.GetConsumingEnumerable()) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } trackedTasks.Add(DownloadPostAsync(downloadItem)); } await Task.WhenAll(trackedTasks); }