public async Task DownloadCrawlerDataAsync()
{
var trackedTasks = new List <Task>();
_blog.CreateDataFolder();
try
{
while (await _xmlQueue.OutputAvailableAsync(_ct))
{
CrawlerData <XDocument> downloadItem = (CrawlerData <XDocument>) await _xmlQueue.ReceiveAsync();
if (_ct.IsCancellationRequested)
{
break;
}
if (_pt.IsPaused)
{
_pt.WaitWhilePausedWithResponseAsyc().Wait();
}
trackedTasks.Add(DownloadPostAsync(downloadItem));
}
}
catch (OperationCanceledException e)
{
System.Diagnostics.Debug.WriteLine(e.ToString());
}
await Task.WhenAll(trackedTasks);
}
public virtual async Task <bool> DownloadBlogAsync()
{
concurrentConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentConnections / crawlerService.ActiveItems.Count);
concurrentVideoConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentVideoConnections / crawlerService.ActiveItems.Count);
var trackedTasks = new List <Task>();
var completeDownload = true;
blog.CreateDataFolder();
await Task.Run(() => Task.CompletedTask);
try
{
while (await postQueue.OutputAvailableAsync(ct))
{
TumblrPost downloadItem = (TumblrPost)await postQueue.ReceiveAsync();
if (downloadItem.GetType() == typeof(VideoPost))
{
await concurrentVideoConnectionsSemaphore.WaitAsync();
}
await concurrentConnectionsSemaphore.WaitAsync();
if (CheckIfShouldStop())
{
break;
}
CheckIfShouldPause();
trackedTasks.Add(DownloadPostAsync(downloadItem));
}
}
catch (OperationCanceledException e)
{
System.Diagnostics.Debug.WriteLine(e.ToString());
}
// TODO: Is this even right?
try
{
await Task.WhenAll(trackedTasks);
}
catch
{
completeDownload = false;
}
blog.LastDownloadedPhoto = null;
blog.LastDownloadedVideo = null;
files.Save();
return(completeDownload);
}