public virtual async Task DownloadCrawlerDataAsync()
{
var trackedTasks = new List <Task>();
blog.CreateDataFolder();
try
{
while (await jsonQueue.OutputAvailableAsync(ct))
{
CrawlerData <T> downloadItem = (CrawlerData <T>) await jsonQueue.ReceiveAsync();
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
trackedTasks.Add(DownloadPostAsync(downloadItem));
}
}
catch (OperationCanceledException e)
{
System.Diagnostics.Debug.WriteLine(e.ToString());
}
await Task.WhenAll(trackedTasks);
}
public virtual async Task <bool> DownloadBlogAsync()
{
concurrentConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentConnections / crawlerService.ActiveItems.Count);
concurrentVideoConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentVideoConnections / crawlerService.ActiveItems.Count);
var trackedTasks = new List <Task>();
var completeDownload = true;
blog.CreateDataFolder();
await Task.Run(() => Task.CompletedTask);
try
{
while (await postQueue.OutputAvailableAsync(ct))
{
TumblrPost downloadItem = (TumblrPost)await postQueue.ReceiveAsync();
if (downloadItem.GetType() == typeof(VideoPost))
{
await concurrentVideoConnectionsSemaphore.WaitAsync();
}
await concurrentConnectionsSemaphore.WaitAsync();
if (CheckIfShouldStop())
{
break;
}
CheckIfShouldPause();
trackedTasks.Add(DownloadPostAsync(downloadItem));
}
}
catch (OperationCanceledException e)
{
System.Diagnostics.Debug.WriteLine(e.ToString());
}
// TODO: Is this even right?
try
{
await Task.WhenAll(trackedTasks);
}
catch
{
completeDownload = false;
}
blog.LastDownloadedPhoto = null;
blog.LastDownloadedVideo = null;
files.Save();
return(completeDownload);
}