public virtual async Task <bool> DownloadBlogAsync()
{
concurrentConnectionsSemaphore =
new SemaphoreSlim(shellService.Settings.ConcurrentConnections / crawlerService.ActiveItems.Count);
concurrentVideoConnectionsSemaphore =
new SemaphoreSlim(shellService.Settings.ConcurrentVideoConnections / crawlerService.ActiveItems.Count);
var trackedTasks = new List <Task>();
var completeDownload = true;
blog.CreateDataFolder();
foreach (TumblrPost downloadItem in postQueue.GetConsumingEnumerable())
{
if (downloadItem.GetType() == typeof(VideoPost))
{
await concurrentVideoConnectionsSemaphore.WaitAsync();
}
await concurrentConnectionsSemaphore.WaitAsync();
if (CheckifShouldStop())
{
break;
}
CheckIfShouldPause();
trackedTasks.Add(DownloadPostAsync(downloadItem));
}
// TODO: Is this even right?
try
{
await Task.WhenAll(trackedTasks);
}
catch
{
completeDownload = false;
}
blog.LastDownloadedPhoto = null;
blog.LastDownloadedVideo = null;
files.Save();
return(completeDownload);
}
public virtual async Task <bool> DownloadBlogAsync()
{
var concurrentConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentConnections / crawlerService.ActiveItems.Count);
var concurrentVideoConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentVideoConnections / crawlerService.ActiveItems.Count);
var trackedTasks = new List <Task>();
var completeDownload = true;
blog.CreateDataFolder();
foreach (TumblrPost downloadItem in producerConsumerCollection.GetConsumingEnumerable())
{
if (downloadItem.PostType == PostTypes.Video)
{
await concurrentVideoConnectionsSemaphore.WaitAsync();
}
await concurrentConnectionsSemaphore.WaitAsync();
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
trackedTasks.Add(new Func <Task>(async() =>
{
try { await DownloadPostAsync(downloadItem); }
catch {}
finally { concurrentConnectionsSemaphore.Release(); concurrentVideoConnectionsSemaphore.Release(); }
})());
}
try { await Task.WhenAll(trackedTasks); }
catch { completeDownload = false; }
blog.LastDownloadedPhoto = null;
blog.LastDownloadedVideo = null;
files.Save();
return(completeDownload);
}
public virtual async Task DownloadCrawlerDataAsync()
{
var trackedTasks = new List <Task>();
blog.CreateDataFolder();
foreach (TumblrCrawlerData <T> downloadItem in jsonQueue.GetConsumingEnumerable())
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
trackedTasks.Add(new Func <Task>(async() =>
{
try
{
await DownloadTextPost(downloadItem);
}
catch
{
}
})());
}
try
{
await Task.WhenAll(trackedTasks);
}
catch
{
}
}
public virtual async Task DownloadCrawlerDataAsync()
{
var trackedTasks = new List <Task>();
blog.CreateDataFolder();
foreach (TumblrCrawlerData <XDocument> downloadItem in xmlQueue.GetConsumingEnumerable())
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
trackedTasks.Add(DownloadPostAsync(downloadItem));
}
await Task.WhenAll(trackedTasks);
}
