public virtual async Task DownloadCrawlerDataAsync()
{
var trackedTasks = new List <Task>();
blog.CreateDataFolder();
try
{
foreach (TumblrCrawlerData <T> downloadItem in jsonQueue.GetConsumingEnumerable(ct))
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
trackedTasks.Add(DownloadPostAsync(downloadItem));
}
}
catch (OperationCanceledException e)
{
System.Diagnostics.Debug.WriteLine(e.ToString());
}
await Task.WhenAll(trackedTasks);
}
public virtual async Task DownloadCrawlerDataAsync()
{
var trackedTasks = new List <Task>();
blog.CreateDataFolder();
foreach (TumblrCrawlerData <XDocument> downloadItem in xmlQueue.GetConsumingEnumerable())
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
trackedTasks.Add(new Func <Task>(async() =>
{
try { await DownloadTextPost(downloadItem); }
catch { }
})());
}
try { await Task.WhenAll(trackedTasks); }
catch { }
}
public virtual async Task <bool> DownloadBlogAsync()
{
concurrentConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentConnections / crawlerService.ActiveItems.Count);
concurrentVideoConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentVideoConnections / crawlerService.ActiveItems.Count);
var trackedTasks = new List <Task>();
var completeDownload = true;
blog.CreateDataFolder();
try
{
foreach (TumblrPost downloadItem in postQueue.GetConsumingEnumerable(ct))
{
if (downloadItem.GetType() == typeof(VideoPost))
{
await concurrentVideoConnectionsSemaphore.WaitAsync();
}
await concurrentConnectionsSemaphore.WaitAsync();
if (CheckIfShouldStop())
{
break;
}
CheckIfShouldPause();
trackedTasks.Add(DownloadPostAsync(downloadItem));
}
}
catch (OperationCanceledException e)
{
System.Diagnostics.Debug.WriteLine(e.ToString());
}
// TODO: Is this even right?
try
{
await Task.WhenAll(trackedTasks);
}
catch
{
completeDownload = false;
}
blog.LastDownloadedPhoto = null;
blog.LastDownloadedVideo = null;
files.Save();
return(completeDownload);
}
public virtual async Task <bool> DownloadBlogAsync()
{
var concurrentConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentConnections / crawlerService.ActiveItems.Count);
var concurrentVideoConnectionsSemaphore = new SemaphoreSlim(shellService.Settings.ConcurrentVideoConnections / crawlerService.ActiveItems.Count);
var trackedTasks = new List <Task>();
var completeDownload = true;
blog.CreateDataFolder();
foreach (TumblrPost downloadItem in postQueue.GetConsumingEnumerable())
{
if (downloadItem.GetType() == typeof(VideoPost))
{
await concurrentVideoConnectionsSemaphore.WaitAsync();
}
await concurrentConnectionsSemaphore.WaitAsync();
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
trackedTasks.Add(new Func <Task>(async() =>
{
try { await DownloadPostAsync(downloadItem); }
catch {}
finally {
concurrentConnectionsSemaphore.Release();
if (downloadItem.GetType() == typeof(VideoPost))
{
concurrentVideoConnectionsSemaphore.Release();
}
}
})());
}
try { await Task.WhenAll(trackedTasks); }
catch { completeDownload = false; }
blog.LastDownloadedPhoto = null;
blog.LastDownloadedVideo = null;
files.Save();
return(completeDownload);
}
public virtual async Task DownloadCrawlerDataAsync()
{
var trackedTasks = new List <Task>();
blog.CreateDataFolder();
foreach (TumblrCrawlerData <T> downloadItem in jsonQueue.GetConsumingEnumerable())
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
trackedTasks.Add(DownloadPostAsync(downloadItem));
}
await Task.WhenAll(trackedTasks);
}