protected virtual async Task <bool> DownloadBinaryFile(string fileLocation, string url, CancellationToken ct)
{
try
{
var fileDownloader = new FileDownloader();
return(await fileDownloader.DownloadFileWithResumeAsync(url, fileLocation,
shellService.Settings, ct));
}
catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70)
{
Logger.Error("ManagerController:Download: {0}", ex);
shellService.ShowError(ex, Resources.DiskFull);
crawlerService.StopCommand.Execute(null);
return(false);
}
catch (WebException webException) when((webException.Response != null))
{
var webRespStatusCode = (int)((HttpWebResponse)webException?.Response).StatusCode;
if (webRespStatusCode >= 400 && webRespStatusCode < 600) // removes inaccessible files: status code 400 -- 599
{
File.Delete(fileLocation);
}
return(false);
}
catch
{
return(false);
}
}
protected virtual async Task <bool> DownloadBinaryFile(string fileLocation, string url)
{
try
{
return(await fileDownloader.DownloadFileWithResumeAsync(url, fileLocation).TimeoutAfter(shellService.Settings.TimeOut));
}
catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70)
{
// Disk Full, HRESULT: -2147024784 == 0xFFFFFFFF80070070
Logger.Error("Downloader:DownloadBinaryFile: {0}", ex);
shellService.ShowError(ex, Resources.DiskFull);
crawlerService.StopCommand.Execute(null);
return(false);
}
catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x20)
{
// The process cannot access the file because it is being used by another process.", HRESULT: -2147024864 == 0xFFFFFFFF80070020
return(true);
}
catch (WebException webException) when((webException.Response != null))
{
var webRespStatusCode = (int)((HttpWebResponse)webException?.Response).StatusCode;
if (webRespStatusCode >= 400 && webRespStatusCode < 600) // removes inaccessible files: http status codes 400 to 599
{
try { File.Delete(fileLocation); } // could be open again in a different thread
catch { }
}
return(false);
}
catch
{
return(false);
}
}
private async Task LoadLibrary()
{
Logger.Verbose("ManagerController.LoadLibrary:Start");
managerService.BlogFiles.Clear();
string path = Path.Combine(shellService.Settings.DownloadLocation, "Index");
try
{
if (Directory.Exists(path))
{
{
IReadOnlyList <IBlog> files = await GetIBlogsAsync(path);
foreach (IBlog file in files)
{
managerService.BlogFiles.Add(file);
}
BlogManagerFinishedLoadingLibrary?.Invoke(this, EventArgs.Empty);
}
}
}
catch (Exception ex)
{
Logger.Verbose("ManagerController:LoadLibrary: {0}", ex);
shellService.ShowError(ex, Resources.CouldNotLoadLibrary, ex.Data["Filename"]);
}
Logger.Verbose("ManagerController.LoadLibrary:End");
}
public override async Task UpdateMetaInformationAsync()
{
try
{
if (blog.Online)
{
string document = await GetSvcPageAsync("1", "0");
var response = ConvertJsonToClass <TumblrJson>(document);
if (response.meta.status == 200)
{
blog.Title = response.response.posts.FirstOrDefault().blog.title;
blog.Description = response.response.posts.FirstOrDefault().blog.description;
}
}
}
catch (WebException webException)
{
if (webException.Message.Contains("503"))
{
Logger.Error("TumblrDownloader:GetUrlsAsync: {0}", "User not logged in");
shellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, blog.Name);
return;
}
}
}
private IReadOnlyList <IBlog> GetIBlogsCore(string directory)
{
Logger.Verbose("ManagerController:GetIBlogsCore Start");
var blogs = new List <IBlog>();
var failedToLoadBlogs = new List <string>();
string[] supportedFileTypes = Enum.GetNames(typeof(BlogTypes)).ToArray();
foreach (string filename in Directory.GetFiles(directory, "*").Where(
fileName => supportedFileTypes.Any(fileName.Contains) &&
!fileName.Contains("_files")))
{
//TODO: Refactor
try
{
if (filename.EndsWith(BlogTypes.tumblr.ToString()))
{
blogs.Add(new TumblrBlog().Load(filename));
}
if (filename.EndsWith(BlogTypes.tmblrpriv.ToString()))
{
blogs.Add(new TumblrHiddenBlog().Load(filename));
}
if (filename.EndsWith(BlogTypes.tlb.ToString()))
{
blogs.Add(new TumblrLikedByBlog().Load(filename));
}
if (filename.EndsWith(BlogTypes.tumblrsearch.ToString()))
{
blogs.Add(new TumblrSearchBlog().Load(filename));
}
if (filename.EndsWith(BlogTypes.tumblrtagsearch.ToString()))
{
blogs.Add(new TumblrTagSearchBlog().Load(filename));
}
}
catch (SerializationException ex)
{
failedToLoadBlogs.Add(ex.Data["Filename"].ToString());
}
}
if (failedToLoadBlogs.Any())
{
string failedBlogNames = failedToLoadBlogs.Aggregate((a, b) => a + ", " + b);
Logger.Verbose("ManagerController:GetIBlogsCore: {0}", failedBlogNames);
_shellService.ShowError(new SerializationException(), Resources.CouldNotLoadLibrary, failedBlogNames);
}
Logger.Verbose("ManagerController.GetIBlogsCore End");
return(blogs);
}
private async Task GetUrlsAsync(IProgress <DownloadProgress> progress, CancellationToken ct, PauseToken pt)
{
var semaphoreSlim = new SemaphoreSlim(shellService.Settings.ParallelScans);
var trackedTasks = new List <Task>();
foreach (int crawlerNumber in Enumerable.Range(1, shellService.Settings.ParallelScans))
{
await semaphoreSlim.WaitAsync();
trackedTasks.Add(new Func <Task>(async() =>
{
var tags = new List <string>();
if (!string.IsNullOrWhiteSpace(blog.Tags))
{
tags = blog.Tags.Split(',').Select(x => x.Trim()).ToList();
}
try
{
string document = await RequestDataAsync(crawlerNumber);
await AddUrlsToDownloadList(document, tags, progress, crawlerNumber, ct, pt);
}
catch (WebException webException)
{
if (webException.Message.Contains("503"))
{
Logger.Error("TumblrDownloader:GetUrlsAsync: {0}", "User not logged in");
shellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, blog.Name);
return;
}
if (webException.Message.Contains("429"))
{
// TODO: add retry logic?
Logger.Error("TumblrDownloader:GetUrls:WebException {0}", webException);
shellService.ShowError(webException, Resources.LimitExceeded, blog.Name);
}
}
catch
{
}
finally
{
semaphoreSlim.Release();
}
})());
}
await Task.WhenAll(trackedTasks);
producerConsumerCollection.CompleteAdding();
if (!ct.IsCancellationRequested)
{
UpdateBlogStats();
}
}
private async Task SaveChangesAsync(MusicFile musicFile)
{
if (musicFile == null)
{
return;
}
IReadOnlyCollection <MusicFile> allFilesToSave;
if (musicFile.SharedMusicFiles.Any())
{
allFilesToSave = musicFile.SharedMusicFiles;
}
else
{
allFilesToSave = new[] { musicFile };
}
// Filter out the music file that is currently playing
var playingMusicFile = PlaylistManager.CurrentItem?.MusicFile;
var filesToSave = allFilesToSave.Except(new[] { playingMusicFile }).ToArray();
foreach (var x in allFilesToSave.Intersect(new[] { playingMusicFile }))
{
musicFilesToSaveAfterPlaying.Add(x);
}
if (!filesToSave.Any())
{
return;
}
var tasks = filesToSave.Select(x => SaveChangesCoreAsync(x)).ToArray();
try
{
await Task.WhenAll(tasks);
}
catch (Exception ex)
{
Log.Default.Error(ex, "SaveChangesAsync");
if (filesToSave.Count() == 1)
{
shellService.ShowError(ex, Resources.CouldNotSaveFile, filesToSave.First().FileName);
}
else
{
shellService.ShowError(ex, Resources.CouldNotSaveFiles);
}
}
finally
{
RemoveMusicFilesToSaveAfterPlaying(filesToSave);
}
}
public virtual async Task IsBlogOnlineAsync()
{
try
{
await RequestDataAsync(blog.Url);
blog.Online = true;
}
catch (WebException webException)
{
Logger.Error("TumblrBlogCrawler:IsBlogOnlineAsync:WebException {0}", webException);
shellService.ShowError(webException, Resources.BlogIsOffline, blog.Name);
blog.Online = false;
}
}
private async Task AppendToTextFileAsync(string fileLocation, T data)
{
try
{
using (var stream = new FileStream(fileLocation, FileMode.Create, FileAccess.Write))
{
using (XmlDictionaryWriter writer = JsonReaderWriterFactory.CreateJsonWriter(
stream, Encoding.UTF8, true, true, " "))
{
var serializer = new DataContractJsonSerializer(data.GetType());
serializer.WriteObject(writer, data);
writer.Flush();
await Task.CompletedTask;
}
}
}
catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70)
{
Logger.Error("TumblrJsonDownloader:AppendToTextFile: {0}", ex);
shellService.ShowError(ex, Resources.DiskFull);
crawlerService.StopCommand.Execute(null);
}
catch
{
}
}
public async Task ConfirmPrivacyConsentAsync()
{
try
{
await PerformPrivacyConsentRequestAsync();
}
catch (TimeoutException timeoutException)
{
Logger.Error("{0}, {1}", string.Format(CultureInfo.CurrentCulture, Resources.TimeoutReachedShort, Resources.ConfirmingTumblrPrivacyConsent), timeoutException);
shellService.ShowError(timeoutException, Resources.ConfirmingTumblrPrivacyConsentFailed);
}
catch (Exception exception)
{
Logger.Error("{0}, {1}", string.Format(CultureInfo.CurrentCulture, Resources.ConfirmingTumblrPrivacyConsentFailed), exception);
shellService.ShowError(exception, Resources.ConfirmingTumblrPrivacyConsentFailed);
}
}
public virtual async Task IsBlogOnlineAsync()
{
try
{
await RequestDataAsync(blog.Url);
blog.Online = true;
}
catch (WebException webException)
{
if (webException.Status == WebExceptionStatus.RequestCanceled)
{
return;
}
Logger.Error("AbstractCrawler:IsBlogOnlineAsync:WebException {0}", webException);
shellService.ShowError(webException, Resources.BlogIsOffline, blog.Name);
blog.Online = false;
}
catch (TimeoutException timeoutException)
{
HandleTimeoutException(timeoutException, Resources.OnlineChecking);
blog.Online = false;
}
}
private void OpenListCore(string playlistFileName)
{
Playlist playlist;
try
{
var playlistFile = StorageFile.GetFileFromPathAsync(playlistFileName).GetResult();
// MS Issue: LoadAsync cannot load a playlist when one of the files do not exists anymore.
playlist = Playlist.LoadAsync(playlistFile).GetResult();
}
catch (Exception ex)
{
Log.Default.Error(ex, "OpenListCore");
shellService.ShowError(ex, Resources.CouldNotLoadPlaylist);
return;
}
InsertFilesCore(PlaylistManager.Items.Count, playlist.Files.Select(x => x.Path).ToArray());
}
protected virtual async Task <bool> DownloadBinaryFile(string fileLocation, string url, CancellationToken ct)
{
try
{
return(await ThrottledStream.DownloadFileWithResumeAsync(url, fileLocation,
shellService.Settings, ct));
}
catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70)
{
Logger.Error("ManagerController:Download: {0}", ex);
shellService.ShowError(ex, Resources.DiskFull);
crawlerService.StopCommand.Execute(null);
return(false);
}
catch
{
return(false);
}
}
private void OpenListCore(string queuelistFileName)
{
QueueSettings queueList;
try
{
using (var stream = new FileStream(queuelistFileName, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var serializer = new DataContractJsonSerializer(typeof(QueueSettings));
queueList = (QueueSettings)serializer.ReadObject(stream);
}
}
catch (Exception ex)
{
Logger.Error("QueueController:OpenListCore: {0}", ex);
shellService.ShowError(ex, Resources.CouldNotLoadQueuelist);
return;
}
InsertFilesCore(QueueManager.Items.Count(), queueList.Names.ToArray(), queueList.Types.ToArray());
}
public void SetText(string text)
{
try
{
Clipboard.SetText(text);
}
catch (Exception ex)
{
Logger.Error($"ClipboardService:SetText: {ex}");
_shellService.ShowError(new ClipboardContentException(ex), "error setting clipboard content");
}
}
public virtual async Task IsBlogOnlineAsync()
{
try
{
await RequestDataAsync(blog.Url);
blog.Online = true;
}
catch (WebException webException)
{
Logger.Error("AbstractCrawler:IsBlogOnlineAsync:WebException {0}", webException);
shellService.ShowError(webException, Resources.BlogIsOffline, blog.Name);
blog.Online = false;
}
catch (TimeoutException timeoutException)
{
Logger.Error("AbstractCrawler:IsBlogOnlineAsync:WebException {0}", timeoutException);
shellService.ShowError(timeoutException, Resources.TimeoutReached, Resources.OnlineChecking, blog.Name);
blog.Online = false;
}
}
private async Task LoadLibrary()
{
Logger.Verbose("ManagerController.LoadLibrary:Start");
managerService.BlogFiles.Clear();
string path = Path.Combine(shellService.Settings.DownloadLocation, "Index");
try
{
if (Directory.Exists(path))
{
{
IReadOnlyList <IBlog> files = await GetFilesAsync(path);
foreach (IBlog file in files)
{
managerService.BlogFiles.Add(file);
}
BlogManagerFinishedLoading?.Invoke(this, EventArgs.Empty);
if (shellService.Settings.CheckOnlineStatusAtStartup)
{
foreach (IBlog blog in files)
{
IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService,
crawlerService, blog);
await downloader.IsBlogOnlineAsync();
}
}
}
}
}
catch (Exception ex)
{
Logger.Error("ManagerController:LoadLibrary: {0}", ex);
shellService.ShowError(ex, Resources.CouldNotLoadLibrary, ex.Data["Filename"]);
}
Logger.Verbose("ManagerController.LoadLibrary:End");
}
private async void DeleteFileAsync(string fileName)
{
try
{
var file = await StorageFile.GetFileFromPathAsync(fileName);
await file.DeleteAsync();
}
catch (Exception ex)
{
shellService.ShowError(ex, Resources.CouldNotDeleteFile, fileName);
}
}
private void OpenListCore(string queuelistFileName)
{
List <string> queueList;
try
{
using (FileStream stream = new FileStream(queuelistFileName,
FileMode.Open, FileAccess.Read, FileShare.Read))
{
IFormatter formatter = new BinaryFormatter();
queueList = (List <string>)formatter.Deserialize(stream);
}
}
catch (Exception ex)
{
Logger.Error("OpenListCore: {0}", ex);
shellService.ShowError(ex, Resources.CouldNotLoadQueuelist);
return;
}
InsertFilesCore(QueueManager.Items.Count(), queueList.ToArray());
}
private async void LoadLibrary()
{
Logger.Verbose("ManagerController.UpdateBlogFiles:Start");
selectionService.BlogFiles.Clear();
var path = Path.Combine(shellService.Settings.DownloadLocation, "Index");
try
{
if (Directory.Exists(path))
{
{
var files = await GetFilesAsync(path);
foreach (var file in files)
{
selectionService.BlogFiles.Add(file);
}
if (shellService.Settings.CheckOnlineStatusAtStartup == true)
{
foreach (var file in files)
{
file.Online = await IsBlogOnline(file.Name);
}
}
}
}
}
catch (Exception ex)
{
Logger.Error("ManagerController:LoadLibrary: {0}", ex);
shellService.ShowError(ex, Resources.CouldNotLoadLibrary, ex.Data["Filename"]);
return;
}
Logger.Verbose("ManagerController.LoadLibrary:End");
}
private void InsertFiles(int index, IEnumerable <string> fileNames)
{
MusicFile[] musicFiles;
try
{
musicFiles = fileNames.Select(musicFileContext.Create).ToArray();
}
catch (Exception ex)
{
Log.Default.Error(ex, "TranscodingController.InsertFile");
shellService.ShowError(ex, Resources.CouldNotOpenFiles);
return;
}
Transcode(GetMusicFilesSupportedToConvert(musicFiles).ToArray());
}
protected AbstractDownloader(IShellService shellService, IManagerService managerService, CancellationToken ct, PauseToken pt, IProgress <DownloadProgress> progress, IPostQueue <AbstractPost> postQueue, FileDownloader fileDownloader, ICrawlerService crawlerService = null, IBlog blog = null, IFiles files = null)
{
this.shellService = shellService;
this.crawlerService = crawlerService;
this.managerService = managerService;
this.blog = blog;
this.files = files;
this.ct = ct;
this.pt = pt;
this.progress = progress;
this.postQueue = postQueue;
this.fileDownloader = fileDownloader;
Progress <Exception> prog = new Progress <Exception>((e) => shellService.ShowError(e, Resources.CouldNotSaveBlog, blog.Name));
_saveTimer = new Timer(_ => OnSaveTimedEvent(prog), null, SAVE_TIMESPAN_SECS * 1000, SAVE_TIMESPAN_SECS * 1000);
}
private IFiles LoadFiles(IBlog blog)
{
if (settings.LoadAllDatabases)
{
var files = managerService.Databases.FirstOrDefault(file => file.Name.Equals(blog.Name) && file.BlogType.Equals(blog.OriginalBlogType));
if (files == null)
{
var s = string.Format("{0} ({1})", blog.Name, blog.BlogType);
Logger.Error(Resources.CouldNotLoadLibrary, s);
shellService.ShowError(new KeyNotFoundException(), Resources.CouldNotLoadLibrary, s);
throw new KeyNotFoundException(s);
}
return(files);
}
return(Files.Load(blog.ChildId));
}
private async Task AppendToTextFile(string fileLocation, XContainer data)
{
try
{
using (var sw = new StreamWriter(fileLocation, true))
{
await sw.WriteAsync(PrettyXml(data));
}
}
catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70)
{
Logger.Error("TumblrXmlDownloader:AppendToTextFile: {0}", ex);
shellService.ShowError(ex, Resources.DiskFull);
crawlerService.StopCommand.Execute(null);
}
catch
{
}
}
private async Task AppendToTextFileAsync(string fileLocation, T data)
{
try
{
if (typeof(T) == typeof(DataModels.TumblrSearchJson.Datum) || typeof(T) == typeof(DataModels.Twitter.TimelineTweets.Tweet))
{
var serializer = new JsonSerializer();
using (StreamWriter sw = new StreamWriter(fileLocation, false))
using (JsonWriter writer = new JsonTextWriter(sw)
{
Formatting = Newtonsoft.Json.Formatting.Indented
})
{
serializer.Serialize(writer, data);
}
}
else
{
using (var stream = new FileStream(fileLocation, FileMode.Create, FileAccess.Write))
{
using (XmlDictionaryWriter writer = JsonReaderWriterFactory.CreateJsonWriter(
stream, Encoding.UTF8, true, true, " "))
{
var serializer = new DataContractJsonSerializer(data.GetType());
serializer.WriteObject(writer, data);
writer.Flush();
}
}
}
await Task.CompletedTask;
}
catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70)
{
Logger.Error("TumblrJsonDownloader:AppendToTextFile: {0}", ex);
shellService.ShowError(ex, Resources.DiskFull);
crawlerService.StopCommand.Execute(null);
}
catch
{
}
}
private async Task GetUrlsAsync(IProgress <DownloadProgress> progress, CancellationToken ct, PauseToken pt)
{
var semaphoreSlim = new SemaphoreSlim(shellService.Settings.ParallelScans);
var trackedTasks = new List <Task>();
foreach (int crawlerNumber in Enumerable.Range(0, shellService.Settings.ParallelScans))
{
await semaphoreSlim.WaitAsync();
trackedTasks.Add(new Func <Task>(async() =>
{
try
{
string document = await RequestDataAsync(blog.Url + "/page/" + crawlerNumber);
if (!CheckIfLoggedIn(document))
{
Logger.Error("TumblrLikedByDownloader:GetUrlsAsync: {0}", "User not logged in");
shellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, blog.Name);
return;
}
await AddUrlsToDownloadList(document, progress, crawlerNumber, ct, pt);
}
catch (WebException)
{
}
finally
{
semaphoreSlim.Release();
}
})());
}
await Task.WhenAll(trackedTasks);
producerConsumerCollection.CompleteAdding();
if (!ct.IsCancellationRequested)
{
UpdateBlogStats();
}
}
public ManagerViewModel(IManagerView view, IShellService shellService, Lazy <ISelectionService> selectionService, ICrawlerService crawlerService) : base(view)
{
ShellService = shellService;
this.selectionService = selectionService;
this.crawlerService = crawlerService;
try
{
if (shellService.Settings.ColumnWidths.Count != 0)
{
view.DataGridColumnRestore = ShellService.Settings.ColumnWidths;
}
}
catch (Exception ex)
{
Logger.Error("ManagerController: {0}", ex);
shellService.ShowError(ex, Resources.CouldNotRestoreUISettings);
return;
}
ShellService.Closing += ViewClosed;
}
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
{
while (true)
{
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
bool lockTaken = false;
Monitor.Enter(_lockObject, ref lockTaken);
try
{
if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count)
{
QueueListItem nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First();
IBlog blog = nextQueueItem.Blog;
ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct);
crawler.IsBlogOnlineAsync().Wait(4000);
crawler.Dispose();
if (_crawlerService.ActiveItems.Any(item =>
item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType)))
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
if (!nextQueueItem.Blog.Online)
{
QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
Monitor.Exit(_lockObject);
continue;
}
_crawlerService.AddActiveItems(nextQueueItem);
Monitor.Exit(_lockObject);
lockTaken = false;
await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct);
}
else
{
Monitor.Exit(_lockObject);
lockTaken = false;
await Task.Delay(4000, ct);
}
}
catch (Exception e)
{
Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e);
_shellService.ShowError(e, "Error starting the next item in the queue.");
if (lockTaken)
{
Monitor.Exit(_lockObject);
}
}
}
}
public static void ShowError(this IShellService shellService, Exception exception, string format, params object[] args) => shellService.ShowError(exception, string.Format(CultureInfo.CurrentCulture, format, args));
private async Task <Tuple <ulong, bool> > GetUrlsAsync(IProgress <DownloadProgress> progress, CancellationToken ct, PauseToken pt)
{
var semaphoreSlim = new SemaphoreSlim(shellService.Settings.ParallelScans);
var trackedTasks = new List <Task>();
var numberOfPostsCrawled = 0;
var apiLimitHit = false;
var completeGrab = true;
ulong lastId = GetLastPostId();
await UpdateTotalPostCount();
int totalPosts = blog.Posts;
ulong highestId = await GetHighestPostId();
foreach (int pageNumber in GetPageNumbers())
{
await semaphoreSlim.WaitAsync();
if (!completeGrab)
{
break;
}
if (ct.IsCancellationRequested)
{
break;
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
trackedTasks.Add(new Func <Task>(async() =>
{
try
{
XDocument document = await GetApiPageAsync(pageNumber);
completeGrab = CheckPostAge(document, lastId);
var tags = new List <string>();
if (!string.IsNullOrWhiteSpace(blog.Tags))
{
tags = blog.Tags.Split(',').Select(x => x.Trim()).ToList();
}
AddUrlsToDownloadList(document, tags);
}
catch (WebException webException)
{
if (webException.Message.Contains("429"))
{
// TODO: add retry logic?
apiLimitHit = true;
Logger.Error("TumblrDownloader:GetUrls:WebException {0}", webException);
shellService.ShowError(webException, Resources.LimitExceeded, blog.Name);
}
}
finally
{
semaphoreSlim.Release();
}
numberOfPostsCrawled += blog.PageSize;
UpdateProgressQueueInformation(progress, Resources.ProgressGetUrlLong, numberOfPostsCrawled, totalPosts);
})());
}
await Task.WhenAll(trackedTasks);
producerConsumerCollection.CompleteAdding();
if (!ct.IsCancellationRequested && completeGrab)
{
UpdateBlogStats();
}
return(new Tuple <ulong, bool>(highestId, apiLimitHit));
}
