private async Task <bool> GetUrlsAsync() { trackedTasks = new List <Task>(); semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans); GenerateTags(); // page already loaded in GetHighestPostIdCoreAsync(), so retrieve new number of posts already there await Task.Run(() => Task.CompletedTask); //await UpdateTotalPostCountAsync(); foreach (int pageNumber in GetPageNumbers()) { await semaphoreSlim.WaitAsync(); if (!completeGrab) { break; } if (CheckIfShouldStop()) { break; } CheckIfShouldPause(); trackedTasks.Add(CrawlPageAsync(pageNumber)); } await Task.WhenAll(trackedTasks); PostQueue.CompleteAdding(); jsonQueue.CompleteAdding(); UpdateBlogStats(GetLastPostId() != 0); return(incompleteCrawl); }
private async Task <bool> GetUrlsAsync() { semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans); trackedTasks = new List <Task>(); GenerateTags(); foreach (int pageNumber in GetPageNumbers()) { await semaphoreSlim.WaitAsync(); trackedTasks.Add(CrawlPageAsync(pageNumber)); } await Task.WhenAll(trackedTasks); jsonQueue.CompleteAdding(); PostQueue.CompleteAdding(); UpdateBlogStats(GetLastPostId() != 0); return(incompleteCrawl); }
private async Task <bool> GetUrlsAsync() { trackedTasks = new List <Task>(); semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans); GenerateTags(); await UpdateTotalPostCountAsync(); foreach (int pageNumber in GetPageNumbers()) { await semaphoreSlim.WaitAsync(); if (!completeGrab) { break; } if (CheckIfShouldStop()) { break; } CheckIfShouldPause(); trackedTasks.Add(CrawlPageAsync(pageNumber)); } await Task.WhenAll(trackedTasks); PostQueue.CompleteAdding(); jsonQueue.CompleteAdding(); UpdateBlogStats(); return(incompleteCrawl); }
private async Task <bool> GetUrlsAsync() { trackedTasks = new List <Task>(); semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans); GenerateTags(); await IsBlogOnlineAsync(); if (!Blog.Online) { PostQueue.CompleteAdding(); jsonQueue.CompleteAdding(); return(true); } Blog.Posts = twUser.Data.User.Legacy.StatusesCount; if (Blog.PageSize == 0) { Blog.PageSize = 50; } int currentPage = (Blog.Posts > 3200) ? (Blog.Posts - 3200) / 20 + 3200 / Blog.PageSize + 1 : Blog.Posts / Blog.PageSize + 1; if (Blog.Posts > 3200) { currentPage += 50; } int pageNo = 1; while (true) { await semaphoreSlim.WaitAsync(); if (!completeGrab) { break; } if (CheckIfShouldStop()) { break; } CheckIfShouldPause(); await CrawlPageAsync(pageNo); if (currentPage > 0) { currentPage--; pageNo++; } else { break; } } PostQueue.CompleteAdding(); jsonQueue.CompleteAdding(); UpdateBlogStats(GetLastPostId() != 0); return(incompleteCrawl); }