private async Task <bool> GetUrlsAsync()
{
trackedTasks = new List <Task>();
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
GenerateTags();
// page already loaded in GetHighestPostIdCoreAsync(), so retrieve new number of posts already there
await Task.Run(() => Task.CompletedTask);
//await UpdateTotalPostCountAsync();
foreach (int pageNumber in GetPageNumbers())
{
await semaphoreSlim.WaitAsync();
if (!completeGrab)
{
break;
}
if (CheckIfShouldStop())
{
break;
}
CheckIfShouldPause();
trackedTasks.Add(CrawlPageAsync(pageNumber));
}
await Task.WhenAll(trackedTasks);
PostQueue.CompleteAdding();
jsonQueue.CompleteAdding();
UpdateBlogStats(GetLastPostId() != 0);
return(incompleteCrawl);
}
private async Task <bool> GetUrlsAsync()
{
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
trackedTasks = new List <Task>();
GenerateTags();
foreach (int pageNumber in GetPageNumbers())
{
await semaphoreSlim.WaitAsync();
trackedTasks.Add(CrawlPageAsync(pageNumber));
}
await Task.WhenAll(trackedTasks);
jsonQueue.CompleteAdding();
PostQueue.CompleteAdding();
UpdateBlogStats(GetLastPostId() != 0);
return(incompleteCrawl);
}
private async Task <bool> GetUrlsAsync()
{
trackedTasks = new List <Task>();
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
GenerateTags();
await UpdateTotalPostCountAsync();
foreach (int pageNumber in GetPageNumbers())
{
await semaphoreSlim.WaitAsync();
if (!completeGrab)
{
break;
}
if (CheckIfShouldStop())
{
break;
}
CheckIfShouldPause();
trackedTasks.Add(CrawlPageAsync(pageNumber));
}
await Task.WhenAll(trackedTasks);
PostQueue.CompleteAdding();
jsonQueue.CompleteAdding();
UpdateBlogStats();
return(incompleteCrawl);
}
private async Task <bool> GetUrlsAsync()
{
trackedTasks = new List <Task>();
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
GenerateTags();
await IsBlogOnlineAsync();
if (!Blog.Online)
{
PostQueue.CompleteAdding();
jsonQueue.CompleteAdding();
return(true);
}
Blog.Posts = twUser.Data.User.Legacy.StatusesCount;
if (Blog.PageSize == 0)
{
Blog.PageSize = 50;
}
int currentPage = (Blog.Posts > 3200) ? (Blog.Posts - 3200) / 20 + 3200 / Blog.PageSize + 1 : Blog.Posts / Blog.PageSize + 1;
if (Blog.Posts > 3200)
{
currentPage += 50;
}
int pageNo = 1;
while (true)
{
await semaphoreSlim.WaitAsync();
if (!completeGrab)
{
break;
}
if (CheckIfShouldStop())
{
break;
}
CheckIfShouldPause();
await CrawlPageAsync(pageNo);
if (currentPage > 0)
{
currentPage--;
pageNo++;
}
else
{
break;
}
}
PostQueue.CompleteAdding();
jsonQueue.CompleteAdding();
UpdateBlogStats(GetLastPostId() != 0);
return(incompleteCrawl);
}
