public async Task <ChattyThread> GetThreadTree(int postId)
{
var chatty = Chatty;
if (chatty != null && chatty.ThreadsByReplyId.TryGetValue(postId, out var thread))
{
return(thread);
}
else
{
return(Cortex.DetectCortexThread(await _threadParser.GetThreadTree(postId)));
}
}
public async Task <(ChattyThread, ThreadLolCounts)> GetThreadAndLols(int postId)
{
var chatty = Chatty;
ChattyThread thread;
ThreadLolCounts threadLolCounts;
if (chatty != null && chatty.ThreadsByReplyId.TryGetValue(postId, out thread))
{
threadLolCounts =
LolCounts?.GetThreadLolCounts(thread.Posts[0].Id)
?? ThreadLolCounts.Empty;
}
else
{
thread = await _threadParser.GetThread(postId);
threadLolCounts = await _lolParser.DownloadThreadLolCounts(thread);
}
Cortex.DetectCortexThread(thread);
return(thread, threadLolCounts);
}
private async Task <TimeSpan> Scrape()
{
var stopwatch = new StepStopwatch();
try
{
var oldEventId = await _eventProvider.GetLastEventId();
stopwatch.Step(nameof(GC));
GC.Collect();
stopwatch.Step(nameof(_lolParser.DownloadChattyLolCounts));
var lolTask = _lolParser.DownloadChattyLolCounts(_state?.LolJson, _state?.LolCounts);
var(newPages, newChatty) = await GetChattyWithoutBodies(_state?.Pages, stopwatch);
if (_state != null)
{
stopwatch.Step(nameof(HandleThreadsThatDisappeared));
await HandleThreadsThatDisappeared(_state.Chatty, newChatty);
}
stopwatch.Step(nameof(ReorderThreads));
ReorderThreads(newChatty);
stopwatch.Step(nameof(newChatty.SetDictionaries));
newChatty.SetDictionaries();
stopwatch.Step(nameof(CopyPostBodies));
CopyPostBodies(newChatty);
stopwatch.Step(nameof(DownloadPostBodies));
await DownloadPostBodies(newChatty);
stopwatch.Step(nameof(RemovePostsWithNoBody));
RemovePostsWithNoBody(newChatty);
stopwatch.Step(nameof(Cortex.DetectCortexThreads));
Cortex.DetectCortexThreads(newChatty);
stopwatch.Step(nameof(FixRelativeLinks));
FixRelativeLinks(newChatty);
stopwatch.Step(nameof(_chattyProvider.Update));
var(lolJson, lolCounts) = await lolTask;
await _eventProvider.Update(newChatty, lolCounts);
_chattyProvider.Update(newChatty, lolCounts);
_state =
new ScrapeState
{
Chatty = newChatty,
Pages = newPages,
LolJson = lolJson,
LolCounts = lolCounts
};
var newEventId = await _eventProvider.GetLastEventId();
if (oldEventId != newEventId)
{
stopwatch.Step(nameof(SaveState));
await SaveState();
}
ThreadPool.GetMaxThreads(out var maxWorkerThreads, out var maxCompletionPortThreads);
ThreadPool.GetAvailableThreads(out var availableWorkerThreads, out var availableCompletionPortThreads);
_logger.LogInformation("Scrape complete. Last event is #{EventId}. {Elapsed}. Worker threads: {WorkerCount}. IOCP threads: {CompletionPortCount}.",
await _eventProvider.GetLastEventId(), stopwatch, maxWorkerThreads - availableWorkerThreads,
maxCompletionPortThreads - availableCompletionPortThreads);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Scrape failed. {stopwatch}. {ex.Message}");
}
return(stopwatch.Elapsed);
}