public async Task <ChattyThread> GetThreadTree(int postId) { var chatty = Chatty; if (chatty != null && chatty.ThreadsByReplyId.TryGetValue(postId, out var thread)) { return(thread); } else { return(Cortex.DetectCortexThread(await _threadParser.GetThreadTree(postId))); } }
public async Task <(ChattyThread, ThreadLolCounts)> GetThreadAndLols(int postId) { var chatty = Chatty; ChattyThread thread; ThreadLolCounts threadLolCounts; if (chatty != null && chatty.ThreadsByReplyId.TryGetValue(postId, out thread)) { threadLolCounts = LolCounts?.GetThreadLolCounts(thread.Posts[0].Id) ?? ThreadLolCounts.Empty; } else { thread = await _threadParser.GetThread(postId); threadLolCounts = await _lolParser.DownloadThreadLolCounts(thread); } Cortex.DetectCortexThread(thread); return(thread, threadLolCounts); }
private async Task <TimeSpan> Scrape() { var stopwatch = new StepStopwatch(); try { var oldEventId = await _eventProvider.GetLastEventId(); stopwatch.Step(nameof(GC)); GC.Collect(); stopwatch.Step(nameof(_lolParser.DownloadChattyLolCounts)); var lolTask = _lolParser.DownloadChattyLolCounts(_state?.LolJson, _state?.LolCounts); var(newPages, newChatty) = await GetChattyWithoutBodies(_state?.Pages, stopwatch); if (_state != null) { stopwatch.Step(nameof(HandleThreadsThatDisappeared)); await HandleThreadsThatDisappeared(_state.Chatty, newChatty); } stopwatch.Step(nameof(ReorderThreads)); ReorderThreads(newChatty); stopwatch.Step(nameof(newChatty.SetDictionaries)); newChatty.SetDictionaries(); stopwatch.Step(nameof(CopyPostBodies)); CopyPostBodies(newChatty); stopwatch.Step(nameof(DownloadPostBodies)); await DownloadPostBodies(newChatty); stopwatch.Step(nameof(RemovePostsWithNoBody)); RemovePostsWithNoBody(newChatty); stopwatch.Step(nameof(Cortex.DetectCortexThreads)); Cortex.DetectCortexThreads(newChatty); stopwatch.Step(nameof(FixRelativeLinks)); FixRelativeLinks(newChatty); stopwatch.Step(nameof(_chattyProvider.Update)); var(lolJson, lolCounts) = await lolTask; await _eventProvider.Update(newChatty, lolCounts); _chattyProvider.Update(newChatty, lolCounts); _state = new ScrapeState { Chatty = newChatty, Pages = newPages, LolJson = lolJson, LolCounts = lolCounts }; var newEventId = await _eventProvider.GetLastEventId(); if (oldEventId != newEventId) { stopwatch.Step(nameof(SaveState)); await SaveState(); } ThreadPool.GetMaxThreads(out var maxWorkerThreads, out var maxCompletionPortThreads); ThreadPool.GetAvailableThreads(out var availableWorkerThreads, out var availableCompletionPortThreads); _logger.LogInformation("Scrape complete. Last event is #{EventId}. {Elapsed}. Worker threads: {WorkerCount}. IOCP threads: {CompletionPortCount}.", await _eventProvider.GetLastEventId(), stopwatch, maxWorkerThreads - availableWorkerThreads, maxCompletionPortThreads - availableCompletionPortThreads); } catch (Exception ex) { _logger.LogError(ex, $"Scrape failed. {stopwatch}. {ex.Message}"); } return(stopwatch.Elapsed); }