public static Chatty DetectCortexThreads(Chatty chatty)
{
foreach (var thread in chatty.Threads)
{
DetectCortexThread(thread);
}
return(chatty);
}
public async Task PrePopulate(Chatty newChatty, ChattyLolCounts newChattyLolCounts, List <EventModel> events)
{
await _lock.WithWriteLock(nameof(PrePopulate),
action : () =>
{
_chatty = newChatty;
_chattyLolCounts = newChattyLolCounts;
_events.Clear();
_events.AddRange(events);
});
}
public async Task Start(Chatty newChatty, ChattyLolCounts newChattyLolCounts)
{
if (_chatty != null || _chattyLolCounts != null)
{
throw new InvalidOperationException($"{nameof(EventProvider)} already started.");
}
await _lock.WithWriteLock(nameof(Start),
action : () =>
{
_chatty = newChatty;
_chattyLolCounts = newChattyLolCounts;
});
}
private async Task <(List <ScrapeState.Page> Pages, Chatty Chatty)> GetChattyWithoutBodies(
List <ScrapeState.Page> previousPages, StepStopwatch stopwatch)
{
ScrapeState.Page GetPreviousPage(int page) =>
previousPages != null && previousPages.Count >= page
? previousPages[page - 1]
: null;
var chatty = new Chatty {
Threads = new List <ChattyThread>(200)
};
var currentPage = 1;
var lastPage = 1;
var newPages = new List <ScrapeState.Page>();
var seenThreadIds = new HashSet <int>();
// The chatty almost always has three or fewer pages, so try downloading the first three in parallel for speed.
stopwatch.Step("First three pages");
var firstThreePages = new (string Html, ChattyPage Page)[3];
public async Task Update(Chatty newChatty, ChattyLolCounts newChattyLolCounts)
{
var oldChatty = await _lock.WithReadLock(nameof(Update), func : () => _chatty);
if (oldChatty != null)
{
// safety first, this should never throw if the scraper has done its job correctly
foreach (var oldThread in oldChatty.Threads)
{
if (!newChatty.ThreadsByRootId.ContainsKey(oldThread.ThreadId) &&
!newChatty.NukedThreadIds.Contains(oldThread.ThreadId) &&
!newChatty.ExpiredThreadIds.Contains(oldThread.ThreadId))
{
throw new Exception($"Thread ID {oldThread.ThreadId} disappeared but is not listed as expired or nuked!");
}
}
}
await _lock.WithWriteLock(nameof(Update),
action : () =>
{
if (_chatty == null || _chattyLolCounts == null)
{
_chatty = newChatty;
_chattyLolCounts = newChattyLolCounts;
return;
}
var nextEventId = _events.Count == 0 ? 1 : _events.Last().EventId + 1;
var newEvents = new List <EventModel>();
foreach (var oldThreadId in
from thread in _chatty.Threads
where !newChatty.ThreadsByRootId.ContainsKey(thread.ThreadId)
select thread.ThreadId)
{
// was this thread nuked or did it expire?
if (newChatty.NukedThreadIds.Contains(oldThreadId))
{
// nuked thread
newEvents.Add(
new EventModel
{
EventId = nextEventId++,
EventDate = DateTimeOffset.Now,
EventType = EventType.CategoryChange,
EventData = new CategoryChangeEventDataModel
{
PostId = oldThreadId,
Category = ModerationFlag.Nuked
}
});
}
}
foreach (var newThread in
from thread in newChatty.Threads
where !_chatty.ThreadsByRootId.ContainsKey(thread.ThreadId)
select thread)
{
// new thread
var newPostModels = GetPostModelsById(newThread, newChattyLolCounts);
foreach (var newPost in newPostModels.Values.OrderBy(x => x.Id))
{
newEvents.Add(
new EventModel
{
EventId = nextEventId++,
EventDate = DateTimeOffset.Now,
EventType = EventType.NewPost,
EventData = new NewPostEventDataModel
{
PostId = newPost.Id,
Post = newPost,
ParentAuthor = newPost.ParentId == 0 ? "" : newPostModels[newPost.ParentId].Author
}
});
}
}
foreach (var newThread in newChatty.Threads)
{
if (!_chatty.ThreadsByRootId.TryGetValue(newThread.ThreadId, out var oldThread))
{
continue;
}
var oldPostModels = GetPostModelsById(oldThread, _chattyLolCounts);
var newPostModels = GetPostModelsById(newThread, newChattyLolCounts);
foreach (var oldPost in
from postPair in oldPostModels
where !newPostModels.ContainsKey(postPair.Key)
orderby postPair.Key
select postPair.Value)
{
// nuked post
newEvents.Add(
new EventModel
{
EventId = nextEventId++,
EventDate = DateTimeOffset.Now,
EventType = EventType.CategoryChange,
EventData = new CategoryChangeEventDataModel
{
PostId = oldPost.Id,
Category = ModerationFlag.Nuked
}
});
}
foreach (var newPost in
from postPair in newPostModels
where !oldPostModels.ContainsKey(postPair.Key)
orderby postPair.Key
select postPair.Value)
{
// new post
newEvents.Add(
new EventModel
{
EventId = nextEventId++,
EventDate = DateTimeOffset.Now,
EventType = EventType.NewPost,
EventData = new NewPostEventDataModel
{
PostId = newPost.Id,
Post = newPost,
ParentAuthor = newPost.ParentId == 0 ? "" : newPostModels[newPost.ParentId].Author
}
});
}
foreach (var newPost in newPostModels.Values)
{
if (!oldPostModels.TryGetValue(newPost.Id, out var oldPost))
{
continue;
}
if (oldPost.Category != newPost.Category)
{
newEvents.Add(
new EventModel
{
EventId = nextEventId++,
EventDate = DateTimeOffset.Now,
EventType = EventType.CategoryChange,
EventData = new CategoryChangeEventDataModel
{
PostId = newPost.Id,
Category = newPost.Category
}
});
}
if (!Enumerable.SequenceEqual(
oldPost.Lols.OrderBy(x => x.Tag),
newPost.Lols.OrderBy(x => x.Tag)))
{
newEvents.Add(
new EventModel
{
EventId = nextEventId++,
EventDate = DateTimeOffset.Now,
EventType = EventType.LolCountsUpdate,
EventData = new LolCountsUpdateEventDataModel
{
Updates = (
from x in newPost.Lols
select new LolCountUpdateModel
{
PostId = newPost.Id,
Tag = x.Tag,
Count = x.Count
}).ToList()
}
});
}
if (oldPost.Author != newPost.Author) // detect deleted users
{
_logger.LogInformation($"postChange: {oldPost.Author} -> {newPost.Author}.");
newEvents.Add(
new EventModel
{
EventId = nextEventId++,
EventDate = DateTimeOffset.Now,
EventType = EventType.PostChange,
EventData = new PostChangeEventDataModel
{
PostId = newPost.Id
}
});
}
if (oldPost.IsFrozen != newPost.IsFrozen)
{
_logger.LogInformation($"Post {newPost.Id} freeze change: {oldPost.IsFrozen} -> {newPost.IsFrozen}.");
newEvents.Add(
new EventModel
{
EventId = nextEventId++,
EventDate = DateTimeOffset.Now,
EventType = EventType.PostFreezeChange,
EventData = new PostFrozenChangeEventDataModel
{
PostId = newPost.Id,
Frozen = newPost.IsFrozen
}
}
);
}
}
}
_chatty = newChatty;
_chattyLolCounts = newChattyLolCounts;
_events.AddRange(newEvents);
while (_events.Count > MAX_EVENTS)
{
_events.RemoveAt(0);
}
});
}
public void Update(Chatty chatty, ChattyLolCounts lolCounts)
{
Chatty = chatty;
LolCounts = lolCounts;
}
