public void ProcessEvents([NotNull] Timestamp indexingStartTimestamp, [NotNull] Timestamp indexingFinishTimestamp) { if (indexingStartTimestamp >= indexingFinishTimestamp) { logger.Info("IndexingFinishTimestamp is reached: {IndexingFinishTimestamp}", new { IndexingFinishTimestamp = indexingFinishTimestamp }); return; } logger.Info("Processing events from {IndexingStartTimestamp} to {IndexingFinishTimestamp}", new { IndexingStartTimestamp = indexingStartTimestamp, IndexingFinishTimestamp = indexingFinishTimestamp }); Timestamp lastEventsBatchStartTimestamp = null; var taskIdsToProcess = new HashSet <string>(); var taskIdsToProcessInChronologicalOrder = new List <string>(); EventsQueryResult <TaskMetaUpdatedEvent, string> eventsQueryResult; var fromOffsetExclusive = offsetInterpreter.GetMaxOffsetForTimestamp(indexingStartTimestamp.AddTicks(-1)); var toOffsetInclusive = offsetInterpreter.GetMaxOffsetForTimestamp(indexingFinishTimestamp); do { eventsQueryResult = eventSource.GetEvents(fromOffsetExclusive, toOffsetInclusive, estimatedCount: 10000); foreach (var @event in eventsQueryResult.Events) { if (taskIdsToProcess.Add(@event.Event.TaskId)) { taskIdsToProcessInChronologicalOrder.Add(@event.Event.TaskId); } var eventTimestamp = new Timestamp(@event.Event.Ticks); if (lastEventsBatchStartTimestamp == null) { lastEventsBatchStartTimestamp = eventTimestamp; } if (eventTimestamp - lastEventsBatchStartTimestamp > indexerSettings.MaxEventsProcessingTimeWindow || taskIdsToProcessInChronologicalOrder.Count > indexerSettings.MaxEventsProcessingTasksCount) { taskMetaProcessor.ProcessTasks(taskIdsToProcessInChronologicalOrder); taskIdsToProcess.Clear(); taskIdsToProcessInChronologicalOrder.Clear(); lastEventsBatchStartTimestamp = null; } } fromOffsetExclusive = eventsQueryResult.LastOffset; } while (!eventsQueryResult.NoMoreEventsInSource); if (taskIdsToProcessInChronologicalOrder.Any()) { taskMetaProcessor.ProcessTasks(taskIdsToProcessInChronologicalOrder); } }
public Message MergePending(Message gatewayData) { Id = gatewayData.Id; Attachments = gatewayData.Attachments; Embeds = gatewayData.Embeds; Mentions = gatewayData.Mentions; MentionEveryone = gatewayData.MentionEveryone; MentionRoles = gatewayData.MentionRoles; if (gatewayData.User.Id != null) { User = gatewayData.User; } if (gatewayData.Timestamp.Ticks > 100000) { Timestamp = gatewayData.Timestamp; } else { Timestamp.AddTicks(gatewayData.Timestamp.Ticks); } return(this); }