private ulong ExtractMessages(int streamIndex, ScaleoutMapping mapping, IList <ArraySegment <Message> > items, ref int totalCount) { // For each of the event keys we care about, extract all of the messages // from the payload lock (EventKeys) { for (var i = 0; i < EventKeys.Count; ++i) { string eventKey = EventKeys[i]; for (int j = 0; j < mapping.LocalKeyInfo.Count; j++) { LocalEventKeyInfo info = mapping.LocalKeyInfo[j]; // Capture info.MessageStore because it could be GC'd while we're working with it. var messageStore = info.MessageStore; if (messageStore != null && info.Key.Equals(eventKey, StringComparison.OrdinalIgnoreCase)) { MessageStoreResult <Message> storeResult = messageStore.GetMessages(info.Id, 1); if (storeResult.Messages.Count > 0) { // TODO: Figure out what to do when we have multiple event keys per mapping Message message = storeResult.Messages.Array[storeResult.Messages.Offset]; // Only add the message to the list if the stream index matches if (message.StreamIndex == streamIndex) { items.Add(storeResult.Messages); totalCount += storeResult.Messages.Count; _trace.TraceVerbose("Adding {0} message(s) for mapping id: {1}, event key: '{2}', event id: {3}, streamIndex: {4}", storeResult.Messages.Count, mapping.Id, info.Key, info.Id, streamIndex); // We got a mapping id bigger than what we expected which // means we missed messages. Use the new mappingId. if (message.MappingId > mapping.Id) { _trace.TraceEvent(TraceEventType.Verbose, 0, $"Extracted additional messages, updating cursor to new Mapping ID: {message.MappingId}"); return(message.MappingId); } } else { // REVIEW: When the stream indexes don't match should we leave the mapping id as is? // If we do nothing then we'll end up querying old cursor ids until // we eventually find a message id that matches this stream index. _trace.TraceInformation( "Stream index mismatch. Mapping id: {0}, event key: '{1}', event id: {2}, message.StreamIndex: {3}, streamIndex: {4}", mapping.Id, info.Key, info.Id, message.StreamIndex, streamIndex); } } } } } } return(mapping.Id); }
private void OnReceivedCore(int streamIndex, ulong id, ScaleoutMessage scaleoutMessage) { Counters.ScaleoutMessageBusMessagesReceivedPerSec.IncrementBy(scaleoutMessage.Messages.Count); _trace.TraceInformation("OnReceived({0}, {1}, {2})", streamIndex, id, scaleoutMessage.Messages.Count); TraceScaleoutMessages(id, scaleoutMessage); var localMapping = new LocalEventKeyInfo[scaleoutMessage.Messages.Count]; var keys = new HashSet <string>(); for (var i = 0; i < scaleoutMessage.Messages.Count; ++i) { Message message = scaleoutMessage.Messages[i]; // Remember where this message came from message.MappingId = id; message.StreamIndex = streamIndex; keys.Add(message.Key); ulong localId = Save(message); _trace.TraceVerbose("Message id: {0}, stream : {1}, eventKey: '{2}' saved with local id: {3}", id, streamIndex, message.Key, localId); MessageStore <Message> messageStore = Topics[message.Key].Store; localMapping[i] = new LocalEventKeyInfo(message.Key, localId, messageStore); } // Get the stream for this payload ScaleoutMappingStore store = StreamManager.Streams[streamIndex]; // Publish only after we've setup the mapping fully store.Add(id, scaleoutMessage, localMapping); if (_trace.Switch.ShouldTrace(TraceEventType.Verbose)) { _trace.TraceVerbose("Scheduling eventkeys: {0}", string.Join(",", keys)); } // Schedule after we're done foreach (var eventKey in keys) { ScheduleEvent(eventKey); } }