protected override void PerformWork(IList <ArraySegment <Message> > items, out int totalCount, out object state)
{
totalCount = 0;
lock (_cursors)
{
// perf sensitive: re-use cursors array to minimize allocations
if ((_cursorsState == null) || (_cursorsState.Length != _cursors.Count))
{
_cursorsState = new ulong[_cursors.Count];
}
for (int i = 0; i < _cursors.Count; i++)
{
MessageStoreResult <Message> storeResult = _cursorTopics[i].Store.GetMessages(_cursors[i].Id, MaxMessages);
_cursorsState[i] = storeResult.FirstMessageId + (ulong)storeResult.Messages.Count;
if (storeResult.Messages.Count > 0)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
}
}
// Return the state as a list of cursors
state = _cursorsState;
}
}
private void Initialize(MessageStoreResult <ScaleoutMapping> result)
{
_result = result;
_offset = _result.Messages.Offset - 1;
_length = _result.Messages.Offset + _result.Messages.Count;
_nextId = _result.FirstMessageId + (ulong)_result.Messages.Count;
}
public bool MoveNext()
{
_offset++;
if (_offset < _length)
{
return(true);
}
if (!_result.HasMoreData)
{
return(false);
}
// If the store falls out of scope
var store = (ScaleoutStore)_storeReference.Target;
if (store == null)
{
return(false);
}
// Get the next result
MessageStoreResult <ScaleoutMapping> result = store.GetMessages(_nextId);
Initialize(result);
_offset++;
return(_offset < _length);
}
private ulong ExtractMessages(int streamIndex, ScaleoutMapping mapping, IList <ArraySegment <Message> > items, ref int totalCount)
{
// For each of the event keys we care about, extract all of the messages
// from the payload
lock (EventKeys)
{
for (var i = 0; i < EventKeys.Count; ++i)
{
string eventKey = EventKeys[i];
for (int j = 0; j < mapping.LocalKeyInfo.Count; j++)
{
LocalEventKeyInfo info = mapping.LocalKeyInfo[j];
// Capture info.MessageStore because it could be GC'd while we're working with it.
var messageStore = info.MessageStore;
if (messageStore != null && info.Key.Equals(eventKey, StringComparison.OrdinalIgnoreCase))
{
MessageStoreResult <Message> storeResult = messageStore.GetMessages(info.Id, 1);
if (storeResult.Messages.Count > 0)
{
// TODO: Figure out what to do when we have multiple event keys per mapping
Message message = storeResult.Messages.Array[storeResult.Messages.Offset];
// Only add the message to the list if the stream index matches
if (message.StreamIndex == streamIndex)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
_trace.TraceVerbose("Adding {0} message(s) for mapping id: {1}, event key: '{2}', event id: {3}, streamIndex: {4}",
storeResult.Messages.Count, mapping.Id, info.Key, info.Id, streamIndex);
// We got a mapping id bigger than what we expected which
// means we missed messages. Use the new mappingId.
if (message.MappingId > mapping.Id)
{
_trace.TraceEvent(TraceEventType.Verbose, 0, $"Extracted additional messages, updating cursor to new Mapping ID: {message.MappingId}");
return(message.MappingId);
}
}
else
{
// REVIEW: When the stream indexes don't match should we leave the mapping id as is?
// If we do nothing then we'll end up querying old cursor ids until
// we eventually find a message id that matches this stream index.
_trace.TraceInformation(
"Stream index mismatch. Mapping id: {0}, event key: '{1}', event id: {2}, message.StreamIndex: {3}, streamIndex: {4}",
mapping.Id, info.Key, info.Id, message.StreamIndex, streamIndex);
}
}
}
}
}
}
return(mapping.Id);
}
protected override void PerformWork(IList <ArraySegment <Message> > items, out int totalCount, out object state)
{
// The list of cursors represent (streamid, payloadid)
var nextCursors = new ulong[_stores.Count];
totalCount = 0;
for (var i = 0; i < _stores.Count; ++i)
{
// Get the mapping for this stream
ScaleoutMappingStore store = _stores[i];
// See if we have a cursor for this key
Cursor cursor = _cursors[i];
nextCursors[i] = cursor.Id;
// Try to find a local mapping for this payload
IEnumerator <ScaleoutMapping> enumerator = store.GetEnumerator(cursor.Id);
// Stop if we got more than max messages
while (totalCount < MaxMessages && enumerator.MoveNext())
{
ScaleoutMapping mapping = enumerator.Current;
// For each of the event keys we care about, extract all of the messages
// from the payload
lock (EventKeys)
{
for (var j = 0; j < EventKeys.Count; ++j)
{
string eventKey = EventKeys[j];
for (int k = 0; k < mapping.LocalKeyInfo.Count; k++)
{
LocalEventKeyInfo info = mapping.LocalKeyInfo[k];
if (info.Key.Equals(eventKey, StringComparison.OrdinalIgnoreCase))
{
MessageStoreResult <Message> storeResult = info.MessageStore.GetMessages(info.Id, 1);
if (storeResult.Messages.Count > 0)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
}
}
}
}
}
// Update the cursor id
nextCursors[i] = mapping.Id;
}
}
state = nextCursors;
}
private ulong ExtractMessages(int streamIndex, ScaleoutMapping mapping, IList <ArraySegment <Message> > items, ref int totalCount)
{
// For each of the event keys we care about, extract all of the messages
// from the payload
lock (EventKeys)
{
for (var i = 0; i < EventKeys.Count; ++i)
{
string eventKey = EventKeys[i];
for (int j = 0; j < mapping.LocalKeyInfo.Count; j++)
{
LocalEventKeyInfo info = mapping.LocalKeyInfo[j];
if (info.MessageStore != null && info.Key.Equals(eventKey, StringComparison.OrdinalIgnoreCase))
{
MessageStoreResult <Message> storeResult = info.MessageStore.GetMessages(info.Id, 1);
if (storeResult.Messages.Count > 0)
{
// TODO: Figure out what to do when we have multiple event keys per mapping
Message message = storeResult.Messages.Array[storeResult.Messages.Offset];
// Only add the message to the list if the stream index matches
if (message.StreamIndex == streamIndex)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
// We got a mapping id bigger than what we expected which
// means we missed messages. Use the new mappingId.
if (message.MappingId > mapping.Id)
{
return(message.MappingId);
}
}
else
{
// REVIEW: When the stream indexes don't match should we leave the mapping id as is?
// If we do nothing then we'll end up querying old cursor ids until
// we eventually find a message id that matches this stream index.
}
}
}
}
}
}
return(mapping.Id);
}
protected override void PerformWork(IList <ArraySegment <Message> > items, out int totalCount, out object state)
{
totalCount = 0;
lock (_cursors)
{
var cursors = new ulong[_cursors.Count];
for (int i = 0; i < _cursors.Count; i++)
{
MessageStoreResult <Message> storeResult = _cursorTopics[i].Store.GetMessages(_cursors[i].Id, MaxMessages);
cursors[i] = storeResult.FirstMessageId + (ulong)storeResult.Messages.Count;
if (storeResult.Messages.Count > 0)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
}
}
// Return the state as a list of cursors
state = cursors;
}
}
public ScaleoutStoreEnumerator(ScaleoutStore store, MessageStoreResult <ScaleoutMapping> result)
: this()
{
_storeReference = new WeakReference(store);
Initialize(result);
}
public IEnumerator <ScaleoutMapping> GetEnumerator(ulong id)
{
MessageStoreResult <ScaleoutMapping> result = _store.GetMessagesByMappingId(id);
return(new ScaleoutStoreEnumerator(_store, result));
}