protected override void PerformWork(IList <ArraySegment <Message> > items, out int totalCount, out object state)
{
// The list of cursors represent (streamid, payloadid)
var nextCursors = new ulong?[_cursors.Count];
totalCount = 0;
// Get the enumerator so that we can extract messages for this subscription
IEnumerator <Tuple <ScaleoutMapping, int> > enumerator = GetMappings().GetEnumerator();
while (totalCount < MaxMessages && enumerator.MoveNext())
{
ScaleoutMapping mapping = enumerator.Current.Item1;
int streamIndex = enumerator.Current.Item2;
ulong?nextCursor = nextCursors[streamIndex];
// Only keep going with this stream if the cursor we're looking at is bigger than
// anything we already processed
if (nextCursor == null || mapping.Id > nextCursor)
{
ulong mappingId = ExtractMessages(streamIndex, mapping, items, ref totalCount);
// Update the cursor id
nextCursors[streamIndex] = mappingId;
}
}
state = nextCursors;
}
public bool TrySearch(ulong id, out int index)
{
int low = 0;
int high = Length;
while (low <= high)
{
int mid = (low + high) / 2;
ScaleoutMapping mapping = Data[mid];
if (id < mapping.Id)
{
high = mid - 1;
}
else if (id > mapping.Id)
{
low = mid + 1;
}
else if (id == mapping.Id)
{
index = mid;
return(true);
}
}
index = -1;
return(false);
}
private ulong ExtractMessages(int streamIndex, ScaleoutMapping mapping, IList <ArraySegment <Message> > items, ref int totalCount)
{
// For each of the event keys we care about, extract all of the messages
// from the payload
lock (EventKeys)
{
for (var i = 0; i < EventKeys.Count; ++i)
{
string eventKey = EventKeys[i];
for (int j = 0; j < mapping.LocalKeyInfo.Count; j++)
{
LocalEventKeyInfo info = mapping.LocalKeyInfo[j];
// Capture info.MessageStore because it could be GC'd while we're working with it.
var messageStore = info.MessageStore;
if (messageStore != null && info.Key.Equals(eventKey, StringComparison.OrdinalIgnoreCase))
{
MessageStoreResult <Message> storeResult = messageStore.GetMessages(info.Id, 1);
if (storeResult.Messages.Count > 0)
{
// TODO: Figure out what to do when we have multiple event keys per mapping
Message message = storeResult.Messages.Array[storeResult.Messages.Offset];
// Only add the message to the list if the stream index matches
if (message.StreamIndex == streamIndex)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
_trace.TraceVerbose("Adding {0} message(s) for mapping id: {1}, event key: '{2}', event id: {3}, streamIndex: {4}",
storeResult.Messages.Count, mapping.Id, info.Key, info.Id, streamIndex);
// We got a mapping id bigger than what we expected which
// means we missed messages. Use the new mappingId.
if (message.MappingId > mapping.Id)
{
_trace.TraceEvent(TraceEventType.Verbose, 0, $"Extracted additional messages, updating cursor to new Mapping ID: {message.MappingId}");
return(message.MappingId);
}
}
else
{
// REVIEW: When the stream indexes don't match should we leave the mapping id as is?
// If we do nothing then we'll end up querying old cursor ids until
// we eventually find a message id that matches this stream index.
_trace.TraceInformation(
"Stream index mismatch. Mapping id: {0}, event key: '{1}', event id: {2}, message.StreamIndex: {3}, streamIndex: {4}",
mapping.Id, info.Key, info.Id, message.StreamIndex, streamIndex);
}
}
}
}
}
}
return(mapping.Id);
}
protected override void PerformWork(IList <ArraySegment <Message> > items, out int totalCount, out object state)
{
// The list of cursors represent (streamid, payloadid)
var nextCursors = new ulong[_stores.Count];
totalCount = 0;
for (var i = 0; i < _stores.Count; ++i)
{
// Get the mapping for this stream
ScaleoutMappingStore store = _stores[i];
// See if we have a cursor for this key
Cursor cursor = _cursors[i];
nextCursors[i] = cursor.Id;
// Try to find a local mapping for this payload
IEnumerator <ScaleoutMapping> enumerator = store.GetEnumerator(cursor.Id);
// Stop if we got more than max messages
while (totalCount < MaxMessages && enumerator.MoveNext())
{
ScaleoutMapping mapping = enumerator.Current;
// For each of the event keys we care about, extract all of the messages
// from the payload
lock (EventKeys)
{
for (var j = 0; j < EventKeys.Count; ++j)
{
string eventKey = EventKeys[j];
for (int k = 0; k < mapping.LocalKeyInfo.Count; k++)
{
LocalEventKeyInfo info = mapping.LocalKeyInfo[k];
if (info.Key.Equals(eventKey, StringComparison.OrdinalIgnoreCase))
{
MessageStoreResult <Message> storeResult = info.MessageStore.GetMessages(info.Id, 1);
if (storeResult.Messages.Count > 0)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
}
}
}
}
}
// Update the cursor id
nextCursors[i] = mapping.Id;
}
}
state = nextCursors;
}
private IEnumerable <Tuple <ScaleoutMapping, int> > GetMappings()
{
var enumerators = new List <CachedStreamEnumerator>();
var singleStream = _streams.Count == 1;
for (var streamIndex = 0; streamIndex < _streams.Count; ++streamIndex)
{
// Get the mapping for this stream
ScaleoutMappingStore store = _streams[streamIndex];
Cursor cursor = _cursors[streamIndex];
// Try to find a local mapping for this payload
var enumerator = new CachedStreamEnumerator(store.GetEnumerator(cursor.Id),
streamIndex);
enumerators.Add(enumerator);
}
var counter = 0;
while (enumerators.Count > 0)
{
ScaleoutMapping minMapping = null;
CachedStreamEnumerator minEnumerator = null;
for (int i = enumerators.Count - 1; i >= 0; i--)
{
counter += 1;
CachedStreamEnumerator enumerator = enumerators[i];
ScaleoutMapping mapping;
if (enumerator.TryMoveNext(out mapping))
{
if (minMapping == null || mapping.ServerCreationTime < minMapping.ServerCreationTime)
{
minMapping = mapping;
minEnumerator = enumerator;
}
}
else
{
enumerators.RemoveAt(i);
}
}
if (minMapping != null)
{
minEnumerator.ClearCachedValue();
yield return(Tuple.Create(minMapping, minEnumerator.StreamIndex));
}
}
_trace.TraceEvent(TraceEventType.Verbose, 0, $"End of mappings (connection ID: {Identity}). Total mappings processed: {counter}");
}
private ulong ExtractMessages(int streamIndex, ScaleoutMapping mapping, IList <ArraySegment <Message> > items, ref int totalCount)
{
// For each of the event keys we care about, extract all of the messages
// from the payload
lock (EventKeys)
{
for (var i = 0; i < EventKeys.Count; ++i)
{
string eventKey = EventKeys[i];
for (int j = 0; j < mapping.LocalKeyInfo.Count; j++)
{
LocalEventKeyInfo info = mapping.LocalKeyInfo[j];
if (info.MessageStore != null && info.Key.Equals(eventKey, StringComparison.OrdinalIgnoreCase))
{
MessageStoreResult <Message> storeResult = info.MessageStore.GetMessages(info.Id, 1);
if (storeResult.Messages.Count > 0)
{
// TODO: Figure out what to do when we have multiple event keys per mapping
Message message = storeResult.Messages.Array[storeResult.Messages.Offset];
// Only add the message to the list if the stream index matches
if (message.StreamIndex == streamIndex)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
// We got a mapping id bigger than what we expected which
// means we missed messages. Use the new mappingId.
if (message.MappingId > mapping.Id)
{
return(message.MappingId);
}
}
else
{
// REVIEW: When the stream indexes don't match should we leave the mapping id as is?
// If we do nothing then we'll end up querying old cursor ids until
// we eventually find a message id that matches this stream index.
}
}
}
}
}
}
return(mapping.Id);
}
private IEnumerable <Tuple <ScaleoutMapping, int> > GetMappings()
{
var enumerators = new List <CachedStreamEnumerator>();
for (var streamIndex = 0; streamIndex < _streams.Count; ++streamIndex)
{
// Get the mapping for this stream
ScaleoutMappingStore store = _streams[streamIndex];
Cursor cursor = _cursors[streamIndex];
// Try to find a local mapping for this payload
var enumerator = new CachedStreamEnumerator(store.GetEnumerator(cursor.Id),
streamIndex);
enumerators.Add(enumerator);
}
while (enumerators.Count > 0)
{
ScaleoutMapping minMapping = null;
CachedStreamEnumerator minEnumerator = null;
for (int i = enumerators.Count - 1; i >= 0; i--)
{
CachedStreamEnumerator enumerator = enumerators[i];
ScaleoutMapping mapping;
if (enumerator.TryMoveNext(out mapping))
{
if (minMapping == null || mapping.ServerCreationTime < minMapping.ServerCreationTime)
{
minMapping = mapping;
minEnumerator = enumerator;
}
}
else
{
enumerators.RemoveAt(i);
}
}
if (minMapping != null)
{
minEnumerator.ClearCachedValue();
yield return(Tuple.Create(minMapping, minEnumerator.StreamIndex));
}
}
}
private void AddCursorForStream(int streamIndex, List <Cursor> cursors)
{
ScaleoutMapping maxMapping = _streams[streamIndex].MaxMapping;
ulong id = UInt64.MaxValue;
string key = streamIndex.ToString(CultureInfo.InvariantCulture);
if (maxMapping != null)
{
id = maxMapping.Id;
}
var newCursor = new Cursor(key, id);
cursors.Add(newCursor);
}
// Adds a message to the store. Returns the ID of the newly added message.
public ulong Add(ScaleoutMapping mapping)
{
// keep looping in TryAddImpl until it succeeds
ulong newMessageId;
while (!TryAddImpl(mapping, out newMessageId)) ;
// When TryAddImpl succeeds, record the fact that a message was just added to the
// store. We increment the next free id rather than set it explicitly since
// multiple threads might be trying to write simultaneously. There is a nifty
// side effect to this: _nextFreeMessageId will *always* return the total number
// of messages that *all* threads agree have ever been added to the store. (The
// actual number may be higher, but this field will eventually catch up as threads
// flush data.)
Interlocked.Increment(ref _nextFreeMessageId);
return newMessageId;
}
public ScaleoutSubscription(string identity,
IList <string> eventKeys,
string cursor,
IList <ScaleoutMappingStore> streams,
Func <MessageResult, object, Task <bool> > callback,
int maxMessages,
IPerformanceCounterManager counters,
object state)
: base(identity, eventKeys, callback, maxMessages, counters, state)
{
if (streams == null)
{
throw new ArgumentNullException("streams");
}
_streams = streams;
List <Cursor> cursors = null;
if (String.IsNullOrEmpty(cursor))
{
cursors = new List <Cursor>(streams.Count);
for (int i = 0; i < streams.Count; i++)
{
ScaleoutMapping maxMapping = streams[i].MaxMapping;
ulong id = UInt64.MaxValue;
string key = i.ToString(CultureInfo.InvariantCulture);
if (maxMapping != null)
{
id = maxMapping.Id;
}
var newCursor = new Cursor(key, id);
cursors.Add(newCursor);
}
}
else
{
cursors = Cursor.GetCursors(cursor);
}
_cursors = cursors;
}
public bool TryMoveNext(out ScaleoutMapping mapping)
{
mapping = null;
if (_cachedValue != null)
{
mapping = _cachedValue;
return(true);
}
if (_enumerator.MoveNext())
{
mapping = _enumerator.Current;
_cachedValue = mapping;
return(true);
}
return(false);
}
// Adds a message to the store. Returns the ID of the newly added message.
public ulong Add(ScaleoutMapping mapping)
{
// keep looping in TryAddImpl until it succeeds
ulong newMessageId;
while (!TryAddImpl(mapping, out newMessageId))
{
;
}
// When TryAddImpl succeeds, record the fact that a message was just added to the
// store. We increment the next free id rather than set it explicitly since
// multiple threads might be trying to write simultaneously. There is a nifty
// side effect to this: _nextFreeMessageId will *always* return the total number
// of messages that *all* threads agree have ever been added to the store. (The
// actual number may be higher, but this field will eventually catch up as threads
// flush data.)
Interlocked.Increment(ref _nextFreeMessageId);
return(newMessageId);
}
protected Task OnReceived(string streamId, ulong id, IList<Message> messages)
{
// Create a local dictionary for this payload
var dictionary = new ConcurrentDictionary<string, LocalEventKeyInfo>();
foreach (var m in messages)
{
// Get the payload info
var info = dictionary.GetOrAdd(m.Key, _ => new LocalEventKeyInfo());
// Save the min and max for this payload for later
ulong localId = Save(m);
// Set the topic pointer for this event key so we don't need to look it up later
info.Store = Topics[m.Key].Store;
info.MinLocal = Math.Min(localId, info.MinLocal);
info.Count++;
}
// Create the mapping for this payload
var mapping = new ScaleoutMapping(dictionary);
// Get the stream for this payload
var stream = _streams.GetOrAdd(streamId, _ => new IndexedDictionary<ulong, ScaleoutMapping>());
// Publish only after we've setup the mapping fully
stream.Add(id, mapping);
// Schedule after we're done
foreach (var eventKey in dictionary.Keys)
{
ScheduleEvent(eventKey);
}
return TaskAsyncHelper.Empty;
}
private bool TryAddImpl(ScaleoutMapping mapping, out ulong newMessageId)
{
ulong nextFreeMessageId = (ulong)Volatile.Read(ref _nextFreeMessageId);
// locate the fragment containing the next free id, which is where we should write
ulong fragmentNum;
int idxIntoFragmentsArray, idxIntoFragment;
GetFragmentOffsets(nextFreeMessageId, out fragmentNum, out idxIntoFragmentsArray, out idxIntoFragment);
Fragment fragment = _fragments[idxIntoFragmentsArray];
if (fragment == null || fragment.FragmentNum < fragmentNum)
{
// the fragment is outdated (or non-existent) and must be replaced
bool overwrite = fragment != null && fragment.FragmentNum < fragmentNum;
if (idxIntoFragment == 0)
{
// this thread is responsible for creating the fragment
Fragment newFragment = new Fragment(fragmentNum, _fragmentSize);
newFragment.Data[0] = mapping;
Fragment existingFragment = Interlocked.CompareExchange(ref _fragments[idxIntoFragmentsArray], newFragment, fragment);
if (existingFragment == fragment)
{
newMessageId = GetMessageId(fragmentNum, offset: 0);
newFragment.MinId = newMessageId;
newFragment.Length = 1;
newFragment.MaxId = GetMessageId(fragmentNum, offset: _fragmentSize - 1);
_maxMapping = mapping;
// Move the minimum id when we overwrite
if (overwrite)
{
_minMessageId = (long)(existingFragment.MaxId + 1);
_minMappingId = existingFragment.MaxId;
}
else if (idxIntoFragmentsArray == 0)
{
_minMappingId = mapping.Id;
}
return(true);
}
}
// another thread is responsible for updating the fragment, so fall to bottom of method
}
else if (fragment.FragmentNum == fragmentNum)
{
// the fragment is valid, and we can just try writing into it until we reach the end of the fragment
ScaleoutMapping[] fragmentData = fragment.Data;
for (int i = idxIntoFragment; i < fragmentData.Length; i++)
{
ScaleoutMapping originalMapping = Interlocked.CompareExchange(ref fragmentData[i], mapping, null);
if (originalMapping == null)
{
newMessageId = GetMessageId(fragmentNum, offset: (uint)i);
fragment.Length++;
_maxMapping = fragmentData[i];
return(true);
}
}
// another thread used the last open space in this fragment, so fall to bottom of method
}
// failure; caller will retry operation
newMessageId = 0;
return(false);
}
public bool TryMoveNext(out ScaleoutMapping mapping)
{
mapping = null;
if (_cachedValue != null)
{
mapping = _cachedValue;
return true;
}
if (_enumerator.MoveNext())
{
mapping = _enumerator.Current;
_cachedValue = mapping;
return true;
}
return false;
}
public void ClearCachedValue()
{
_cachedValue = null;
}
private ulong ExtractMessages(int streamIndex, ScaleoutMapping mapping, IList<ArraySegment<Message>> items, ref int totalCount)
{
// For each of the event keys we care about, extract all of the messages
// from the payload
lock (EventKeys)
{
for (var i = 0; i < EventKeys.Count; ++i)
{
string eventKey = EventKeys[i];
for (int j = 0; j < mapping.LocalKeyInfo.Count; j++)
{
LocalEventKeyInfo info = mapping.LocalKeyInfo[j];
if (info.MessageStore != null && info.Key.Equals(eventKey, StringComparison.OrdinalIgnoreCase))
{
MessageStoreResult<Message> storeResult = info.MessageStore.GetMessages(info.Id, 1);
if (storeResult.Messages.Count > 0)
{
// TODO: Figure out what to do when we have multiple event keys per mapping
Message message = storeResult.Messages.Array[storeResult.Messages.Offset];
// Only add the message to the list if the stream index matches
if (message.StreamIndex == streamIndex)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
// We got a mapping id bigger than what we expected which
// means we missed messages. Use the new mappingId.
if (message.MappingId > mapping.Id)
{
return message.MappingId;
}
}
else
{
// REVIEW: When the stream indexes don't match should we leave the mapping id as is?
// If we do nothing then we'll end up querying old cursor ids until
// we eventually find a message id that matches this stream index.
}
}
}
}
}
}
return mapping.Id;
}
public void ClearCachedValue()
{
_cachedValue = null;
}
private bool TryAddImpl(ScaleoutMapping mapping, out ulong newMessageId)
{
ulong nextFreeMessageId = (ulong)Volatile.Read(ref _nextFreeMessageId);
// locate the fragment containing the next free id, which is where we should write
ulong fragmentNum;
int idxIntoFragmentsArray, idxIntoFragment;
GetFragmentOffsets(nextFreeMessageId, out fragmentNum, out idxIntoFragmentsArray, out idxIntoFragment);
Fragment fragment = _fragments[idxIntoFragmentsArray];
if (fragment == null || fragment.FragmentNum < fragmentNum)
{
// the fragment is outdated (or non-existent) and must be replaced
bool overwrite = fragment != null && fragment.FragmentNum < fragmentNum;
if (idxIntoFragment == 0)
{
// this thread is responsible for creating the fragment
Fragment newFragment = new Fragment(fragmentNum, _fragmentSize);
newFragment.Data[0] = mapping;
Fragment existingFragment = Interlocked.CompareExchange(ref _fragments[idxIntoFragmentsArray], newFragment, fragment);
if (existingFragment == fragment)
{
newMessageId = GetMessageId(fragmentNum, offset: 0);
newFragment.MinId = newMessageId;
newFragment.Length = 1;
newFragment.MaxId = GetMessageId(fragmentNum, offset: _fragmentSize - 1);
_maxMapping = mapping;
// Move the minimum id when we overwrite
if (overwrite)
{
_minMessageId = (long)(existingFragment.MaxId + 1);
_minMappingId = existingFragment.MaxId;
}
else if (idxIntoFragmentsArray == 0)
{
_minMappingId = mapping.Id;
}
return true;
}
}
// another thread is responsible for updating the fragment, so fall to bottom of method
}
else if (fragment.FragmentNum == fragmentNum)
{
// the fragment is valid, and we can just try writing into it until we reach the end of the fragment
ScaleoutMapping[] fragmentData = fragment.Data;
for (int i = idxIntoFragment; i < fragmentData.Length; i++)
{
ScaleoutMapping originalMapping = Interlocked.CompareExchange(ref fragmentData[i], mapping, null);
if (originalMapping == null)
{
newMessageId = GetMessageId(fragmentNum, offset: (uint)i);
fragment.Length++;
_maxMapping = fragmentData[i];
return true;
}
}
// another thread used the last open space in this fragment, so fall to bottom of method
}
// failure; caller will retry operation
newMessageId = 0;
return false;
}
