protected override void PerformWork(ref List <ArraySegment <Message> > items, out string nextCursor, ref int totalCount, out object state)
{
var cursors = new List <Cursor>();
lock (_lockObj)
{
items = new List <ArraySegment <Message> >(_cursors.Count);
for (int i = 0; i < _cursors.Count; i++)
{
Cursor cursor = Cursor.Clone(_cursors[i]);
cursors.Add(cursor);
MessageStoreResult <Message> storeResult = _cursorTopics[i].Store.GetMessages(cursor.Id, MaxMessages);
ulong next = storeResult.FirstMessageId + (ulong)storeResult.Messages.Count;
cursor.Id = next;
if (storeResult.Messages.Count > 0)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
}
}
nextCursor = Cursor.MakeCursor(cursors);
// Return the state as a list of cursors
state = cursors;
}
}
public override string GetCursor()
{
return(Cursor.MakeCursor(_cursors));
}
private void WorkImpl(ConcurrentDictionary <string, Topic> topics, TaskCompletionSource <object> taskCompletionSource)
{
Process:
int totalCount = 0;
string nextCursor = null;
List <ArraySegment <Message> > items = null;
lock (_lockObj)
{
items = new List <ArraySegment <Message> >(Cursors.Count);
for (int i = 0; i < Cursors.Count; i++)
{
Cursor cursor = Cursors[i];
MessageStoreResult <Message> storeResult = cursor.Topic.Store.GetMessages(cursor.Id, _maxMessages);
ulong next = storeResult.FirstMessageId + (ulong)storeResult.Messages.Count;
cursor.Id = next;
if (storeResult.Messages.Count > 0)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
}
}
nextCursor = Cursor.MakeCursor(Cursors);
}
if (Alive && items.Count > 0)
{
var messageResult = new MessageResult(items, nextCursor, totalCount);
Task <bool> callbackTask = Invoke(messageResult);
if (callbackTask.IsCompleted)
{
try
{
// Make sure exceptions propagate
callbackTask.Wait();
if (callbackTask.Result)
{
// Sync path
goto Process;
}
else
{
// If the callback said it's done then stop
taskCompletionSource.TrySetResult(null);
}
}
catch (Exception ex)
{
taskCompletionSource.TrySetException(ex);
}
}
else
{
WorkImplAsync(callbackTask, topics, taskCompletionSource);
}
}
else
{
taskCompletionSource.TrySetResult(null);
}
}
protected override void PerformWork(ref List <ArraySegment <Message> > items, out string nextCursor, ref int totalCount, out object state)
{
// The list of cursors represent (streamid, payloadid)
var cursors = new List <Cursor>();
foreach (var stream in _streams)
{
// Get the mapping for this stream
Linktionary <ulong, ScaleoutMapping> mapping = stream.Value;
// See if we have a cursor for this key
Cursor cursor = null;
// REVIEW: We should optimize this
int index = _cursors.FindIndex(c => c.Key == stream.Key);
bool consumed = true;
if (index != -1)
{
cursor = Cursor.Clone(_cursors[index]);
// If there's no node for this cursor id it's likely because we've
// had an app domain restart and the cursor position is now invalid.
if (mapping[cursor.Id] == null)
{
// Set it to the first id in this mapping
cursor.Id = stream.Value.MinKey;
// Mark this cursor as unconsumed
consumed = false;
}
}
else
{
// Create a cursor and add it to the list.
// Point the Id to the first value
cursor = new Cursor
{
Id = stream.Value.MinKey,
Key = stream.Key
};
consumed = false;
}
cursors.Add(cursor);
// Try to find a local mapping for this payload
LinkedListNode <KeyValuePair <ulong, ScaleoutMapping> > node = mapping[cursor.Id];
// Skip this node only if this isn't a new cursor
if (node != null && consumed)
{
// Skip this node since we've already consumed it
node = node.Next;
}
while (node != null)
{
KeyValuePair <ulong, ScaleoutMapping> pair = node.Value;
// Stop if we got more than max messages
if (totalCount >= MaxMessages)
{
break;
}
// It should be ok to lock here since groups aren't modified that often
lock (EventKeys)
{
// For each of the event keys we care about, extract all of the messages
// from the payload
foreach (var eventKey in EventKeys)
{
LocalEventKeyInfo info;
if (pair.Value.EventKeyMappings.TryGetValue(eventKey, out info))
{
int maxMessages = Math.Min(info.Count, MaxMessages);
MessageStoreResult <Message> storeResult = info.Store.GetMessages(info.MinLocal, maxMessages);
if (storeResult.Messages.Count > 0)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
}
}
}
}
// Update the cursor id
cursor.Id = pair.Key;
node = node.Next;
}
}
nextCursor = Cursor.MakeCursor(cursors);
state = cursors;
}
/// <summary>
///
/// </summary>
/// <param name="subscriber"></param>
/// <param name="cursor"></param>
/// <param name="callback"></param>
/// <returns></returns>
public IDisposable Subscribe(ISubscriber subscriber, string cursor, Func <MessageResult, Task <bool> > callback, int messageBufferSize)
{
IEnumerable <Cursor> cursors = null;
if (cursor == null)
{
cursors = from key in subscriber.EventKeys
select new Cursor
{
Key = key,
Id = GetMessageId(key)
};
}
else
{
cursors = Cursor.GetCursors(cursor);
}
var subscription = new Subscription(subscriber.Identity, cursors, callback, messageBufferSize);
var topics = new HashSet <Topic>();
foreach (var key in subscriber.EventKeys)
{
Topic topic = _topics.GetOrAdd(key, _ => new Topic());
// Set the subscription for this topic
subscription.SetCursorTopic(key, topic);
// Add it to the list of topics
topics.Add(topic);
}
foreach (var topic in topics)
{
topic.AddSubscription(subscription);
}
if (!String.IsNullOrEmpty(cursor))
{
// Update all of the cursors so we're within the range
foreach (var pair in subscription.Cursors)
{
Topic topic;
if (_topics.TryGetValue(pair.Key, out topic) && pair.Id > topic.Store.GetMessageCount())
{
subscription.UpdateCursor(pair.Key, 0);
}
}
}
Action <string, string> eventAdded = (eventKey, eventCursor) =>
{
Topic topic = _topics.GetOrAdd(eventKey, _ => new Topic());
// Get the cursor for this event key
ulong id = eventCursor == null ? 0 : UInt64.Parse(eventCursor);
// Add or update the cursor (in case it already exists)
subscription.AddOrUpdateCursor(eventKey, id, topic);
// Add it to the list of subs
topic.AddSubscription(subscription);
};
Action <string> eventRemoved = eventKey => RemoveEvent(subscription, eventKey);
subscriber.EventAdded += eventAdded;
subscriber.EventRemoved += eventRemoved;
// If there's a cursor then schedule work for this subscription
if (!String.IsNullOrEmpty(cursor))
{
_engine.Schedule(subscription);
}
return(new DisposableAction(() =>
{
// This will stop work from continuting to happen
subscription.Dispose();
subscriber.EventAdded -= eventAdded;
subscriber.EventRemoved -= eventRemoved;
string currentCursor = Cursor.MakeCursor(subscription.Cursors);
foreach (var eventKey in subscriber.EventKeys)
{
RemoveEvent(subscription, eventKey);
}
subscription.Invoke(new MessageResult(currentCursor));
}));
}
