public ScaleoutSubscription(string identity,
IList<string> eventKeys,
string cursor,
IList<ScaleoutMappingStore> streams,
Func<MessageResult, object, Task<bool>> callback,
int maxMessages,
IPerformanceCounterManager counters,
object state)
: base(identity, eventKeys, callback, maxMessages, counters, state)
{
if (streams == null)
{
throw new ArgumentNullException("streams");
}
_streams = streams;
List<Cursor> cursors = null;
if (String.IsNullOrEmpty(cursor))
{
cursors = new List<Cursor>(streams.Count);
for (int i = 0; i < streams.Count; i++)
{
ScaleoutMapping maxMapping = streams[i].MaxMapping;
ulong id = UInt64.MaxValue;
string key = i.ToString(CultureInfo.InvariantCulture);
if (maxMapping != null)
{
id = maxMapping.Id;
}
var newCursor = new Cursor(key, id);
cursors.Add(newCursor);
}
}
else
{
cursors = Cursor.GetCursors(cursor);
}
_cursors = cursors;
}
private List<Cursor> GetCursorsFromEventKeys(IList<string> eventKeys, TopicLookup topics)
{
var list = new List<Cursor>(eventKeys.Count);
foreach (var eventKey in eventKeys)
{
var cursor = new Cursor(eventKey, GetMessageId(topics, eventKey), _stringMinifier.Minify(eventKey));
list.Add(cursor);
}
return list;
}
private void AddCursorForStream(int streamIndex, List<Cursor> cursors)
{
ScaleoutMapping maxMapping = _streams[streamIndex].MaxMapping;
ulong id = UInt64.MaxValue;
string key = streamIndex.ToString(CultureInfo.InvariantCulture);
if (maxMapping != null)
{
id = maxMapping.Id;
}
var newCursor = new Cursor(key, id);
cursors.Add(newCursor);
}
public static List<Cursor> GetCursors(string cursor, Func<string, object, string> keyMaximizer, object state)
{
// Technically GetCursors should never be called with a null value, so this is extra cautious
if (String.IsNullOrEmpty(cursor))
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
var signals = new HashSet<string>();
var cursors = new List<Cursor>();
string currentKey = null;
string currentEscapedKey = null;
ulong currentId;
bool escape = false;
bool consumingKey = true;
var sb = new StringBuilder();
var sbEscaped = new StringBuilder();
Cursor parsedCursor;
foreach (var ch in cursor)
{
// escape can only be true if we are consuming the key
if (escape)
{
if (ch != '\\' && ch != ',' && ch != '|')
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
sb.Append(ch);
sbEscaped.Append(ch);
escape = false;
}
else
{
if (ch == '\\')
{
if (!consumingKey)
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
sbEscaped.Append('\\');
escape = true;
}
else if (ch == ',')
{
if (!consumingKey)
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
// For now String.Empty is an acceptable key, but this should change once we verify
// that empty keys cannot be created legitimately.
currentKey = keyMaximizer(sb.ToString(), state);
// If the keyMap cannot find a key, we cannot create an array of cursors.
// This most likely means there was an AppDomain restart or a misbehaving client.
if (currentKey == null)
{
return null;
}
// Don't allow duplicate keys
if (!signals.Add(currentKey))
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
currentEscapedKey = sbEscaped.ToString();
sb.Clear();
sbEscaped.Clear();
consumingKey = false;
}
else if (ch == '|')
{
if (consumingKey)
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
ParseCursorId(sb, out currentId);
parsedCursor = new Cursor(currentKey, currentId, currentEscapedKey);
cursors.Add(parsedCursor);
sb.Clear();
consumingKey = true;
}
else
{
sb.Append(ch);
if (consumingKey)
{
sbEscaped.Append(ch);
}
}
}
}
if (consumingKey)
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
ParseCursorId(sb, out currentId);
parsedCursor = new Cursor(currentKey, currentId, currentEscapedKey);
cursors.Add(parsedCursor);
return cursors;
}
public static Cursor Clone(Cursor cursor)
{
return new Cursor(cursor.Key, cursor.Id, cursor._escapedKey);
}
protected override void PerformWork(IList<ArraySegment<Message>> items, out int totalCount, out object state)
{
// The list of cursors represent (streamid, payloadid)
var cursors = new List<Cursor>();
totalCount = 0;
foreach (var stream in _streams)
{
// Get the mapping for this stream
IndexedDictionary<ulong, ScaleoutMapping> mapping = stream.Value;
// See if we have a cursor for this key
Cursor cursor = null;
// REVIEW: We should optimize this
int index = _cursors.FindIndex(c => c.Key == stream.Key);
bool consumed = true;
if (index != -1)
{
cursor = Cursor.Clone(_cursors[index]);
// If there's no node for this cursor id it's likely because we've
// had an app domain restart and the cursor position is now invalid.
if (mapping[cursor.Id] == null)
{
// Set it to the first id in this mapping
cursor.Id = stream.Value.MinKey;
// Mark this cursor as unconsumed
consumed = false;
}
}
else
{
// Create a cursor and add it to the list.
// Point the Id to the first value
cursor = new Cursor(key: stream.Key, id: stream.Value.MinKey);
consumed = false;
}
cursors.Add(cursor);
// Try to find a local mapping for this payload
LinkedListNode<KeyValuePair<ulong, ScaleoutMapping>> node = mapping[cursor.Id];
// Skip this node only if this isn't a new cursor
if (node != null && consumed)
{
// Skip this node since we've already consumed it
node = node.Next;
}
while (node != null)
{
KeyValuePair<ulong, ScaleoutMapping> pair = node.Value;
// Stop if we got more than max messages
if (totalCount >= MaxMessages)
{
break;
}
// It should be ok to lock here since groups aren't modified that often
lock (EventKeys)
{
// For each of the event keys we care about, extract all of the messages
// from the payload
foreach (var eventKey in EventKeys)
{
LocalEventKeyInfo info;
if (pair.Value.EventKeyMappings.TryGetValue(eventKey, out info))
{
int maxMessages = Math.Min(info.Count, MaxMessages);
MessageStoreResult<Message> storeResult = info.Store.GetMessages(info.MinLocal, maxMessages);
if (storeResult.Messages.Count > 0)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
}
}
}
}
// Update the cursor id
cursor.Id = pair.Key;
node = node.Next;
}
}
state = cursors;
}
public static List<Cursor> GetCursors(string cursor, string prefix, Func<string, object, string> keyMaximizer, object state)
{
// Technically GetCursors should never be called with a null value, so this is extra cautious
if (String.IsNullOrEmpty(cursor))
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
// If the cursor does not begin with the prefix stream, it isn't necessarily a formatting problem.
// The cursor with a different prefix might have had different, but also valid, formatting.
// Null should be returned so new cursors will be generated
if (!cursor.StartsWith(prefix, StringComparison.Ordinal))
{
return null;
}
var signals = new HashSet<string>();
var cursors = new List<Cursor>();
string currentKey = null;
string currentEscapedKey = null;
ulong currentId;
bool escape = false;
bool consumingKey = true;
var sb = new StringBuilder();
var sbEscaped = new StringBuilder();
Cursor parsedCursor;
for (int i = prefix.Length; i < cursor.Length; i++)
{
var ch = cursor[i];
// escape can only be true if we are consuming the key
if (escape)
{
if (ch != '\\' && ch != ',' && ch != '|')
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
sb.Append(ch);
sbEscaped.Append(ch);
escape = false;
}
else
{
if (ch == '\\')
{
if (!consumingKey)
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
sbEscaped.Append('\\');
escape = true;
}
else if (ch == ',')
{
if (!consumingKey)
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
// For now String.Empty is an acceptable key, but this should change once we verify
// that empty keys cannot be created legitimately.
currentKey = keyMaximizer(sb.ToString(), state);
// If the keyMap cannot find a key, we cannot create an array of cursors.
// This most likely means there was an AppDomain restart or a misbehaving client.
if (currentKey == null)
{
return null;
}
// Don't allow duplicate keys
if (!signals.Add(currentKey))
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
currentEscapedKey = sbEscaped.ToString();
sb.Clear();
sbEscaped.Clear();
consumingKey = false;
}
else if (ch == '|')
{
if (consumingKey)
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
ParseCursorId(sb, out currentId);
parsedCursor = new Cursor(currentKey, currentId, currentEscapedKey);
cursors.Add(parsedCursor);
sb.Clear();
consumingKey = true;
}
else
{
sb.Append(ch);
if (consumingKey)
{
sbEscaped.Append(ch);
}
}
}
}
if (consumingKey)
{
throw new FormatException(Resources.Error_InvalidCursorFormat);
}
ParseCursorId(sb, out currentId);
parsedCursor = new Cursor(currentKey, currentId, currentEscapedKey);
cursors.Add(parsedCursor);
return cursors;
}
public void SymmetricWithManyCursors()
{
var repeatedCursor = new Cursor(Guid.NewGuid().ToString(), 0xffffffffffffffff);
var manyCursors = Enumerable.Repeat(repeatedCursor, 8192).ToList();
var serialized = Cursor.MakeCursor(manyCursors);
var deserializedCursors = Cursor.GetCursors(serialized);
Assert.Equal(deserializedCursors.Length, 8192);
for (var i = 0; i < 8192; i++)
{
Assert.Equal(repeatedCursor.Id, deserializedCursors[i].Id);
Assert.Equal(repeatedCursor.Key, deserializedCursors[i].Key);
}
}