public ScaleoutSubscription(string identity, IList<string> eventKeys, string cursor, IList<ScaleoutMappingStore> streams, Func<MessageResult, object, Task<bool>> callback, int maxMessages, IPerformanceCounterManager counters, object state) : base(identity, eventKeys, callback, maxMessages, counters, state) { if (streams == null) { throw new ArgumentNullException("streams"); } _streams = streams; List<Cursor> cursors = null; if (String.IsNullOrEmpty(cursor)) { cursors = new List<Cursor>(streams.Count); for (int i = 0; i < streams.Count; i++) { ScaleoutMapping maxMapping = streams[i].MaxMapping; ulong id = UInt64.MaxValue; string key = i.ToString(CultureInfo.InvariantCulture); if (maxMapping != null) { id = maxMapping.Id; } var newCursor = new Cursor(key, id); cursors.Add(newCursor); } } else { cursors = Cursor.GetCursors(cursor); } _cursors = cursors; }
private List<Cursor> GetCursorsFromEventKeys(IList<string> eventKeys, TopicLookup topics) { var list = new List<Cursor>(eventKeys.Count); foreach (var eventKey in eventKeys) { var cursor = new Cursor(eventKey, GetMessageId(topics, eventKey), _stringMinifier.Minify(eventKey)); list.Add(cursor); } return list; }
private void AddCursorForStream(int streamIndex, List<Cursor> cursors) { ScaleoutMapping maxMapping = _streams[streamIndex].MaxMapping; ulong id = UInt64.MaxValue; string key = streamIndex.ToString(CultureInfo.InvariantCulture); if (maxMapping != null) { id = maxMapping.Id; } var newCursor = new Cursor(key, id); cursors.Add(newCursor); }
public static List<Cursor> GetCursors(string cursor, Func<string, object, string> keyMaximizer, object state) { // Technically GetCursors should never be called with a null value, so this is extra cautious if (String.IsNullOrEmpty(cursor)) { throw new FormatException(Resources.Error_InvalidCursorFormat); } var signals = new HashSet<string>(); var cursors = new List<Cursor>(); string currentKey = null; string currentEscapedKey = null; ulong currentId; bool escape = false; bool consumingKey = true; var sb = new StringBuilder(); var sbEscaped = new StringBuilder(); Cursor parsedCursor; foreach (var ch in cursor) { // escape can only be true if we are consuming the key if (escape) { if (ch != '\\' && ch != ',' && ch != '|') { throw new FormatException(Resources.Error_InvalidCursorFormat); } sb.Append(ch); sbEscaped.Append(ch); escape = false; } else { if (ch == '\\') { if (!consumingKey) { throw new FormatException(Resources.Error_InvalidCursorFormat); } sbEscaped.Append('\\'); escape = true; } else if (ch == ',') { if (!consumingKey) { throw new FormatException(Resources.Error_InvalidCursorFormat); } // For now String.Empty is an acceptable key, but this should change once we verify // that empty keys cannot be created legitimately. currentKey = keyMaximizer(sb.ToString(), state); // If the keyMap cannot find a key, we cannot create an array of cursors. // This most likely means there was an AppDomain restart or a misbehaving client. if (currentKey == null) { return null; } // Don't allow duplicate keys if (!signals.Add(currentKey)) { throw new FormatException(Resources.Error_InvalidCursorFormat); } currentEscapedKey = sbEscaped.ToString(); sb.Clear(); sbEscaped.Clear(); consumingKey = false; } else if (ch == '|') { if (consumingKey) { throw new FormatException(Resources.Error_InvalidCursorFormat); } ParseCursorId(sb, out currentId); parsedCursor = new Cursor(currentKey, currentId, currentEscapedKey); cursors.Add(parsedCursor); sb.Clear(); consumingKey = true; } else { sb.Append(ch); if (consumingKey) { sbEscaped.Append(ch); } } } } if (consumingKey) { throw new FormatException(Resources.Error_InvalidCursorFormat); } ParseCursorId(sb, out currentId); parsedCursor = new Cursor(currentKey, currentId, currentEscapedKey); cursors.Add(parsedCursor); return cursors; }
public static Cursor Clone(Cursor cursor) { return new Cursor(cursor.Key, cursor.Id, cursor._escapedKey); }
protected override void PerformWork(IList<ArraySegment<Message>> items, out int totalCount, out object state) { // The list of cursors represent (streamid, payloadid) var cursors = new List<Cursor>(); totalCount = 0; foreach (var stream in _streams) { // Get the mapping for this stream IndexedDictionary<ulong, ScaleoutMapping> mapping = stream.Value; // See if we have a cursor for this key Cursor cursor = null; // REVIEW: We should optimize this int index = _cursors.FindIndex(c => c.Key == stream.Key); bool consumed = true; if (index != -1) { cursor = Cursor.Clone(_cursors[index]); // If there's no node for this cursor id it's likely because we've // had an app domain restart and the cursor position is now invalid. if (mapping[cursor.Id] == null) { // Set it to the first id in this mapping cursor.Id = stream.Value.MinKey; // Mark this cursor as unconsumed consumed = false; } } else { // Create a cursor and add it to the list. // Point the Id to the first value cursor = new Cursor(key: stream.Key, id: stream.Value.MinKey); consumed = false; } cursors.Add(cursor); // Try to find a local mapping for this payload LinkedListNode<KeyValuePair<ulong, ScaleoutMapping>> node = mapping[cursor.Id]; // Skip this node only if this isn't a new cursor if (node != null && consumed) { // Skip this node since we've already consumed it node = node.Next; } while (node != null) { KeyValuePair<ulong, ScaleoutMapping> pair = node.Value; // Stop if we got more than max messages if (totalCount >= MaxMessages) { break; } // It should be ok to lock here since groups aren't modified that often lock (EventKeys) { // For each of the event keys we care about, extract all of the messages // from the payload foreach (var eventKey in EventKeys) { LocalEventKeyInfo info; if (pair.Value.EventKeyMappings.TryGetValue(eventKey, out info)) { int maxMessages = Math.Min(info.Count, MaxMessages); MessageStoreResult<Message> storeResult = info.Store.GetMessages(info.MinLocal, maxMessages); if (storeResult.Messages.Count > 0) { items.Add(storeResult.Messages); totalCount += storeResult.Messages.Count; } } } } // Update the cursor id cursor.Id = pair.Key; node = node.Next; } } state = cursors; }
public static List<Cursor> GetCursors(string cursor, string prefix, Func<string, object, string> keyMaximizer, object state) { // Technically GetCursors should never be called with a null value, so this is extra cautious if (String.IsNullOrEmpty(cursor)) { throw new FormatException(Resources.Error_InvalidCursorFormat); } // If the cursor does not begin with the prefix stream, it isn't necessarily a formatting problem. // The cursor with a different prefix might have had different, but also valid, formatting. // Null should be returned so new cursors will be generated if (!cursor.StartsWith(prefix, StringComparison.Ordinal)) { return null; } var signals = new HashSet<string>(); var cursors = new List<Cursor>(); string currentKey = null; string currentEscapedKey = null; ulong currentId; bool escape = false; bool consumingKey = true; var sb = new StringBuilder(); var sbEscaped = new StringBuilder(); Cursor parsedCursor; for (int i = prefix.Length; i < cursor.Length; i++) { var ch = cursor[i]; // escape can only be true if we are consuming the key if (escape) { if (ch != '\\' && ch != ',' && ch != '|') { throw new FormatException(Resources.Error_InvalidCursorFormat); } sb.Append(ch); sbEscaped.Append(ch); escape = false; } else { if (ch == '\\') { if (!consumingKey) { throw new FormatException(Resources.Error_InvalidCursorFormat); } sbEscaped.Append('\\'); escape = true; } else if (ch == ',') { if (!consumingKey) { throw new FormatException(Resources.Error_InvalidCursorFormat); } // For now String.Empty is an acceptable key, but this should change once we verify // that empty keys cannot be created legitimately. currentKey = keyMaximizer(sb.ToString(), state); // If the keyMap cannot find a key, we cannot create an array of cursors. // This most likely means there was an AppDomain restart or a misbehaving client. if (currentKey == null) { return null; } // Don't allow duplicate keys if (!signals.Add(currentKey)) { throw new FormatException(Resources.Error_InvalidCursorFormat); } currentEscapedKey = sbEscaped.ToString(); sb.Clear(); sbEscaped.Clear(); consumingKey = false; } else if (ch == '|') { if (consumingKey) { throw new FormatException(Resources.Error_InvalidCursorFormat); } ParseCursorId(sb, out currentId); parsedCursor = new Cursor(currentKey, currentId, currentEscapedKey); cursors.Add(parsedCursor); sb.Clear(); consumingKey = true; } else { sb.Append(ch); if (consumingKey) { sbEscaped.Append(ch); } } } } if (consumingKey) { throw new FormatException(Resources.Error_InvalidCursorFormat); } ParseCursorId(sb, out currentId); parsedCursor = new Cursor(currentKey, currentId, currentEscapedKey); cursors.Add(parsedCursor); return cursors; }
public void SymmetricWithManyCursors() { var repeatedCursor = new Cursor(Guid.NewGuid().ToString(), 0xffffffffffffffff); var manyCursors = Enumerable.Repeat(repeatedCursor, 8192).ToList(); var serialized = Cursor.MakeCursor(manyCursors); var deserializedCursors = Cursor.GetCursors(serialized); Assert.Equal(deserializedCursors.Length, 8192); for (var i = 0; i < 8192; i++) { Assert.Equal(repeatedCursor.Id, deserializedCursors[i].Id); Assert.Equal(repeatedCursor.Key, deserializedCursors[i].Key); } }