public ScaleoutStreamManager(Func <int, IList <Message>, Task> send,
Action <int, ulong, ScaleoutMessage> receive,
int streamCount,
TraceSource trace,
IPerformanceCounterManager performanceCounters,
ScaleoutConfiguration configuration)
{
if (configuration.QueueBehavior != QueuingBehavior.Disabled && configuration.MaxQueueLength == 0)
{
throw new InvalidOperationException(Resources.Error_ScaleoutQueuingConfig);
}
_streams = new ScaleoutStream[streamCount];
_send = send;
_receive = receive;
var receiveMapping = new ScaleoutMappingStore[streamCount];
performanceCounters.ScaleoutStreamCountTotal.RawValue = streamCount;
performanceCounters.ScaleoutStreamCountBuffering.RawValue = streamCount;
performanceCounters.ScaleoutStreamCountOpen.RawValue = 0;
for (int i = 0; i < streamCount; i++)
{
_streams[i] = new ScaleoutStream(trace, "Stream(" + i + ")", configuration.QueueBehavior, configuration.MaxQueueLength, performanceCounters);
receiveMapping[i] = new ScaleoutMappingStore();
}
Streams = new ReadOnlyCollection <ScaleoutMappingStore>(receiveMapping);
}
private void OnReceivedCore(int streamIndex, ulong id, IList <Message> messages)
{
Counters.ScaleoutMessageBusMessagesReceivedPerSec.IncrementBy(messages.Count);
_trace.TraceInformation("OnReceived({0}, {1}, {2})", streamIndex, id, messages.Count);
var localMapping = new LocalEventKeyInfo[messages.Count];
var keys = new HashSet <string>();
for (var i = 0; i < messages.Count; ++i)
{
Message message = messages[i];
keys.Add(message.Key);
ulong localId = Save(message);
MessageStore <Message> messageStore = Topics[message.Key].Store;
localMapping[i] = new LocalEventKeyInfo(message.Key, localId, messageStore);
}
// Get the stream for this payload
ScaleoutMappingStore store = StreamManager.Streams[streamIndex];
// Publish only after we've setup the mapping fully
store.Add(id, localMapping);
// Schedule after we're done
foreach (var eventKey in keys)
{
ScheduleEvent(eventKey);
}
}
public ScaleoutStreamManager(Func <int, IList <Message>, Task> send,
Action <int, ulong, ScaleoutMessage> receive,
int streamCount,
TraceSource trace,
IPerformanceCounterManager performanceCounters,
ScaleoutConfiguration configuration)
{
_streams = new ScaleoutStream[streamCount];
_send = send;
_receive = receive;
var receiveMapping = new ScaleoutMappingStore[streamCount];
performanceCounters.ScaleoutStreamCountTotal.RawValue = streamCount;
performanceCounters.ScaleoutStreamCountBuffering.RawValue = streamCount;
performanceCounters.ScaleoutStreamCountOpen.RawValue = 0;
for (int i = 0; i < streamCount; i++)
{
_streams[i] = new ScaleoutStream(trace, "Stream(" + i + ")", configuration.MaxQueueLength, performanceCounters);
receiveMapping[i] = new ScaleoutMappingStore();
}
Streams = new ReadOnlyCollection <ScaleoutMappingStore>(receiveMapping);
}
protected override void PerformWork(IList <ArraySegment <Message> > items, out int totalCount, out object state)
{
// The list of cursors represent (streamid, payloadid)
var nextCursors = new ulong[_stores.Count];
totalCount = 0;
for (var i = 0; i < _stores.Count; ++i)
{
// Get the mapping for this stream
ScaleoutMappingStore store = _stores[i];
// See if we have a cursor for this key
Cursor cursor = _cursors[i];
nextCursors[i] = cursor.Id;
// Try to find a local mapping for this payload
IEnumerator <ScaleoutMapping> enumerator = store.GetEnumerator(cursor.Id);
// Stop if we got more than max messages
while (totalCount < MaxMessages && enumerator.MoveNext())
{
ScaleoutMapping mapping = enumerator.Current;
// For each of the event keys we care about, extract all of the messages
// from the payload
lock (EventKeys)
{
for (var j = 0; j < EventKeys.Count; ++j)
{
string eventKey = EventKeys[j];
for (int k = 0; k < mapping.LocalKeyInfo.Count; k++)
{
LocalEventKeyInfo info = mapping.LocalKeyInfo[k];
if (info.Key.Equals(eventKey, StringComparison.OrdinalIgnoreCase))
{
MessageStoreResult <Message> storeResult = info.MessageStore.GetMessages(info.Id, 1);
if (storeResult.Messages.Count > 0)
{
items.Add(storeResult.Messages);
totalCount += storeResult.Messages.Count;
}
}
}
}
}
// Update the cursor id
nextCursors[i] = mapping.Id;
}
}
state = nextCursors;
}
private IEnumerable <Tuple <ScaleoutMapping, int> > GetMappings()
{
var enumerators = new List <CachedStreamEnumerator>();
var singleStream = _streams.Count == 1;
for (var streamIndex = 0; streamIndex < _streams.Count; ++streamIndex)
{
// Get the mapping for this stream
ScaleoutMappingStore store = _streams[streamIndex];
Cursor cursor = _cursors[streamIndex];
// Try to find a local mapping for this payload
var enumerator = new CachedStreamEnumerator(store.GetEnumerator(cursor.Id),
streamIndex);
enumerators.Add(enumerator);
}
var counter = 0;
while (enumerators.Count > 0)
{
ScaleoutMapping minMapping = null;
CachedStreamEnumerator minEnumerator = null;
for (int i = enumerators.Count - 1; i >= 0; i--)
{
counter += 1;
CachedStreamEnumerator enumerator = enumerators[i];
ScaleoutMapping mapping;
if (enumerator.TryMoveNext(out mapping))
{
if (minMapping == null || mapping.ServerCreationTime < minMapping.ServerCreationTime)
{
minMapping = mapping;
minEnumerator = enumerator;
}
}
else
{
enumerators.RemoveAt(i);
}
}
if (minMapping != null)
{
minEnumerator.ClearCachedValue();
yield return(Tuple.Create(minMapping, minEnumerator.StreamIndex));
}
}
_trace.TraceEvent(TraceEventType.Verbose, 0, $"End of mappings (connection ID: {Identity}). Total mappings processed: {counter}");
}
private IEnumerable <Tuple <ScaleoutMapping, int> > GetMappings()
{
var enumerators = new List <CachedStreamEnumerator>();
for (var streamIndex = 0; streamIndex < _streams.Count; ++streamIndex)
{
// Get the mapping for this stream
ScaleoutMappingStore store = _streams[streamIndex];
Cursor cursor = _cursors[streamIndex];
// Try to find a local mapping for this payload
var enumerator = new CachedStreamEnumerator(store.GetEnumerator(cursor.Id),
streamIndex);
enumerators.Add(enumerator);
}
while (enumerators.Count > 0)
{
ScaleoutMapping minMapping = null;
CachedStreamEnumerator minEnumerator = null;
for (int i = enumerators.Count - 1; i >= 0; i--)
{
CachedStreamEnumerator enumerator = enumerators[i];
ScaleoutMapping mapping;
if (enumerator.TryMoveNext(out mapping))
{
if (minMapping == null || mapping.ServerCreationTime < minMapping.ServerCreationTime)
{
minMapping = mapping;
minEnumerator = enumerator;
}
}
else
{
enumerators.RemoveAt(i);
}
}
if (minMapping != null)
{
minEnumerator.ClearCachedValue();
yield return(Tuple.Create(minMapping, minEnumerator.StreamIndex));
}
}
}
private void OnReceivedCore(int streamIndex, ulong id, ScaleoutMessage scaleoutMessage)
{
Counters.ScaleoutMessageBusMessagesReceivedPerSec.IncrementBy(scaleoutMessage.Messages.Count);
_trace.TraceInformation("OnReceived({0}, {1}, {2})", streamIndex, id, scaleoutMessage.Messages.Count);
TraceScaleoutMessages(id, scaleoutMessage);
var localMapping = new LocalEventKeyInfo[scaleoutMessage.Messages.Count];
var keys = new HashSet <string>();
for (var i = 0; i < scaleoutMessage.Messages.Count; ++i)
{
Message message = scaleoutMessage.Messages[i];
// Remember where this message came from
message.MappingId = id;
message.StreamIndex = streamIndex;
keys.Add(message.Key);
ulong localId = Save(message);
_trace.TraceVerbose("Message id: {0}, stream : {1}, eventKey: '{2}' saved with local id: {3}",
id, streamIndex, message.Key, localId);
MessageStore <Message> messageStore = Topics[message.Key].Store;
localMapping[i] = new LocalEventKeyInfo(message.Key, localId, messageStore);
}
// Get the stream for this payload
ScaleoutMappingStore store = StreamManager.Streams[streamIndex];
// Publish only after we've setup the mapping fully
store.Add(id, scaleoutMessage, localMapping);
if (_trace.Switch.ShouldTrace(TraceEventType.Verbose))
{
_trace.TraceVerbose("Scheduling eventkeys: {0}", string.Join(",", keys));
}
// Schedule after we're done
foreach (var eventKey in keys)
{
ScheduleEvent(eventKey);
}
}
public ScaleoutStreamManager(Func <int, IList <Message>, Task> send,
Action <int, ulong, IList <Message> > receive,
int streamCount,
ScaleoutConfiguration configuration,
TraceSource trace)
{
_sendQueues = new ScaleoutTaskQueue[streamCount];
_send = send;
_receive = receive;
var receiveMapping = new ScaleoutMappingStore[streamCount];
for (int i = 0; i < streamCount; i++)
{
_sendQueues[i] = new ScaleoutTaskQueue(trace, "Stream(" + i + ")", configuration);
receiveMapping[i] = new ScaleoutMappingStore();
}
Streams = new ReadOnlyCollection <ScaleoutMappingStore>(receiveMapping);
}
private void OnReceivedCore(int streamIndex, ulong id, ScaleoutMessage scaleoutMessage)
{
Counters.ScaleoutMessageBusMessagesReceivedPerSec.IncrementBy(scaleoutMessage.Messages.Count);
_trace.TraceInformation("OnReceived({0}, {1}, {2})", streamIndex, id, scaleoutMessage.Messages.Count);
var localMapping = new Dictionary <string, IList <LocalEventKeyInfo> >(StringComparer.OrdinalIgnoreCase);
for (var i = 0; i < scaleoutMessage.Messages.Count; ++i)
{
Message message = scaleoutMessage.Messages[i];
// Remember where this message came from
message.MappingId = id;
message.StreamIndex = streamIndex;
IList <LocalEventKeyInfo> keyInfo;
if (!localMapping.TryGetValue(message.Key, out keyInfo))
{
keyInfo = new List <LocalEventKeyInfo>();
localMapping.Add(message.Key, keyInfo);
}
ulong localId = Save(message);
MessageStore <Message> messageStore = Topics[message.Key].Store;
keyInfo.Add(new LocalEventKeyInfo(localId, messageStore));
}
// Get the stream for this payload
ScaleoutMappingStore store = StreamManager.Streams[streamIndex];
// Publish only after we've setup the mapping fully
store.Add(id, scaleoutMessage, localMapping);
// Schedule after we're done
foreach (var eventKey in localMapping.Keys)
{
ScheduleEvent(eventKey);
}
}
private void OnReceivedCore(int streamIndex, ulong id, ScaleoutMessage scaleoutMessage)
{
Counters.ScaleoutMessageBusMessagesReceivedPerSec.IncrementBy(scaleoutMessage.Messages.Count);
_logger.WriteInformation(String.Format("OnReceived({0}, {1}, {2})", streamIndex, id, scaleoutMessage.Messages.Count));
var localMapping = new LocalEventKeyInfo[scaleoutMessage.Messages.Count];
var keys = new HashSet <string>();
for (var i = 0; i < scaleoutMessage.Messages.Count; ++i)
{
Message message = scaleoutMessage.Messages[i];
// Remember where this message came from
message.MappingId = id;
message.StreamIndex = streamIndex;
keys.Add(message.Key);
ulong localId = Save(message);
MessageStore <Message> messageStore = Topics[message.Key].Store;
localMapping[i] = new LocalEventKeyInfo(message.Key, localId, messageStore);
}
// Get the stream for this payload
ScaleoutMappingStore store = StreamManager.Streams[streamIndex];
// Publish only after we've setup the mapping fully
store.Add(id, scaleoutMessage, localMapping);
// Schedule after we're done
foreach (var eventKey in keys)
{
ScheduleEvent(eventKey);
}
}
