internal PersistentStreamPullingManager( SystemTargetGrainId managerId, string strProviderName, IStreamProviderRuntime runtime, IStreamPubSub streamPubSub, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, IStreamFilter streamFilter, StreamPullingAgentOptions options, ILoggerFactory loggerFactory, SiloAddress siloAddress) : base(managerId, siloAddress, lowPriority: false, loggerFactory) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (runtime == null) { throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null"); } if (streamPubSub == null) { throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; providerRuntime = runtime; pubSub = streamPubSub; this.options = options; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; latestCommandNumber = 0; queueBalancer = streamQueueBalancer; this.streamFilter = streamFilter; this.adapterFactory = adapterFactory; queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = loggerFactory.CreateLogger($"{GetType().FullName}.{streamProviderName}"); Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); this.loggerFactory = loggerFactory; IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); }
internal PersistentStreamPullingManager( GrainId id, string strProviderName, IStreamProviderRuntime runtime, IStreamPubSub streamPubSub, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, TimeSpan queueGetPeriod, TimeSpan initQueueTimeout, TimeSpan maxEvenDeliveryTime) : base(id, runtime.ExecutingSiloAddress) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (runtime == null) { throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null"); } if (streamPubSub == null) { throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; providerRuntime = runtime; pubSub = streamPubSub; this.queueGetPeriod = queueGetPeriod; this.initQueueTimeout = initQueueTimeout; this.maxEvenDeliveryTime = maxEvenDeliveryTime; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; latestCommandNumber = 0; queueBalancer = streamQueueBalancer; this.adapterFactory = adapterFactory; queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = providerRuntime.GetLogger(GetType().Name + "-" + streamProviderName); logger.Info((int)ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); }
internal PersistentStreamPullingManager( GrainId id, string strProviderName, IStreamProviderRuntime runtime, IStreamPubSub streamPubSub, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, PersistentStreamProviderConfig config) : base(id, runtime.ExecutingSiloAddress) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (runtime == null) { throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null"); } if (streamPubSub == null) { throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; providerRuntime = runtime; pubSub = streamPubSub; this.config = config; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; latestCommandNumber = 0; queueBalancer = streamQueueBalancer; this.adapterFactory = adapterFactory; queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = TraceLogger.GetLogger(GetType().Name + "-" + streamProviderName, TraceLogger.LoggerType.Provider); Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD); }
internal PersistentStreamPullingManager( GrainId id, string strProviderName, IStreamProviderRuntime runtime, IStreamPubSub streamPubSub, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, PersistentStreamProviderConfig config) : base(id, runtime.ExecutingSiloAddress) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (runtime == null) { throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null"); } if (streamPubSub == null) { throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; providerRuntime = runtime; pubSub = streamPubSub; this.config = config; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; latestCommandNumber = 0; queueBalancer = streamQueueBalancer; this.adapterFactory = adapterFactory; queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = LogManager.GetLogger(GetType().Name + "-" + streamProviderName, LoggerType.Provider); Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD); }
internal PersistentStreamPullingManager( GrainId id, string strProviderName, IStreamProviderRuntime runtime, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, TimeSpan queueGetPeriod, TimeSpan initQueueTimeout) : base(id, runtime.ExecutingSiloAddress) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (runtime == null) { throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; providerRuntime = runtime; this.queueGetPeriod = queueGetPeriod; this.initQueueTimeout = initQueueTimeout; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; queueBalancer = streamQueueBalancer; queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = providerRuntime.GetLogger(GetType().Name + "-" + streamProviderName); logger.Info((int)ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); }
private async Task SendAndReceiveFromQueueAdapter(IQueueAdapterFactory adapterFactory, IProviderConfiguration config) { IQueueAdapter adapter = await adapterFactory.CreateAdapter(); IQueueAdapterCache cache = adapterFactory.GetQueueAdapterCache(); // Create receiver per queue IStreamQueueMapper mapper = adapterFactory.GetStreamQueueMapper(); Dictionary <QueueId, IQueueAdapterReceiver> receivers = mapper.GetAllQueues().ToDictionary(queueId => queueId, adapter.CreateReceiver); Dictionary <QueueId, IQueueCache> caches = mapper.GetAllQueues().ToDictionary(queueId => queueId, cache.CreateQueueCache); await Task.WhenAll(receivers.Values.Select(receiver => receiver.Initialize(TimeSpan.FromSeconds(5)))); // test using 2 streams Guid streamId1 = Guid.NewGuid(); Guid streamId2 = Guid.NewGuid(); int receivedBatches = 0; var streamsPerQueue = new ConcurrentDictionary <QueueId, HashSet <IStreamIdentity> >(); // reader threads (at most 2 active queues because only two streams) var work = new List <Task>(); foreach (KeyValuePair <QueueId, IQueueAdapterReceiver> receiverKvp in receivers) { QueueId queueId = receiverKvp.Key; var receiver = receiverKvp.Value; var qCache = caches[queueId]; Task task = Task.Factory.StartNew(() => { while (receivedBatches < NumBatches) { var messages = receiver.GetQueueMessagesAsync(CloudQueueMessage.MaxNumberOfMessagesToPeek).Result.ToArray(); if (!messages.Any()) { continue; } foreach (AzureQueueBatchContainer message in messages.Cast <AzureQueueBatchContainer>()) { streamsPerQueue.AddOrUpdate(queueId, id => new HashSet <IStreamIdentity> { new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString()) }, (id, set) => { set.Add(new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString())); return(set); }); output.WriteLine("Queue {0} received message on stream {1}", queueId, message.StreamGuid); Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <int>().Count()); // "Half the events were ints" Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <string>().Count()); // "Half the events were strings" } Interlocked.Add(ref receivedBatches, messages.Length); qCache.AddToCache(messages); } }); work.Add(task); } // send events List <object> events = CreateEvents(NumMessagesPerBatch); work.Add(Task.Factory.StartNew(() => Enumerable.Range(0, NumBatches) .Select(i => i % 2 == 0 ? streamId1 : streamId2) .ToList() .ForEach(streamId => adapter.QueueMessageBatchAsync(streamId, streamId.ToString(), events.Take(NumMessagesPerBatch).ToArray(), null, RequestContext.Export()).Wait()))); await Task.WhenAll(work); // Make sure we got back everything we sent Assert.Equal(NumBatches, receivedBatches); // check to see if all the events are in the cache and we can enumerate through them StreamSequenceToken firstInCache = new EventSequenceToken(0); foreach (KeyValuePair <QueueId, HashSet <IStreamIdentity> > kvp in streamsPerQueue) { var receiver = receivers[kvp.Key]; var qCache = caches[kvp.Key]; foreach (IStreamIdentity streamGuid in kvp.Value) { // read all messages in cache for stream IQueueCacheCursor cursor = qCache.GetCacheCursor(streamGuid, firstInCache); int messageCount = 0; StreamSequenceToken tenthInCache = null; StreamSequenceToken lastToken = firstInCache; while (cursor.MoveNext()) { Exception ex; messageCount++; IBatchContainer batch = cursor.GetCurrent(out ex); output.WriteLine("Token: {0}", batch.SequenceToken); Assert.True(batch.SequenceToken.CompareTo(lastToken) >= 0, $"order check for event {messageCount}"); lastToken = batch.SequenceToken; if (messageCount == 10) { tenthInCache = batch.SequenceToken; } } output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid); Assert.Equal(NumBatches / 2, messageCount); Assert.NotNull(tenthInCache); // read all messages from the 10th cursor = qCache.GetCacheCursor(streamGuid, tenthInCache); messageCount = 0; while (cursor.MoveNext()) { messageCount++; } output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid); const int expected = NumBatches / 2 - 10 + 1; // all except the first 10, including the 10th (10 + 1) Assert.Equal(expected, messageCount); } } }
private async Task SendAndReceiveFromQueueAdapter(IQueueAdapterFactory adapterFactory, IProviderConfiguration config) { IQueueAdapter adapter = await adapterFactory.CreateAdapter(); IQueueAdapterCache cache = adapterFactory.GetQueueAdapterCache(); // Create receiver per queue IStreamQueueMapper mapper = adapterFactory.GetStreamQueueMapper(); Dictionary<QueueId, IQueueAdapterReceiver> receivers = mapper.GetAllQueues().ToDictionary(queueId => queueId, adapter.CreateReceiver); Dictionary<QueueId, IQueueCache> caches = mapper.GetAllQueues().ToDictionary(queueId => queueId, cache.CreateQueueCache); await Task.WhenAll(receivers.Values.Select(receiver => receiver.Initialize(TimeSpan.FromSeconds(5)))); // test using 2 streams Guid streamId1 = Guid.NewGuid(); Guid streamId2 = Guid.NewGuid(); int receivedBatches = 0; var streamsPerQueue = new ConcurrentDictionary<QueueId, HashSet<IStreamIdentity>>(); // reader threads (at most 2 active queues because only two streams) var work = new List<Task>(); foreach( KeyValuePair<QueueId, IQueueAdapterReceiver> receiverKvp in receivers) { QueueId queueId = receiverKvp.Key; var receiver = receiverKvp.Value; var qCache = caches[queueId]; Task task = Task.Factory.StartNew(() => { while (receivedBatches < NumBatches) { var messages = receiver.GetQueueMessagesAsync(CloudQueueMessage.MaxNumberOfMessagesToPeek).Result.ToArray(); if (!messages.Any()) { continue; } foreach (AzureQueueBatchContainer message in messages.Cast<AzureQueueBatchContainer>()) { streamsPerQueue.AddOrUpdate(queueId, id => new HashSet<IStreamIdentity> { new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString()) }, (id, set) => { set.Add(new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString())); return set; }); output.WriteLine("Queue {0} received message on stream {1}", queueId, message.StreamGuid); Assert.AreEqual(NumMessagesPerBatch / 2, message.GetEvents<int>().Count(), "Half the events were ints"); Assert.AreEqual(NumMessagesPerBatch / 2, message.GetEvents<string>().Count(), "Half the events were strings"); } Interlocked.Add(ref receivedBatches, messages.Length); qCache.AddToCache(messages); } }); work.Add(task); } // send events List<object> events = CreateEvents(NumMessagesPerBatch); work.Add(Task.Factory.StartNew(() => Enumerable.Range(0, NumBatches) .Select(i => i % 2 == 0 ? streamId1 : streamId2) .ToList() .ForEach(streamId => adapter.QueueMessageBatchAsync(streamId, streamId.ToString(), events.Take(NumMessagesPerBatch).ToArray(), null, RequestContext.Export()).Wait()))); await Task.WhenAll(work); // Make sure we got back everything we sent Assert.AreEqual(NumBatches, receivedBatches); // check to see if all the events are in the cache and we can enumerate through them StreamSequenceToken firstInCache = new EventSequenceToken(0); foreach (KeyValuePair<QueueId, HashSet<IStreamIdentity>> kvp in streamsPerQueue) { var receiver = receivers[kvp.Key]; var qCache = caches[kvp.Key]; foreach (IStreamIdentity streamGuid in kvp.Value) { // read all messages in cache for stream IQueueCacheCursor cursor = qCache.GetCacheCursor(streamGuid, firstInCache); int messageCount = 0; StreamSequenceToken tenthInCache = null; StreamSequenceToken lastToken = firstInCache; while (cursor.MoveNext()) { Exception ex; messageCount++; IBatchContainer batch = cursor.GetCurrent(out ex); output.WriteLine("Token: {0}", batch.SequenceToken); Assert.IsTrue(batch.SequenceToken.CompareTo(lastToken) >= 0, "order check for event {0}", messageCount); lastToken = batch.SequenceToken; if (messageCount == 10) { tenthInCache = batch.SequenceToken; } } output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid); Assert.AreEqual(NumBatches / 2, messageCount); Assert.IsNotNull(tenthInCache); // read all messages from the 10th cursor = qCache.GetCacheCursor(streamGuid, tenthInCache); messageCount = 0; while (cursor.MoveNext()) { messageCount++; } output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid); const int expected = NumBatches / 2 - 10 + 1; // all except the first 10, including the 10th (10 + 1) Assert.AreEqual(expected, messageCount); } } }