public Task <int> ConsumerCount(Guid guidId, string streamProvider, string streamNamespace) { StreamId streamId = StreamId.GetStreamId(guidId, streamProvider, streamNamespace); var streamRendezvous = GetRendezvousGrain(streamId); return(streamRendezvous.ConsumerCount(streamId)); }
private async Task AsyncTimerCallback(object state) { try { var myQueueId = (QueueId)(state); if (timer == null) { return; // timer was already removed, last tick } IQueueAdapterReceiver rcvr = receiver; // loop through the queue until it is empty. while (true) { // Retrive one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events. IEnumerable <IBatchContainer> msgsEnumerable = await rcvr.GetQueueMessagesAsync(); List <IBatchContainer> multiBatch = null; if (msgsEnumerable != null) { multiBatch = msgsEnumerable.ToList(); } if (multiBatch == null || multiBatch.Count == 0) { return; // queue is empty. Exit the loop. Will attempt again in the next timer callback. } rcvr.AddToCache(multiBatch); numMessages += multiBatch.Count; numReadMessagesCounter.IncrementBy(multiBatch.Count); if (logger.IsVerbose2) { logger.Verbose2((int)ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.", multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages); } foreach (var group in multiBatch.Where(m => m != null) .GroupBy(container => new Tuple <Guid, string>(container.StreamGuid, container.StreamNamespace))) { var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2); StreamConsumerCollection streamData; if (pubSubCache.TryGetValue(streamId, out streamData)) { StartInactiveCursors(streamId, streamData); // if this is an existing stream, start any inactive cursors } else { RegisterStream(streamId, group.First().SequenceToken); // if this is a new stream register as producer of stream in pub sub system } } } } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_12, String.Format("Exception while PersistentStreamPullingAgentGrain.AsyncTimerCallback"), exc); } }
/// <summary> /// Read from queue. /// Returns true, if data was read, false if it was not /// </summary> /// <param name="myQueueId"></param> /// <param name="rcvr"></param> /// <param name="maxCacheAddCount"></param> /// <returns></returns> private async Task <bool> ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver rcvr, int maxCacheAddCount) { if (rcvr == null) { return(false); } var now = DateTime.UtcNow; // Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod. if ((now - lastTimeCleanedPubSubCache) >= config.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency)) { lastTimeCleanedPubSubCache = now; CleanupPubSubCache(now); } if (queueCache != null) { IList <IBatchContainer> purgedItems; if (queueCache.TryPurgeFromCache(out purgedItems)) { try { await rcvr.MessagesDeliveredAsync(purgedItems); } catch (Exception exc) { logger.Warn(ErrorCode.PersistentStreamPullingAgent_27, $"Exception calling MessagesDeliveredAsync on queue {myQueueId}. Ignoring.", exc); } } } if (queueCache != null && queueCache.IsUnderPressure()) { // Under back pressure. Exit the loop. Will attempt again in the next timer callback. logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off."); return(false); } // Retrieve one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events. IList <IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount); if (multiBatch == null || multiBatch.Count == 0) { return(false); // queue is empty. Exit the loop. Will attempt again in the next timer callback. } queueCache?.AddToCache(multiBatch); numMessages += multiBatch.Count; numReadMessagesCounter.IncrementBy(multiBatch.Count); if (logger.IsVerbose2) { logger.Verbose2(ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.", multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages); } foreach (var group in multiBatch .Where(m => m != null) .GroupBy(container => new Tuple <Guid, string>(container.StreamGuid, container.StreamNamespace))) { var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2); StreamSequenceToken startToken = group.First().SequenceToken; StreamConsumerCollection streamData; if (pubSubCache.TryGetValue(streamId, out streamData)) { streamData.RefreshActivity(now); StartInactiveCursors(streamData, startToken); // if this is an existing stream, start any inactive cursors } else { RegisterStream(streamId, startToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system } } return(true); }
private async Task AsyncTimerCallback(object state) { try { var myQueueId = (QueueId)(state); if (IsShutdown) return; // timer was already removed, last tick IQueueAdapterReceiver rcvr = receiver; int maxCacheAddCount = queueCache != null ? queueCache.MaxAddCount : QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG; // loop through the queue until it is empty. while (!IsShutdown) // timer will be set to null when we are asked to shudown. { var now = DateTime.UtcNow; // Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod. if ((now - lastTimeCleanedPubSubCache) >= config.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency)) { lastTimeCleanedPubSubCache = now; CleanupPubSubCache(now); } if (queueCache != null) { IList<IBatchContainer> purgedItems; if (queueCache.TryPurgeFromCache(out purgedItems)) { try { await rcvr.MessagesDeliveredAsync(purgedItems); } catch (Exception exc) { logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_27, String.Format("Exception calling MessagesDeliveredAsync on queue {0}. Ignoring.", myQueueId), exc); } } } if (queueCache != null && queueCache.IsUnderPressure()) { // Under back pressure. Exit the loop. Will attempt again in the next timer callback. logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off."); return; } // Retrive one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events. IList<IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount); if (multiBatch == null || multiBatch.Count == 0) return; // queue is empty. Exit the loop. Will attempt again in the next timer callback. if (queueCache != null) { queueCache.AddToCache(multiBatch); } numMessages += multiBatch.Count; numReadMessagesCounter.IncrementBy(multiBatch.Count); if (logger.IsVerbose2) logger.Verbose2((int)ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.", multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages); foreach (var group in multiBatch .Where(m => m != null) .GroupBy(container => new Tuple<Guid, string>(container.StreamGuid, container.StreamNamespace))) { var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2); StreamConsumerCollection streamData; if (pubSubCache.TryGetValue(streamId, out streamData)) { streamData.RefreshActivity(now); StartInactiveCursors(streamData); // if this is an existing stream, start any inactive cursors } else { RegisterStream(streamId, group.First().SequenceToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system } } } } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_12, "Exception while PersistentStreamPullingAgentGrain.AsyncTimerCallback", exc); } }