internal PersistentStreamPullingAgent( GrainId id, string strProviderName, IStreamProviderRuntime runtime, IStreamPubSub streamPubSub, QueueId queueId, PersistentStreamProviderConfig config) : base(id, runtime.ExecutingSiloAddress, true) { if (runtime == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null"); if (strProviderName == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null"); QueueId = queueId; streamProviderName = strProviderName; providerRuntime = runtime; pubSub = streamPubSub; pubSubCache = new Dictionary<StreamId, StreamConsumerCollection>(); safeRandom = new SafeRandom(); this.config = config; numMessages = 0; logger = providerRuntime.GetLogger(GrainId + "-" + streamProviderName); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_01, "Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.", GetType().Name, GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode()); string statUniquePostfix = strProviderName + "." + QueueId; numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, statUniquePostfix)); numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, statUniquePostfix)); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, statUniquePostfix), () => pubSubCache.Count); // TODO: move queue cache size statistics tracking into queue cache implementation once Telemetry APIs and LogStatistics have been reconciled. //IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache != null ? queueCache.Size : 0); }
public string QueueToPartition(QueueId queue) { if (queue == null) { throw new ArgumentNullException("queue"); } string partitionId; if (!partitionDictionary.TryGetValue(queue, out partitionId)) { throw new ArgumentOutOfRangeException(string.Format(CultureInfo.InvariantCulture, "queue {0}", queue.ToStringWithHashCode())); } return partitionId; }
public string QueueToPartition(QueueId queue) { if (queue == null) { throw new ArgumentNullException("queue"); } string partitionId; if (!partitionDictionary.TryGetValue(queue, out partitionId)) { throw new ArgumentOutOfRangeException(string.Format(CultureInfo.InvariantCulture, "queue {0}", queue.ToStringWithHashCode())); } return(partitionId); }
internal PersistentStreamPullingAgent( GrainId id, string strProviderName, IStreamProviderRuntime runtime, QueueId queueId, TimeSpan queueGetPeriod, TimeSpan initQueueTimeout, TimeSpan maxDeliveryTime) : base(id, runtime.ExecutingSiloAddress, true) { if (runtime == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null"); if (strProviderName == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null"); QueueId = queueId; streamProviderName = strProviderName; providerRuntime = runtime; pubSub = runtime.PubSub(StreamPubSubType.GrainBased); pubSubCache = new Dictionary<StreamId, StreamConsumerCollection>(); safeRandom = new SafeRandom(); this.queueGetPeriod = queueGetPeriod; this.initQueueTimeout = initQueueTimeout; this.maxDeliveryTime = maxDeliveryTime; numMessages = 0; logger = providerRuntime.GetLogger(GrainId + "-" + streamProviderName); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_01, "Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.", GetType().Name, GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode()); numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, strProviderName)); numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, strProviderName)); }
/// <summary> /// Read from queue. /// Returns true, if data was read, false if it was not /// </summary> /// <param name="myQueueId"></param> /// <param name="rcvr"></param> /// <param name="maxCacheAddCount"></param> /// <returns></returns> private async Task<bool> ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver rcvr, int maxCacheAddCount) { try { var now = DateTime.UtcNow; // Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod. if ((now - lastTimeCleanedPubSubCache) >= config.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency)) { lastTimeCleanedPubSubCache = now; CleanupPubSubCache(now); } if (queueCache != null) { IList<IBatchContainer> purgedItems; if (queueCache.TryPurgeFromCache(out purgedItems)) { try { await rcvr.MessagesDeliveredAsync(purgedItems); } catch (Exception exc) { logger.Warn(ErrorCode.PersistentStreamPullingAgent_27, $"Exception calling MessagesDeliveredAsync on queue {myQueueId}. Ignoring.", exc); } } } if (queueCache != null && queueCache.IsUnderPressure()) { // Under back pressure. Exit the loop. Will attempt again in the next timer callback. logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off."); return false; } // Retrive one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events. IList<IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount); if (multiBatch == null || multiBatch.Count == 0) return false; // queue is empty. Exit the loop. Will attempt again in the next timer callback. queueCache?.AddToCache(multiBatch); numMessages += multiBatch.Count; numReadMessagesCounter.IncrementBy(multiBatch.Count); if (logger.IsVerbose2) logger.Verbose2(ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.", multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages); foreach (var group in multiBatch .Where(m => m != null) .GroupBy(container => new Tuple<Guid, string>(container.StreamGuid, container.StreamNamespace))) { var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2); StreamConsumerCollection streamData; if (pubSubCache.TryGetValue(streamId, out streamData)) { streamData.RefreshActivity(now); StartInactiveCursors(streamData); // if this is an existing stream, start any inactive cursors } else { RegisterStream(streamId, group.First().SequenceToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system } } return true; } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_28, "Exception while reading from queue.", exc); throw; } }