internal PersistentStreamPullingAgent(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
QueueId queueId,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress, true)
{
if (runtime == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null");
if (strProviderName == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
QueueId = queueId;
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
pubSubCache = new Dictionary<StreamId, StreamConsumerCollection>();
safeRandom = new SafeRandom();
this.config = config;
numMessages = 0;
logger = providerRuntime.GetLogger(GrainId + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
string statUniquePostfix = strProviderName + "." + QueueId;
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, statUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, statUniquePostfix));
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, statUniquePostfix), () => pubSubCache.Count);
// TODO: move queue cache size statistics tracking into queue cache implementation once Telemetry APIs and LogStatistics have been reconciled.
//IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache != null ? queueCache.Size : 0);
}
public string QueueToPartition(QueueId queue)
{
if (queue == null)
{
throw new ArgumentNullException("queue");
}
string partitionId;
if (!partitionDictionary.TryGetValue(queue, out partitionId))
{
throw new ArgumentOutOfRangeException(string.Format(CultureInfo.InvariantCulture, "queue {0}", queue.ToStringWithHashCode()));
}
return partitionId;
}
public string QueueToPartition(QueueId queue)
{
if (queue == null)
{
throw new ArgumentNullException("queue");
}
string partitionId;
if (!partitionDictionary.TryGetValue(queue, out partitionId))
{
throw new ArgumentOutOfRangeException(string.Format(CultureInfo.InvariantCulture, "queue {0}", queue.ToStringWithHashCode()));
}
return(partitionId);
}
internal PersistentStreamPullingAgent(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
QueueId queueId,
TimeSpan queueGetPeriod,
TimeSpan initQueueTimeout,
TimeSpan maxDeliveryTime)
: base(id, runtime.ExecutingSiloAddress, true)
{
if (runtime == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null");
if (strProviderName == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
QueueId = queueId;
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = runtime.PubSub(StreamPubSubType.GrainBased);
pubSubCache = new Dictionary<StreamId, StreamConsumerCollection>();
safeRandom = new SafeRandom();
this.queueGetPeriod = queueGetPeriod;
this.initQueueTimeout = initQueueTimeout;
this.maxDeliveryTime = maxDeliveryTime;
numMessages = 0;
logger = providerRuntime.GetLogger(GrainId + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, strProviderName));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, strProviderName));
}
/// <summary>
/// Read from queue.
/// Returns true, if data was read, false if it was not
/// </summary>
/// <param name="myQueueId"></param>
/// <param name="rcvr"></param>
/// <param name="maxCacheAddCount"></param>
/// <returns></returns>
private async Task<bool> ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver rcvr, int maxCacheAddCount)
{
try
{
var now = DateTime.UtcNow;
// Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod.
if ((now - lastTimeCleanedPubSubCache) >= config.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency))
{
lastTimeCleanedPubSubCache = now;
CleanupPubSubCache(now);
}
if (queueCache != null)
{
IList<IBatchContainer> purgedItems;
if (queueCache.TryPurgeFromCache(out purgedItems))
{
try
{
await rcvr.MessagesDeliveredAsync(purgedItems);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.PersistentStreamPullingAgent_27,
$"Exception calling MessagesDeliveredAsync on queue {myQueueId}. Ignoring.", exc);
}
}
}
if (queueCache != null && queueCache.IsUnderPressure())
{
// Under back pressure. Exit the loop. Will attempt again in the next timer callback.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off.");
return false;
}
// Retrive one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events.
IList<IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount);
if (multiBatch == null || multiBatch.Count == 0) return false; // queue is empty. Exit the loop. Will attempt again in the next timer callback.
queueCache?.AddToCache(multiBatch);
numMessages += multiBatch.Count;
numReadMessagesCounter.IncrementBy(multiBatch.Count);
if (logger.IsVerbose2) logger.Verbose2(ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.",
multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages);
foreach (var group in
multiBatch
.Where(m => m != null)
.GroupBy(container => new Tuple<Guid, string>(container.StreamGuid, container.StreamNamespace)))
{
var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2);
StreamConsumerCollection streamData;
if (pubSubCache.TryGetValue(streamId, out streamData))
{
streamData.RefreshActivity(now);
StartInactiveCursors(streamData); // if this is an existing stream, start any inactive cursors
}
else
{
RegisterStream(streamId, group.First().SequenceToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system
}
}
return true;
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_28, "Exception while reading from queue.", exc);
throw;
}
}
