private async Task AsyncTimerCallback(object state)
{
var queueId = (QueueId)state;
try
{
Task localReceiverInitTask = receiverInitTask;
if (localReceiverInitTask != null)
{
await localReceiverInitTask;
receiverInitTask = null;
}
if (IsShutdown)
{
return; // timer was already removed, last tick
}
// loop through the queue until it is empty.
while (!IsShutdown) // timer will be set to null when we are asked to shudown.
{
int maxCacheAddCount = queueCache?.GetMaxAddCount() ?? QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG;
if (maxCacheAddCount != QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG && maxCacheAddCount <= 0)
{
return;
}
// If read succeeds and there is more data, we continue reading.
// If read succeeds and there is no more data, we break out of loop
// If read fails, we retry 6 more times, with backoff policy.
// we log each failure as warnings. After 6 times retry if still fail, we break out of loop and log an error
bool moreData = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => ReadFromQueue(queueId, receiver, maxCacheAddCount),
ReadLoopRetryMax,
ReadLoopRetryExceptionFilter,
Constants.INFINITE_TIMESPAN,
ReadLoopBackoff);
if (!moreData)
{
return;
}
}
}
catch (Exception exc)
{
receiverInitTask = null;
logger.LogError(
(int)ErrorCode.PersistentStreamPullingAgent_12,
exc,
"Giving up reading from queue {QueueId} after retry attempts {ReadLoopRetryMax}",
queueId,
ReadLoopRetryMax);
}
}
private async Task AsyncTimerCallback(object state)
{
try
{
Task localReceiverInitTask = receiverInitTask;
if (localReceiverInitTask != null)
{
await localReceiverInitTask;
receiverInitTask = null;
}
if (IsShutdown)
{
return; // timer was already removed, last tick
}
// loop through the queue until it is empty.
while (!IsShutdown) // timer will be set to null when we are asked to shudown.
{
int maxCacheAddCount = queueCache?.GetMaxAddCount() ?? QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG;
if (maxCacheAddCount != QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG && maxCacheAddCount <= 0)
{
return;
}
// If read succeeds and there is more data, we continue reading.
// If read succeeds and there is no more data, we break out of loop
// If read fails, we try again, with backoff policy.
// This prevents spamming backend queue which may be encountering transient errors.
// We retry until the operation succeeds or we are shutdown.
bool moreData = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => ReadFromQueue((QueueId)state, receiver, maxCacheAddCount),
AsyncExecutorWithRetries.INFINITE_RETRIES,
(e, i) => !IsShutdown,
Constants.INFINITE_TIMESPAN,
ReadLoopBackoff);
if (!moreData)
{
return;
}
}
}
catch (Exception exc)
{
receiverInitTask = null;
logger.Error(ErrorCode.PersistentStreamPullingAgent_12, "Exception while PersistentStreamPullingAgentGrain.AsyncTimerCallback", exc);
}
}
private async Task AsyncTimerCallback(object state)
{
try
{
var myQueueId = (QueueId)(state);
if (IsShutdown)
{
return; // timer was already removed, last tick
}
IQueueAdapterReceiver rcvr = receiver;
int maxCacheAddCount = queueCache != null?queueCache.GetMaxAddCount() : QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG;
// loop through the queue until it is empty.
while (!IsShutdown) // timer will be set to null when we are asked to shudown.
{
var now = DateTime.UtcNow;
// Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod.
if ((now - lastTimeCleanedPubSubCache) >= config.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency))
{
lastTimeCleanedPubSubCache = now;
CleanupPubSubCache(now);
}
if (queueCache != null)
{
IList <IBatchContainer> purgedItems;
if (queueCache.TryPurgeFromCache(out purgedItems))
{
try
{
await rcvr.MessagesDeliveredAsync(purgedItems);
}
catch (Exception exc)
{
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_27,
String.Format("Exception calling MessagesDeliveredAsync on queue {0}. Ignoring.", myQueueId), exc);
}
}
}
if (queueCache != null && queueCache.IsUnderPressure())
{
// Under back pressure. Exit the loop. Will attempt again in the next timer callback.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off.");
return;
}
// Retrive one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events.
IList <IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount);
if (multiBatch == null || multiBatch.Count == 0)
{
return; // queue is empty. Exit the loop. Will attempt again in the next timer callback.
}
if (queueCache != null)
{
queueCache.AddToCache(multiBatch);
}
numMessages += multiBatch.Count;
numReadMessagesCounter.IncrementBy(multiBatch.Count);
if (logger.IsVerbose2)
{
logger.Verbose2((int)ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.",
multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages);
}
foreach (var group in
multiBatch
.Where(m => m != null)
.GroupBy(container => new Tuple <Guid, string>(container.StreamGuid, container.StreamNamespace)))
{
var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2);
StreamConsumerCollection streamData;
if (pubSubCache.TryGetValue(streamId, out streamData))
{
streamData.RefreshActivity(now);
StartInactiveCursors(streamData); // if this is an existing stream, start any inactive cursors
}
else
{
RegisterStream(streamId, group.First().SequenceToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system
}
}
}
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_12, "Exception while PersistentStreamPullingAgentGrain.AsyncTimerCallback", exc);
}
}