/// <summary>
/// Take responsibility for a new queues that was assigned to me via a new range.
/// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer.
/// ERROR HANDLING:
/// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code.
/// The agent will call Initialize once and log an error. It will not call initiliaze again.
/// The receiver itself may attempt later to recover from this error and do initialization again.
/// The agent will assume initialization has succeeded and will subsequently start calling pumping receive.
/// Same applies to shutdown.
/// </summary>
/// <param name="qAdapter"></param>
/// <param name="queueAdapterCache"></param>
/// <param name="failureHandler"></param>
/// <returns></returns>
public async Task Initialize(Immutable <IQueueAdapter> qAdapter, Immutable <IQueueAdapterCache> queueAdapterCache, Immutable <IStreamFailureHandler> failureHandler)
{
if (qAdapter.Value == null)
{
throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
}
if (failureHandler.Value == null)
{
throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null");
}
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
streamFailureHandler = failureHandler.Value;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, String.Format("Exception while calling IQueueAdapter.CreateNewReceiver."), exc);
return;
}
try
{
if (queueAdapterCache.Value != null)
{
queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, String.Format("Exception while calling IQueueAdapterCache.CreateQueueCache."), exc);
return;
}
try
{
var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(initQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode()));
await task;
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(queueGetPeriod);
timer = providerRuntime.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, queueGetPeriod);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
private async Task AsyncTimerCallback(object state)
{
try
{
var myQueueId = (QueueId)(state);
if (timer == null)
{
return; // timer was already removed, last tick
}
IQueueAdapterReceiver rcvr = receiver;
// loop through the queue until it is empty.
while (true)
{
// Retrive one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events.
IEnumerable <IBatchContainer> msgsEnumerable = await rcvr.GetQueueMessagesAsync();
List <IBatchContainer> multiBatch = null;
if (msgsEnumerable != null)
{
multiBatch = msgsEnumerable.ToList();
}
if (multiBatch == null || multiBatch.Count == 0)
{
return; // queue is empty. Exit the loop. Will attempt again in the next timer callback.
}
rcvr.AddToCache(multiBatch);
numMessages += multiBatch.Count;
numReadMessagesCounter.IncrementBy(multiBatch.Count);
if (logger.IsVerbose2)
{
logger.Verbose2((int)ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.",
multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages);
}
foreach (var group in multiBatch.Where(m => m != null)
.GroupBy(container => new Tuple <Guid, string>(container.StreamGuid, container.StreamNamespace)))
{
var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2);
StreamConsumerCollection streamData;
if (pubSubCache.TryGetValue(streamId, out streamData))
{
StartInactiveCursors(streamId, streamData); // if this is an existing stream, start any inactive cursors
}
else
{
RegisterStream(streamId, group.First().SequenceToken); // if this is a new stream register as producer of stream in pub sub system
}
}
}
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_12,
String.Format("Exception while PersistentStreamPullingAgentGrain.AsyncTimerCallback"), exc);
}
}
private void InitializeInternal()
{
logger.LogInformation(
(int)ErrorCode.PersistentStreamPullingAgent_02,
"Init of {Name} {Id} on silo {Silo} for queue {Queue}.",
GetType().Name,
((ISystemTargetBase)this).GrainId.ToString(),
Silo,
QueueId.ToStringWithHashCode());
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.LogError((int)ErrorCode.PersistentStreamPullingAgent_23, exc, "Exception while calling IQueueAdapterCache.CreateQueueCache.");
throw;
}
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.LogError((int)ErrorCode.PersistentStreamPullingAgent_02, exc, "Exception while calling IQueueAdapter.CreateNewReceiver.");
throw;
}
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(this.options.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = ThreadSafeRandom.NextTimeSpan(this.options.GetQueueMsgsTimerPeriod);
timer = RegisterGrainTimer(AsyncTimerCallback, QueueId, randomTimerOffset, this.options.GetQueueMsgsTimerPeriod);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix), () => pubSubCache.Count);
logger.LogInformation((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {Queue} under my responsibility.", QueueId.ToStringWithHashCode());
}
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler)
{
logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter;
streamFailureHandler = failureHandler;
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
throw;
}
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
throw;
}
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod);
timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
/// <summary>
/// Read from queue.
/// Returns true, if data was read, false if it was not
/// </summary>
/// <param name="myQueueId"></param>
/// <param name="rcvr"></param>
/// <param name="maxCacheAddCount"></param>
/// <returns></returns>
private async Task <bool> ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver rcvr, int maxCacheAddCount)
{
if (rcvr == null)
{
return(false);
}
var now = DateTime.UtcNow;
// Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod.
if ((now - lastTimeCleanedPubSubCache) >= config.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency))
{
lastTimeCleanedPubSubCache = now;
CleanupPubSubCache(now);
}
if (queueCache != null)
{
IList <IBatchContainer> purgedItems;
if (queueCache.TryPurgeFromCache(out purgedItems))
{
try
{
await rcvr.MessagesDeliveredAsync(purgedItems);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.PersistentStreamPullingAgent_27,
$"Exception calling MessagesDeliveredAsync on queue {myQueueId}. Ignoring.", exc);
}
}
}
if (queueCache != null && queueCache.IsUnderPressure())
{
// Under back pressure. Exit the loop. Will attempt again in the next timer callback.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off.");
return(false);
}
// Retrieve one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events.
IList <IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount);
if (multiBatch == null || multiBatch.Count == 0)
{
return(false); // queue is empty. Exit the loop. Will attempt again in the next timer callback.
}
queueCache?.AddToCache(multiBatch);
numMessages += multiBatch.Count;
numReadMessagesCounter.IncrementBy(multiBatch.Count);
if (logger.IsVerbose2)
{
logger.Verbose2(ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.",
multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages);
}
foreach (var group in
multiBatch
.Where(m => m != null)
.GroupBy(container => new Tuple <Guid, string>(container.StreamGuid, container.StreamNamespace)))
{
var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2);
StreamSequenceToken startToken = group.First().SequenceToken;
StreamConsumerCollection streamData;
if (pubSubCache.TryGetValue(streamId, out streamData))
{
streamData.RefreshActivity(now);
StartInactiveCursors(streamData, startToken); // if this is an existing stream, start any inactive cursors
}
else
{
RegisterStream(streamId, startToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system
}
}
return(true);
}
public async Task Shutdown()
{
// Stop pulling from queues that are not in my range anymore.
logger.Info(ErrorCode.PersistentStreamPullingAgent_05, "Shutdown of {0} responsible for queue: {1}", GetType().Name, QueueId.ToStringWithHashCode());
if (timer != null)
{
IDisposable tmp = timer;
timer = null;
Utils.SafeExecute(tmp.Dispose);
}
this.queueCache = null;
Task localReceiverInitTask = receiverInitTask;
if (localReceiverInitTask != null)
{
try
{
await localReceiverInitTask;
receiverInitTask = null;
}
catch (Exception)
{
receiverInitTask = null;
// squelch
}
}
try
{
IQueueAdapterReceiver localReceiver = this.receiver;
this.receiver = null;
if (localReceiver != null)
{
var task = OrleansTaskExtentions.SafeExecute(() => localReceiver.Shutdown(config.InitQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07,
$"QueueAdapterReceiver {QueueId} failed to Shutdown.");
await task;
}
}
catch
{
// Just ignore this exception and proceed as if Shutdown has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
var unregisterTasks = new List <Task>();
var meAsStreamProducer = this.AsReference <IStreamProducerExtension>();
foreach (var tuple in pubSubCache)
{
tuple.Value.DisposeAll(logger);
var streamId = tuple.Key;
logger.Info(ErrorCode.PersistentStreamPullingAgent_06, "Unregister PersistentStreamPullingAgent Producer for stream {0}.", streamId);
unregisterTasks.Add(pubSub.UnregisterProducer(streamId, streamProviderName, meAsStreamProducer));
}
try
{
await Task.WhenAll(unregisterTasks);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.PersistentStreamPullingAgent_08,
"Failed to unregister myself as stream producer to some streams that used to be in my responsibility.", exc);
}
pubSubCache.Clear();
IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix));
//IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, StatisticUniquePostfix));
}
private async Task AsyncTimerCallback(object state)
{
try
{
var myQueueId = (QueueId)(state);
if (IsShutdown) return; // timer was already removed, last tick
IQueueAdapterReceiver rcvr = receiver;
int maxCacheAddCount = queueCache != null ? queueCache.MaxAddCount : QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG;
// loop through the queue until it is empty.
while (!IsShutdown) // timer will be set to null when we are asked to shudown.
{
var now = DateTime.UtcNow;
// Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod.
if ((now - lastTimeCleanedPubSubCache) >= config.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency))
{
lastTimeCleanedPubSubCache = now;
CleanupPubSubCache(now);
}
if (queueCache != null)
{
IList<IBatchContainer> purgedItems;
if (queueCache.TryPurgeFromCache(out purgedItems))
{
try
{
await rcvr.MessagesDeliveredAsync(purgedItems);
}
catch (Exception exc)
{
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_27,
String.Format("Exception calling MessagesDeliveredAsync on queue {0}. Ignoring.", myQueueId), exc);
}
}
}
if (queueCache != null && queueCache.IsUnderPressure())
{
// Under back pressure. Exit the loop. Will attempt again in the next timer callback.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off.");
return;
}
// Retrive one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events.
IList<IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount);
if (multiBatch == null || multiBatch.Count == 0) return; // queue is empty. Exit the loop. Will attempt again in the next timer callback.
if (queueCache != null)
{
queueCache.AddToCache(multiBatch);
}
numMessages += multiBatch.Count;
numReadMessagesCounter.IncrementBy(multiBatch.Count);
if (logger.IsVerbose2) logger.Verbose2((int)ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.",
multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages);
foreach (var group in
multiBatch
.Where(m => m != null)
.GroupBy(container => new Tuple<Guid, string>(container.StreamGuid, container.StreamNamespace)))
{
var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2);
StreamConsumerCollection streamData;
if (pubSubCache.TryGetValue(streamId, out streamData))
{
streamData.RefreshActivity(now);
StartInactiveCursors(streamData); // if this is an existing stream, start any inactive cursors
}
else
{
RegisterStream(streamId, group.First().SequenceToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system
}
}
}
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_12, "Exception while PersistentStreamPullingAgentGrain.AsyncTimerCallback", exc);
}
}
/// <summary>
/// Take responsibility for a new queues that was assigned to me via a new range.
/// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer.
/// ERROR HANDLING:
/// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code.
/// The agent will call Initialize once and log an error. It will not call initiliaze again.
/// The receiver itself may attempt later to recover from this error and do initialization again.
/// The agent will assume initialization has succeeded and will subsequently start calling pumping receive.
/// Same applies to shutdown.
/// </summary>
/// <param name="qAdapter"></param>
/// <param name="queueAdapterCache"></param>
/// <param name="failureHandler"></param>
/// <returns></returns>
public async Task Initialize(Immutable<IQueueAdapter> qAdapter, Immutable<IQueueAdapterCache> queueAdapterCache, Immutable<IStreamFailureHandler> failureHandler)
{
if (qAdapter.Value == null) throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
if (failureHandler.Value == null) throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null");
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
streamFailureHandler = failureHandler.Value;
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
return;
}
try
{
if (queueAdapterCache.Value != null)
{
queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
return;
}
try
{
var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode()));
await task;
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod);
timer = base.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod);
logger.Info((int) ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
internal PersistentStreamPullingAgent(
SystemTargetGrainId id,
string strProviderName,
ILoggerFactory loggerFactory,
IStreamPubSub streamPubSub,
IStreamFilter streamFilter,
QueueId queueId,
StreamPullingAgentOptions options,
SiloAddress siloAddress,
IQueueAdapter queueAdapter,
IQueueAdapterCache queueAdapterCache,
IStreamFailureHandler streamFailureHandler)
: base(id, siloAddress, true, loggerFactory)
{
if (strProviderName == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
}
QueueId = queueId;
streamProviderName = strProviderName;
pubSub = streamPubSub;
this.streamFilter = streamFilter;
pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>();
this.options = options;
this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter));
this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler));;
numMessages = 0;
logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}");
logger.Info(ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix));
// TODO: move queue cache size statistics tracking into queue cache implementation once Telemetry APIs and LogStatistics have been reconciled.
//IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache != null ? queueCache.Size : 0);
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
throw;
}
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
throw;
}
}
public IQueueAdapterReceiver CreateReceiver(QueueId queueId)
{
IQueueAdapterReceiver receiver = (IQueueAdapterReceiver) new MemoryAdapterReceiver(GetQueueGrain(queueId), logger);
return(receiver);
}
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler)
{
logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter;
streamFailureHandler = failureHandler;
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
throw;
}
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
throw;
}
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod);
timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
/// <summary>
/// Read from queue.
/// Returns true, if data was read, false if it was not
/// </summary>
/// <param name="myQueueId"></param>
/// <param name="rcvr"></param>
/// <param name="maxCacheAddCount"></param>
/// <returns></returns>
private async Task<bool> ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver rcvr, int maxCacheAddCount)
{
try
{
var now = DateTime.UtcNow;
// Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod.
if ((now - lastTimeCleanedPubSubCache) >= config.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency))
{
lastTimeCleanedPubSubCache = now;
CleanupPubSubCache(now);
}
if (queueCache != null)
{
IList<IBatchContainer> purgedItems;
if (queueCache.TryPurgeFromCache(out purgedItems))
{
try
{
await rcvr.MessagesDeliveredAsync(purgedItems);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.PersistentStreamPullingAgent_27,
$"Exception calling MessagesDeliveredAsync on queue {myQueueId}. Ignoring.", exc);
}
}
}
if (queueCache != null && queueCache.IsUnderPressure())
{
// Under back pressure. Exit the loop. Will attempt again in the next timer callback.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off.");
return false;
}
// Retrive one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events.
IList<IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount);
if (multiBatch == null || multiBatch.Count == 0) return false; // queue is empty. Exit the loop. Will attempt again in the next timer callback.
queueCache?.AddToCache(multiBatch);
numMessages += multiBatch.Count;
numReadMessagesCounter.IncrementBy(multiBatch.Count);
if (logger.IsVerbose2) logger.Verbose2(ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.",
multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages);
foreach (var group in
multiBatch
.Where(m => m != null)
.GroupBy(container => new Tuple<Guid, string>(container.StreamGuid, container.StreamNamespace)))
{
var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2);
StreamConsumerCollection streamData;
if (pubSubCache.TryGetValue(streamId, out streamData))
{
streamData.RefreshActivity(now);
StartInactiveCursors(streamData); // if this is an existing stream, start any inactive cursors
}
else
{
RegisterStream(streamId, group.First().SequenceToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system
}
}
return true;
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_28, "Exception while reading from queue.", exc);
throw;
}
}
/// <summary>
/// Read from queue.
/// Returns true, if data was read, false if it was not
/// </summary>
/// <param name="myQueueId"></param>
/// <param name="rcvr"></param>
/// <param name="maxCacheAddCount"></param>
/// <returns></returns>
private async Task <bool> ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver rcvr, int maxCacheAddCount)
{
if (rcvr == null)
{
return(false);
}
var now = DateTime.UtcNow;
// Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod.
if ((now - lastTimeCleanedPubSubCache) >= this.options.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency))
{
lastTimeCleanedPubSubCache = now;
CleanupPubSubCache(now);
}
if (queueCache != null)
{
IList <IBatchContainer> purgedItems;
if (queueCache.TryPurgeFromCache(out purgedItems))
{
try
{
await rcvr.MessagesDeliveredAsync(purgedItems);
}
catch (Exception exc)
{
logger.LogWarning(
(int)ErrorCode.PersistentStreamPullingAgent_27,
exc,
"Exception calling MessagesDeliveredAsync on queue {MyQueueId}. Ignoring.",
myQueueId);
}
}
}
if (queueCache != null && queueCache.IsUnderPressure())
{
// Under back pressure. Exit the loop. Will attempt again in the next timer callback.
logger.LogInformation((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off.");
return(false);
}
// Retrieve one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events.
IList <IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount);
if (multiBatch == null || multiBatch.Count == 0)
{
return(false); // queue is empty. Exit the loop. Will attempt again in the next timer callback.
}
queueCache?.AddToCache(multiBatch);
numMessages += multiBatch.Count;
numReadMessagesCounter.IncrementBy(multiBatch.Count);
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace(
(int)ErrorCode.PersistentStreamPullingAgent_11,
"Got {ReceivedCount} messages from queue {Queue}. So far {MessageCount} messages from this queue.",
multiBatch.Count,
myQueueId.ToStringWithHashCode(),
numMessages);
}
foreach (var group in
multiBatch
.Where(m => m != null)
.GroupBy(container => container.StreamId))
{
var streamId = new InternalStreamId(queueAdapter.Name, group.Key);
StreamSequenceToken startToken = group.First().SequenceToken;
StreamConsumerCollection streamData;
if (pubSubCache.TryGetValue(streamId, out streamData))
{
streamData.RefreshActivity(now);
if (streamData.StreamRegistered)
{
StartInactiveCursors(streamData,
startToken); // if this is an existing stream, start any inactive cursors
}
else
{
if (this.logger.IsEnabled(LogLevel.Debug))
{
this.logger.LogDebug(
$"Pulled new messages in stream {streamId} from the queue, but pulling agent haven't succeeded in" +
$"RegisterStream yet, will start deliver on this stream after RegisterStream succeeded");
}
}
}
else
{
RegisterStream(streamId, startToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system
}
}
return(true);
}