/// <summary> /// Take responsibility for a new queues that was assigned to me via a new range. /// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer. /// ERROR HANDLING: /// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code. /// The agent will call Initialize once and log an error. It will not call initiliaze again. /// The receiver itself may attempt later to recover from this error and do initialization again. /// The agent will assume initialization has succeeded and will subsequently start calling pumping receive. /// Same applies to shutdown. /// </summary> /// <param name="qAdapter"></param> /// <param name="queueAdapterCache"></param> /// <param name="failureHandler"></param> /// <returns></returns> public async Task Initialize(Immutable <IQueueAdapter> qAdapter, Immutable <IQueueAdapterCache> queueAdapterCache, Immutable <IStreamFailureHandler> failureHandler) { if (qAdapter.Value == null) { throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); } if (failureHandler.Value == null) { throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null"); } logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter.Value; streamFailureHandler = failureHandler.Value; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, String.Format("Exception while calling IQueueAdapter.CreateNewReceiver."), exc); return; } try { if (queueAdapterCache.Value != null) { queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, String.Format("Exception while calling IQueueAdapterCache.CreateQueueCache."), exc); return; } try { var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(initQueueTimeout)); task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode())); await task; } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(queueGetPeriod); timer = providerRuntime.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, queueGetPeriod); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
/// <summary> /// Take responsibility for a set of new queues that were assigned to me via a new range. /// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents. /// ERROR HANDLING: /// The responsibility to handle initialization and shutdown failures is inside the Agents code. /// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded. /// Same applies to shutdown. /// </summary> /// <param name="myQueues"></param> /// <param name="failOnInit"></param> /// <returns></returns> private async Task AddNewQueues(IEnumerable <QueueId> myQueues, bool failOnInit) { // Create agents for queues in range that we don't yet have. // First create them and store in local queuesToAgentsMap. // Only after that Initialize them all. var agents = new List <PersistentStreamPullingAgent>(); foreach (var queueId in myQueues) { if (queuesToAgentsMap.ContainsKey(queueId)) { continue; } try { var agentIdNumber = Interlocked.Increment(ref nextAgentId); var agentId = SystemTargetGrainId.Create(Constants.StreamPullingAgentType, this.Silo, $"{streamProviderName}_{agentIdNumber}_{queueId.ToStringWithHashCode()}"); IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(queueId); var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, this.loggerFactory, pubSub, streamFilter, queueId, this.options, this.Silo, queueAdapter, queueAdapterCache, deliveryFailureHandler); this.ActivationServices.GetRequiredService <Catalog>().RegisterSystemTarget(agent); queuesToAgentsMap.Add(queueId, agent); agents.Add(agent); } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingManager_07, "Exception while creating PersistentStreamPullingAgent.", exc); // What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down. // If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers. if (failOnInit) { throw; } } } try { var initTasks = new List <Task>(); foreach (var agent in agents) { initTasks.Add(InitAgent(agent)); } await Task.WhenAll(initTasks); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } if (agents.Count > 0) { Log(ErrorCode.PersistentStreamPullingManager_08, "Added {0} new queues: {1}. Now own total of {2} queues: {3}", agents.Count, Utils.EnumerableToString(agents, agent => agent.QueueId.ToString()), NumberRunningAgents, PrintQueues(queuesToAgentsMap.Keys)); } }
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler) { logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter; streamFailureHandler = failureHandler; lastTimeCleanedPubSubCache = DateTime.UtcNow; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc); throw; } try { if (queueAdapterCache != null) { queueCache = queueAdapterCache.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc); throw; } try { receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout)) .LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize."); receiverInitTask.Ignore(); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod); timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix), () => pubSubCache.Count); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
private async Task InitAgent(PersistentStreamPullingAgent agent) { // Init the agent only after it was registered locally. var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>(); var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null); IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(agent.QueueId); // Need to call it as a grain reference. var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable, deliveryFailureHandler.AsImmutable())); await task.LogException(logger, ErrorCode.PersistentStreamPullingManager_09, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId)); }
internal PersistentStreamPullingAgent( SystemTargetGrainId id, string strProviderName, ILoggerFactory loggerFactory, IStreamPubSub streamPubSub, IStreamFilter streamFilter, QueueId queueId, StreamPullingAgentOptions options, SiloAddress siloAddress, IQueueAdapter queueAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler streamFailureHandler) : base(id, siloAddress, true, loggerFactory) { if (strProviderName == null) { throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null"); } QueueId = queueId; streamProviderName = strProviderName; pubSub = streamPubSub; this.streamFilter = streamFilter; pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>(); this.options = options; this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter)); this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler));; this.queueAdapterCache = queueAdapterCache; numMessages = 0; logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}"); logger.LogInformation( (int)ErrorCode.PersistentStreamPullingAgent_01, "Created {Name} {Id} for Stream Provider {StreamProvider} on silo {Silo} for Queue {Queue}.", GetType().Name, ((ISystemTargetBase)this).GrainId.ToString(), streamProviderName, Silo, QueueId.ToStringWithHashCode()); numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix)); numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix)); }
/// <summary> /// Acquire delivery failure handler for a queue /// </summary> /// <param name="queueId"></param> /// <returns></returns> public Task <IStreamFailureHandler> GetDeliveryFailureHandler(QueueId queueId) { return(Task.FromResult(streamFailureHandler ?? (streamFailureHandler = new NoOpStreamDeliveryFailureHandler()))); }
/// <summary> /// Take responsibility for a new queues that was assigned to me via a new range. /// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer. /// ERROR HANDLING: /// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code. /// The agent will call Initialize once and log an error. It will not call initiliaze again. /// The receiver itself may attempt later to recover from this error and do initialization again. /// The agent will assume initialization has succeeded and will subsequently start calling pumping receive. /// Same applies to shutdown. /// </summary> /// <param name="qAdapter"></param> /// <param name="queueAdapterCache"></param> /// <param name="failureHandler"></param> /// <returns></returns> public async Task Initialize(Immutable<IQueueAdapter> qAdapter, Immutable<IQueueAdapterCache> queueAdapterCache, Immutable<IStreamFailureHandler> failureHandler) { if (qAdapter.Value == null) throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); if (failureHandler.Value == null) throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null"); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter.Value; streamFailureHandler = failureHandler.Value; lastTimeCleanedPubSubCache = DateTime.UtcNow; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc); return; } try { if (queueAdapterCache.Value != null) { queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc); return; } try { var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout)); task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode())); await task; } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod); timer = base.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod); logger.Info((int) ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
internal PersistentStreamPullingAgent( SystemTargetGrainId id, string strProviderName, ILoggerFactory loggerFactory, IStreamPubSub streamPubSub, IStreamFilter streamFilter, QueueId queueId, StreamPullingAgentOptions options, SiloAddress siloAddress, IQueueAdapter queueAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler streamFailureHandler) : base(id, siloAddress, true, loggerFactory) { if (strProviderName == null) { throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null"); } QueueId = queueId; streamProviderName = strProviderName; pubSub = streamPubSub; this.streamFilter = streamFilter; pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>(); this.options = options; this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter)); this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler));; numMessages = 0; logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}"); logger.Info(ErrorCode.PersistentStreamPullingAgent_01, "Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.", GetType().Name, ((ISystemTargetBase)this).GrainId.ToString(), streamProviderName, Silo, QueueId.ToStringWithHashCode()); numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix)); numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix)); // TODO: move queue cache size statistics tracking into queue cache implementation once Telemetry APIs and LogStatistics have been reconciled. //IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache != null ? queueCache.Size : 0); try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc); throw; } try { if (queueAdapterCache != null) { queueCache = queueAdapterCache.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc); throw; } }
public Task <IStreamFailureHandler> GetDeliveryFailureHandler(QueueId queueId) { //TODO: Add a queue specific default failure handler with reasonable error reporting. //TODO: Try to get failure handler from service provider so users can inject their own. return(Task.FromResult(streamFailureHandler ?? (streamFailureHandler = new NoOpStreamDeliveryFailureHandler()))); }
public Task <IStreamFailureHandler> GetDeliveryFailureHandler(QueueId queueId) { return(Task.FromResult <IStreamFailureHandler>( _failureHandler ?? (_failureHandler = new LoggerStreamFailureHandler(_logger)) )); }
public Task<IStreamFailureHandler> GetDeliveryFailureHandler(QueueId queueId) { return Task.FromResult<IStreamFailureHandler>( _failureHandler ?? (_failureHandler = new LoggerStreamFailureHandler(_logger)) ); }
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler) { logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter; streamFailureHandler = failureHandler; lastTimeCleanedPubSubCache = DateTime.UtcNow; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc); throw; } try { if (queueAdapterCache != null) { queueCache = queueAdapterCache.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc); throw; } try { receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout)) .LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize."); receiverInitTask.Ignore(); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod); timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
public Task <IStreamFailureHandler> GetDeliveryFailureHandler(QueueId queueId) { IStreamFailureHandler streamFailureHandler = this; return(Task.FromResult(streamFailureHandler)); }