/// <summary>
/// Take responsibility for a set of new queues that were assigned to me via a new range.
/// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents.
/// ERROR HANDLING:
/// The responsibility to handle initialization and shutdown failures is inside the Agents code.
/// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded.
/// Same applies to shutdown.
/// </summary>
/// <param name="myQueues"></param>
/// <param name="failOnInit"></param>
/// <returns></returns>
private async Task AddNewQueues(IEnumerable <QueueId> myQueues, bool failOnInit)
{
// Create agents for queues in range that we don't yet have.
// First create them and store in local queuesToAgentsMap.
// Only after that Initialize them all.
var agents = new List <PersistentStreamPullingAgent>();
foreach (var queueId in myQueues)
{
if (queuesToAgentsMap.ContainsKey(queueId))
{
continue;
}
try
{
var agentIdNumber = Interlocked.Increment(ref nextAgentId);
var agentId = SystemTargetGrainId.Create(Constants.StreamPullingAgentType, this.Silo, $"{streamProviderName}_{agentIdNumber}_{queueId.ToStringWithHashCode()}");
IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(queueId);
var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, this.loggerFactory, pubSub, streamFilter, queueId, this.options, this.Silo, queueAdapter, queueAdapterCache, deliveryFailureHandler);
this.ActivationServices.GetRequiredService <Catalog>().RegisterSystemTarget(agent);
queuesToAgentsMap.Add(queueId, agent);
agents.Add(agent);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingManager_07, "Exception while creating PersistentStreamPullingAgent.", exc);
// What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down.
// If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers.
if (failOnInit)
{
throw;
}
}
}
try
{
var initTasks = new List <Task>();
foreach (var agent in agents)
{
initTasks.Add(InitAgent(agent));
}
await Task.WhenAll(initTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
if (agents.Count > 0)
{
Log(ErrorCode.PersistentStreamPullingManager_08, "Added {0} new queues: {1}. Now own total of {2} queues: {3}",
agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToString()),
NumberRunningAgents,
PrintQueues(queuesToAgentsMap.Keys));
}
}
private async Task InitAgent(PersistentStreamPullingAgent agent)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
// Need to call it as a grain reference.
var task = agentGrainRef.Initialize();
await task.LogException(logger, ErrorCode.PersistentStreamPullingManager_09, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
}
/// <summary>
/// Take responsibility for a set of new queues that were assigned to me via a new range.
/// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents.
/// ERROR HANDLING:
/// The responsibility to handle initialization and shutdown failures is inside the Agents code.
/// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded.
/// Same applies to shutdown.
/// </summary>
/// <param name="myQueues"></param>
/// <param name="failOnInit"></param>
/// <returns></returns>
private async Task AddNewQueues(IEnumerable <QueueId> myQueues, bool failOnInit)
{
// Create agents for queues in range that we don't yet have.
// First create them and store in local queuesToAgentsMap.
// Only after that Initialize them all.
var agents = new List <PersistentStreamPullingAgent>();
foreach (var queueId in myQueues)
{
if (queuesToAgentsMap.ContainsKey(queueId))
{
continue;
}
try
{
var agentId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENT_SYSTEM_TARGET_TYPE_CODE);
var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, providerRuntime, this.loggerFactory, pubSub, queueId, this.options, this.Silo);
providerRuntime.RegisterSystemTarget(agent);
queuesToAgentsMap.Add(queueId, agent);
agents.Add(agent);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingManager_07, "Exception while creating PersistentStreamPullingAgent.", exc);
// What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down.
// If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers.
if (failOnInit)
{
throw;
}
}
}
try
{
var initTasks = new List <Task>();
foreach (var agent in agents)
{
initTasks.Add(InitAgent(agent));
}
await Task.WhenAll(initTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
if (agents.Count > 0)
{
Log(ErrorCode.PersistentStreamPullingManager_08, "Added {0} new queues: {1}. Now own total of {2} queues: {3}",
agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToString()),
NumberRunningAgents,
PrintQueues(queuesToAgentsMap.Keys));
}
}
private async Task InitAgent(PersistentStreamPullingAgent agent)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null);
IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(agent.QueueId);
// Need to call it as a grain reference.
var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable, deliveryFailureHandler.AsImmutable()));
await task.LogException(logger, ErrorCode.PersistentStreamPullingManager_09, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
}
/// <summary>
/// Take responsibility for a set of new queues that were assigned to me via a new range.
/// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents.
/// ERROR HANDLING:
/// The responsibility to handle initialization and shutdown failures is inside the Agents code.
/// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded.
/// Same applies to shutdown.
/// </summary>
/// <param name="myQueues"></param>
/// <param name="failOnInit"></param>
/// <returns></returns>
private async Task AddNewQueues(IEnumerable <QueueId> myQueues, bool failOnInit)
{
// Create agents for queues in range that we don't yet have.
// First create them and store in local queuesToAgentsMap.
// Only after that Initialize them all.
var agents = new List <PersistentStreamPullingAgent>();
foreach (var queueId in myQueues.Where(queueId => !queuesToAgentsMap.ContainsKey(queueId)))
{
try
{
var agentId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENT_SYSTEM_TARGET_TYPE_CODE);
var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, providerRuntime, queueId, queueGetPeriod, initQueueTimeout);
providerRuntime.RegisterSystemTarget(agent);
queuesToAgentsMap.Add(queueId, agent);
agents.Add(agent);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingManager_07, String.Format("Exception while creating PersistentStreamPullingAgent."), exc);
// What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down.
// If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers.
if (failOnInit)
{
throw;
}
}
}
try
{
var initTasks = new List <Task>();
foreach (var agent in agents)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null);
// Need to call it as a grain reference.
var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_08, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
initTasks.Add(task);
}
await Task.WhenAll(initTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
logger.Info((int)ErrorCode.PersistentStreamPullingManager_09, "Took {0} new queues under my responsibility: {1}", agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToStringWithHashCode()));
}
/// <summary>
/// Take responsibility for a set of new queues that were assigned to me via a new range.
/// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents.
/// ERROR HANDLING:
/// The responsibility to handle initialization and shutdown failures is inside the Agents code.
/// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded.
/// Same applies to shutdown.
/// </summary>
/// <param name="myQueues"></param>
/// <param name="failOnInit"></param>
/// <returns></returns>
private async Task AddNewQueues(IEnumerable<QueueId> myQueues, bool failOnInit)
{
// Create agents for queues in range that we don't yet have.
// First create them and store in local queuesToAgentsMap.
// Only after that Initialize them all.
var agents = new List<PersistentStreamPullingAgent>();
foreach (var queueId in myQueues.Where(queueId => !queuesToAgentsMap.ContainsKey(queueId)))
{
try
{
var agentId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENT_SYSTEM_TARGET_TYPE_CODE);
var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, providerRuntime, queueId, queueGetPeriod, initQueueTimeout);
providerRuntime.RegisterSystemTarget(agent);
queuesToAgentsMap.Add(queueId, agent);
agents.Add(agent);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingManager_07, String.Format("Exception while creating PersistentStreamPullingAgent."), exc);
// What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down.
// If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers.
if (failOnInit) throw;
}
}
try
{
var initTasks = new List<Task>();
foreach (var agent in agents)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference<IPersistentStreamPullingAgent>();
var queueAdapterCacheAsImmutable = queueAdapterCache != null ? queueAdapterCache.AsImmutable() : new Immutable<IQueueAdapterCache>(null);
// Need to call it as a grain reference.
var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_08, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
initTasks.Add(task);
}
await Task.WhenAll(initTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
logger.Info((int)ErrorCode.PersistentStreamPullingManager_09, "Took {0} new queues under my responsibility: {1}", agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToStringWithHashCode()));
}
private async Task InitAgent(PersistentStreamPullingAgent agent)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference<IPersistentStreamPullingAgent>();
var queueAdapterCacheAsImmutable = queueAdapterCache != null ? queueAdapterCache.AsImmutable() : new Immutable<IQueueAdapterCache>(null);
IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(agent.QueueId);
// Need to call it as a grain reference.
var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable, deliveryFailureHandler.AsImmutable()));
await task.LogException(logger, ErrorCode.PersistentStreamPullingManager_09, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
}
/// <summary>
/// Take responsibility for a set of new queues that were assigned to me via a new range.
/// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents.
/// ERROR HANDLING:
/// The responsibility to handle initialization and shutdown failures is inside the Agents code.
/// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded.
/// Same applies to shutdown.
/// </summary>
/// <param name="myQueues"></param>
/// <param name="failOnInit"></param>
/// <returns></returns>
private async Task AddNewQueues(IEnumerable<QueueId> myQueues, bool failOnInit)
{
// Create agents for queues in range that we don't yet have.
// First create them and store in local queuesToAgentsMap.
// Only after that Initialize them all.
var agents = new List<PersistentStreamPullingAgent>();
foreach (var queueId in myQueues.Where(queueId => !queuesToAgentsMap.ContainsKey(queueId)))
{
try
{
var agentId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENT_SYSTEM_TARGET_TYPE_CODE);
var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, providerRuntime, pubSub, queueId, config);
providerRuntime.RegisterSystemTarget(agent);
queuesToAgentsMap.Add(queueId, agent);
agents.Add(agent);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingManager_07, "Exception while creating PersistentStreamPullingAgent.", exc);
// What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down.
// If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers.
if (failOnInit) throw;
}
}
try
{
var initTasks = new List<Task>();
foreach (var agent in agents)
{
initTasks.Add(InitAgent(agent));
}
await Task.WhenAll(initTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
if (agents.Count > 0)
{
Log(ErrorCode.PersistentStreamPullingManager_08, "Added {0} new queues: {1}. Now own total of {2} queues: {3}",
agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToString()),
NumberRunningAgents,
PrintQueues(queuesToAgentsMap.Keys));
}
}
