public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter)
{
IStreamQueueBalancer queueBalancer = CreateQueueBalancer(streamProviderName);
var managerId = SystemTargetGrainId.Create(Constants.StreamPullingAgentManagerType, this.siloDetails.SiloAddress, streamProviderName);
var pubsubOptions = this.ServiceProvider.GetOptionsByName <StreamPubSubOptions>(streamProviderName);
var pullingAgentOptions = this.ServiceProvider.GetOptionsByName <StreamPullingAgentOptions>(streamProviderName);
var manager = new PersistentStreamPullingManager(
managerId,
streamProviderName,
this,
this.PubSub(pubsubOptions.PubSubType),
adapterFactory,
queueBalancer,
pullingAgentOptions,
this.loggerFactory,
this.siloDetails.SiloAddress);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize(queueAdapter.AsImmutable());
return(pullingAgentManager);
}
public async Task Start()
{
if (providerRuntime.InSilo)
{
IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(balancerType, Name, providerRuntime, queueAdapter.GetStreamQueueMapper());
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, Name, providerRuntime, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout);
providerRuntime.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var managerGrainRef = PersistentStreamPullingManagerFactory.Cast(manager.AsReference());
// Need to call it as a grain reference though.
await managerGrainRef.Initialize(queueAdapter.AsImmutable());
}
}
public async Task StartPullingAgents(
string streamProviderName,
StreamQueueBalancerType balancerType,
IQueueAdapter queueAdapter,
TimeSpan getQueueMsgsTimerPeriod,
TimeSpan initQueueTimeout)
{
IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(
balancerType, streamProviderName, Silo.CurrentSilo.LocalSiloStatusOracle, this, queueAdapter.GetStreamQueueMapper());
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var managerGrainRef = PersistentStreamPullingManagerFactory.Cast(manager.AsReference());
// Need to call it as a grain reference though.
await managerGrainRef.Initialize(queueAdapter.AsImmutable());
}
public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter,
PersistentStreamOptions options)
{
IStreamQueueBalancer queueBalancer = CreateQueueBalancer(options, streamProviderName);
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(options.PubSubType), adapterFactory, queueBalancer, options, this.loggerFactory);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize(queueAdapter.AsImmutable());
return(pullingAgentManager);
}
public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter,
PersistentStreamProviderConfig config)
{
IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(config.BalancerType, streamProviderName, this.siloStatusOracle, this.siloDetails.ClusterConfig, this, adapterFactory.GetStreamQueueMapper(), config.SiloMaturityPeriod);
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(config.PubSubType), adapterFactory, queueBalancer, config);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize(queueAdapter.AsImmutable());
return(pullingAgentManager);
}
public Task InitializePullingAgents(
string streamProviderName,
StreamQueueBalancerType balancerType,
StreamPubSubType pubSubType,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter,
TimeSpan getQueueMsgsTimerPeriod,
TimeSpan initQueueTimeout,
TimeSpan maxEventDeliveryTime)
{
IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(
balancerType, streamProviderName, Silo.CurrentSilo.LocalSiloStatusOracle, Silo.CurrentSilo.OrleansConfig, this, adapterFactory.GetStreamQueueMapper());
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(pubSubType), adapterFactory, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout, maxEventDeliveryTime);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
return(pullingAgentManager.Initialize(queueAdapter.AsImmutable()));
}
private async Task InitAgent(PersistentStreamPullingAgent agent)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null);
IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(agent.QueueId);
// Need to call it as a grain reference.
var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable, deliveryFailureHandler.AsImmutable()));
await task.LogException(logger, ErrorCode.PersistentStreamPullingManager_09, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
}
/// <summary>
/// Take responsibility for a set of new queues that were assigned to me via a new range.
/// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents.
/// ERROR HANDLING:
/// The responsibility to handle initialization and shutdown failures is inside the Agents code.
/// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded.
/// Same applies to shutdown.
/// </summary>
/// <param name="myQueues"></param>
/// <param name="failOnInit"></param>
/// <returns></returns>
private async Task AddNewQueues(IEnumerable <QueueId> myQueues, bool failOnInit)
{
// Create agents for queues in range that we don't yet have.
// First create them and store in local queuesToAgentsMap.
// Only after that Initialize them all.
var agents = new List <PersistentStreamPullingAgent>();
foreach (var queueId in myQueues.Where(queueId => !queuesToAgentsMap.ContainsKey(queueId)))
{
try
{
var agentId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENT_SYSTEM_TARGET_TYPE_CODE);
var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, providerRuntime, queueId, queueGetPeriod, initQueueTimeout);
providerRuntime.RegisterSystemTarget(agent);
queuesToAgentsMap.Add(queueId, agent);
agents.Add(agent);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingManager_07, String.Format("Exception while creating PersistentStreamPullingAgent."), exc);
// What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down.
// If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers.
if (failOnInit)
{
throw;
}
}
}
try
{
var initTasks = new List <Task>();
foreach (var agent in agents)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null);
// Need to call it as a grain reference.
var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_08, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
initTasks.Add(task);
}
await Task.WhenAll(initTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
logger.Info((int)ErrorCode.PersistentStreamPullingManager_09, "Took {0} new queues under my responsibility: {1}", agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToStringWithHashCode()));
}