Ejemplo n.º 1
0
        public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
            string streamProviderName,
            IQueueAdapterFactory adapterFactory,
            IQueueAdapter queueAdapter)
        {
            IStreamQueueBalancer queueBalancer = CreateQueueBalancer(streamProviderName);
            var managerId           = SystemTargetGrainId.Create(Constants.StreamPullingAgentManagerType, this.siloDetails.SiloAddress, streamProviderName);
            var pubsubOptions       = this.ServiceProvider.GetOptionsByName <StreamPubSubOptions>(streamProviderName);
            var pullingAgentOptions = this.ServiceProvider.GetOptionsByName <StreamPullingAgentOptions>(streamProviderName);
            var manager             = new PersistentStreamPullingManager(
                managerId,
                streamProviderName,
                this,
                this.PubSub(pubsubOptions.PubSubType),
                adapterFactory,
                queueBalancer,
                pullingAgentOptions,
                this.loggerFactory,
                this.siloDetails.SiloAddress);

            this.RegisterSystemTarget(manager);
            // Init the manager only after it was registered locally.
            var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
            // Need to call it as a grain reference though.
            await pullingAgentManager.Initialize(queueAdapter.AsImmutable());

            return(pullingAgentManager);
        }
Ejemplo n.º 2
0
 public async Task Start()
 {
     if (providerRuntime.InSilo)
     {
         IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(balancerType, Name, providerRuntime, queueAdapter.GetStreamQueueMapper());
         var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
         var manager   = new PersistentStreamPullingManager(managerId, Name, providerRuntime, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout);
         providerRuntime.RegisterSystemTarget(manager);
         // Init the manager only after it was registered locally.
         var managerGrainRef = PersistentStreamPullingManagerFactory.Cast(manager.AsReference());
         // Need to call it as a grain reference though.
         await managerGrainRef.Initialize(queueAdapter.AsImmutable());
     }
 }
Ejemplo n.º 3
0
        public async Task StartPullingAgents(
            string streamProviderName,
            StreamQueueBalancerType balancerType,
            IQueueAdapter queueAdapter,
            TimeSpan getQueueMsgsTimerPeriod,
            TimeSpan initQueueTimeout)
        {
            IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(
                balancerType, streamProviderName, Silo.CurrentSilo.LocalSiloStatusOracle, this, queueAdapter.GetStreamQueueMapper());
            var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
            var manager   = new PersistentStreamPullingManager(managerId, streamProviderName, this, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout);

            this.RegisterSystemTarget(manager);
            // Init the manager only after it was registered locally.
            var managerGrainRef = PersistentStreamPullingManagerFactory.Cast(manager.AsReference());
            // Need to call it as a grain reference though.
            await managerGrainRef.Initialize(queueAdapter.AsImmutable());
        }
Ejemplo n.º 4
0
        public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
            string streamProviderName,
            IQueueAdapterFactory adapterFactory,
            IQueueAdapter queueAdapter,
            PersistentStreamOptions options)
        {
            IStreamQueueBalancer queueBalancer = CreateQueueBalancer(options, streamProviderName);
            var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
            var manager   = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(options.PubSubType), adapterFactory, queueBalancer, options, this.loggerFactory);

            this.RegisterSystemTarget(manager);
            // Init the manager only after it was registered locally.
            var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
            // Need to call it as a grain reference though.
            await pullingAgentManager.Initialize(queueAdapter.AsImmutable());

            return(pullingAgentManager);
        }
Ejemplo n.º 5
0
        public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
            string streamProviderName,
            IQueueAdapterFactory adapterFactory,
            IQueueAdapter queueAdapter,
            PersistentStreamProviderConfig config)
        {
            IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(config.BalancerType, streamProviderName, this.siloStatusOracle, this.siloDetails.ClusterConfig, this, adapterFactory.GetStreamQueueMapper(), config.SiloMaturityPeriod);
            var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
            var manager   = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(config.PubSubType), adapterFactory, queueBalancer, config);

            this.RegisterSystemTarget(manager);
            // Init the manager only after it was registered locally.
            var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
            // Need to call it as a grain reference though.
            await pullingAgentManager.Initialize(queueAdapter.AsImmutable());

            return(pullingAgentManager);
        }
Ejemplo n.º 6
0
        public Task InitializePullingAgents(
            string streamProviderName,
            StreamQueueBalancerType balancerType,
            StreamPubSubType pubSubType,
            IQueueAdapterFactory adapterFactory,
            IQueueAdapter queueAdapter,
            TimeSpan getQueueMsgsTimerPeriod,
            TimeSpan initQueueTimeout,
            TimeSpan maxEventDeliveryTime)
        {
            IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(
                balancerType, streamProviderName, Silo.CurrentSilo.LocalSiloStatusOracle, Silo.CurrentSilo.OrleansConfig, this, adapterFactory.GetStreamQueueMapper());
            var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
            var manager   = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(pubSubType), adapterFactory, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout, maxEventDeliveryTime);

            this.RegisterSystemTarget(manager);
            // Init the manager only after it was registered locally.
            pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
            // Need to call it as a grain reference though.
            return(pullingAgentManager.Initialize(queueAdapter.AsImmutable()));
        }
        private async Task InitAgent(PersistentStreamPullingAgent agent)
        {
            // Init the agent only after it was registered locally.
            var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
            var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null);

            IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(agent.QueueId);

            // Need to call it as a grain reference.
            var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable, deliveryFailureHandler.AsImmutable()));
            await task.LogException(logger, ErrorCode.PersistentStreamPullingManager_09, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
        }
Ejemplo n.º 8
0
        /// <summary>
        /// Take responsibility for a set of new queues that were assigned to me via a new range.
        /// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents.
        /// ERROR HANDLING:
        ///     The responsibility to handle initialization and shutdown failures is inside the Agents code.
        ///     The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded.
        ///     Same applies to shutdown.
        /// </summary>
        /// <param name="myQueues"></param>
        /// <param name="failOnInit"></param>
        /// <returns></returns>
        private async Task AddNewQueues(IEnumerable <QueueId> myQueues, bool failOnInit)
        {
            // Create agents for queues in range that we don't yet have.
            // First create them and store in local queuesToAgentsMap.
            // Only after that Initialize them all.
            var agents = new List <PersistentStreamPullingAgent>();

            foreach (var queueId in myQueues.Where(queueId => !queuesToAgentsMap.ContainsKey(queueId)))
            {
                try
                {
                    var agentId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENT_SYSTEM_TARGET_TYPE_CODE);
                    var agent   = new PersistentStreamPullingAgent(agentId, streamProviderName, providerRuntime, queueId, queueGetPeriod, initQueueTimeout);
                    providerRuntime.RegisterSystemTarget(agent);
                    queuesToAgentsMap.Add(queueId, agent);
                    agents.Add(agent);
                }
                catch (Exception exc)
                {
                    logger.Error((int)ErrorCode.PersistentStreamPullingManager_07, String.Format("Exception while creating PersistentStreamPullingAgent."), exc);
                    // What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down.
                    // If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers.
                    if (failOnInit)
                    {
                        throw;
                    }
                }
            }

            try
            {
                var initTasks = new List <Task>();
                foreach (var agent in agents)
                {
                    // Init the agent only after it was registered locally.
                    var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
                    var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null);

                    // Need to call it as a grain reference.
                    var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable));
                    task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_08, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
                    initTasks.Add(task);
                }
                await Task.WhenAll(initTasks);
            }
            catch
            {
                // Just ignore this exception and proceed as if Initialize has succeeded.
                // We already logged individual exceptions for individual calls to Initialize. No need to log again.
            }
            logger.Info((int)ErrorCode.PersistentStreamPullingManager_09, "Took {0} new queues under my responsibility: {1}", agents.Count,
                        Utils.EnumerableToString(agents, agent => agent.QueueId.ToStringWithHashCode()));
        }