예제 #1
0
        private async Task OnRuntimeServicesStart(CancellationToken ct)
        {
            //TODO: Setup all (or as many as possible) of the class started in this call to work directly with lifecyce

            // The order of these 4 is pretty much arbitrary.
            scheduler.Start();
            messageCenter.Start();
            incomingPingAgent.Start();
            incomingSystemAgent.Start();
            incomingAgent.Start();

            LocalGrainDirectory.Start();

            // Set up an execution context for this thread so that the target creation steps can use asynch values.
            RuntimeContext.InitializeMainThread();

            // Initialize the implicit stream subscribers table.
            var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();
            var grainTypeManager = Services.GetRequiredService <GrainTypeManager>();

            implicitStreamSubscriberTable.InitImplicitStreamSubscribers(grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());

            var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>();

            runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
            statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>();
            string statsProviderName = await statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
                                       .WithTimeout(initTimeout);

            if (statsProviderName != null)
            {
                LocalConfig.StatisticsProviderName = statsProviderName;
            }

            // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
            await siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WithTimeout(initTimeout);

            await siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WithTimeout(initTimeout);


            // This has to follow the above steps that start the runtime components
            CreateSystemTargets();

            await InjectDependencies();

            // Validate the configuration.
            GlobalConfig.Application.ValidateConfiguration(logger);

            // Initialize storage providers once we have a basic silo runtime environment operating
            storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>();
            await scheduler.QueueTask(
                () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WithTimeout(initTimeout);

            ITransactionAgent  transactionAgent        = this.Services.GetRequiredService <ITransactionAgent>();
            ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;
            await scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
            .WithTimeout(initTimeout);

            var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore;

            versionStore?.SetStorageManager(storageProviderManager);
            logger.Debug("Storage provider manager created successfully.");

            // Initialize log consistency providers once we have a basic silo runtime environment operating
            logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>();
            await scheduler.QueueTask(
                () => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WithTimeout(initTimeout);

            logger.Debug("Log consistency provider manager created successfully.");

            // Load and init stream providers before silo becomes active
            var siloStreamProviderManager = (StreamProviderManager)this.Services.GetRequiredService <IStreamProviderManager>();
            await scheduler.QueueTask(
                () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
                providerManagerSystemTarget.SchedulingContext)
            .WithTimeout(initTimeout);

            runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
            logger.Debug("Stream provider manager created successfully.");

            // Load and init grain services before silo becomes active.
            await CreateGrainServices(GlobalConfig.GrainServiceConfigurations);

            this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
                                           this.providerManagerSystemTarget.SchedulingContext;
            await scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
            .WithTimeout(initTimeout);

            logger.Debug("Local silo status oracle created successfully.");
            await scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
            .WithTimeout(initTimeout);

            logger.Debug("Local silo status oracle became active successfully.");
            await scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
            .WithTimeout(this.initTimeout);

            //if running in multi cluster scenario, start the MultiClusterNetwork Oracle
            if (GlobalConfig.HasMultiClusterNetwork)
            {
                logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
                            GlobalConfig.ServiceId, GlobalConfig.ClusterId);

                this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
                                                 this.providerManagerSystemTarget.SchedulingContext;
                await scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
                .WithTimeout(initTimeout);

                logger.Debug("multicluster oracle created successfully.");
            }

            try
            {
                this.siloStatistics.Start(this.LocalConfig);
                logger.Debug("Silo statistics manager started successfully.");

                // Finally, initialize the deployment load collector, for grains with load-based placement
                var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
                await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
                .WithTimeout(this.initTimeout);

                logger.Debug("Silo deployment load publisher started successfully.");

                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants, this.executorService, this.loggerFactory);
                this.platformWatchdog.Start();
                if (this.logger.IsEnabled(LogLevel.Debug))
                {
                    logger.Debug("Silo platform watchdog started successfully.");
                }

                if (this.reminderService != null)
                {
                    // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                    this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
                                                  this.providerManagerSystemTarget.SchedulingContext;
                    await this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
                    .WithTimeout(this.initTimeout);

                    this.logger.Debug("Reminder service started successfully.");
                }

                this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>();
                await this.scheduler.QueueTask(
                    () => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
                    this.providerManagerSystemTarget.SchedulingContext)
                .WithTimeout(this.initTimeout);

                this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics

                logger.Debug("App bootstrap calls done successfully.");

                // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
                // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
                await this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
                .WithTimeout(this.initTimeout);

                logger.Debug("Stream providers started successfully.");

                // Now that we're active, we can start the gateway
                var mc = this.messageCenter as MessageCenter;
                mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
                logger.Debug("Message gateway service started successfully.");

                this.SystemStatus = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
                throw;
            }
            if (logger.IsEnabled(LogLevel.Debug))
            {
                logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
            }
        }
예제 #2
0
        private void DoStart()
        {
            lock (lockable)
            {
                if (!this.SystemStatus.Equals(SystemStatus.Created))
                {
                    throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", this.SystemStatus));
                }

                this.SystemStatus = SystemStatus.Starting;
            }

            logger.Info(ErrorCode.SiloStarting, "Silo Start()");

            // Hook up to receive notification of process exit / Ctrl-C events
            AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
            if (GlobalConfig.FastKillOnCancelKeyPress)
            {
                Console.CancelKeyPress += HandleProcessExit;
            }

            ConfigureThreadPoolAndServicePointSettings();

            // This has to start first so that the directory system target factory gets loaded before we start the router.
            grainTypeManager.Start();
            runtimeClient.Start();

            // The order of these 4 is pretty much arbitrary.
            scheduler.Start();
            messageCenter.Start();
            incomingPingAgent.Start();
            incomingSystemAgent.Start();
            incomingAgent.Start();

            LocalGrainDirectory.Start();

            // Set up an execution context for this thread so that the target creation steps can use asynch values.
            RuntimeContext.InitializeMainThread();

            // Initialize the implicit stream subscribers table.
            var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();

            implicitStreamSubscriberTable.InitImplicitStreamSubscribers(this.grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());

            var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>();

            runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
            statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>();
            string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
                                       .WaitForResultWithThrow(initTimeout);

            if (statsProviderName != null)
            {
                LocalConfig.StatisticsProviderName = statsProviderName;
            }

            // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
            siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);
            siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);


            // This has to follow the above steps that start the runtime components
            CreateSystemTargets();

            InjectDependencies();

            // Validate the configuration.
            GlobalConfig.Application.ValidateConfiguration(logger);

            // Initialize storage providers once we have a basic silo runtime environment operating
            storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>();
            scheduler.QueueTask(
                () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);

            ITransactionAgent  transactionAgent        = this.Services.GetRequiredService <ITransactionAgent>();
            ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;

            scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
            .WaitWithThrow(initTimeout);

            var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore;

            versionStore?.SetStorageManager(storageProviderManager);
            if (logger.IsVerbose)
            {
                logger.Verbose("Storage provider manager created successfully.");
            }

            // Initialize log consistency providers once we have a basic silo runtime environment operating
            logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>();
            scheduler.QueueTask(
                () => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Log consistency provider manager created successfully.");
            }

            // Load and init stream providers before silo becomes active
            var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;

            scheduler.QueueTask(
                () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
            if (logger.IsVerbose)
            {
                logger.Verbose("Stream provider manager created successfully.");
            }

            // Load and init grain services before silo becomes active.
            CreateGrainServices(GlobalConfig.GrainServiceConfigurations);

            this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
                                           this.providerManagerSystemTarget.SchedulingContext;
            scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Local silo status oracle created successfully.");
            }
            scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Local silo status oracle became active successfully.");
            }
            scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
            .WaitWithThrow(this.initTimeout);

            //if running in multi cluster scenario, start the MultiClusterNetwork Oracle
            if (GlobalConfig.HasMultiClusterNetwork)
            {
                logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
                            GlobalConfig.ServiceId, GlobalConfig.ClusterId);

                this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
                                                 this.providerManagerSystemTarget.SchedulingContext;
                scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
                .WaitWithThrow(initTimeout);
                if (logger.IsVerbose)
                {
                    logger.Verbose("multicluster oracle created successfully.");
                }
            }

            try
            {
                this.siloStatistics.Start(this.LocalConfig);
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Silo statistics manager started successfully.");
                }

                // Finally, initialize the deployment load collector, for grains with load-based placement
                var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
                this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
                .WaitWithThrow(this.initTimeout);
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Silo deployment load publisher started successfully.");
                }

                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants);
                this.platformWatchdog.Start();
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Silo platform watchdog started successfully.");
                }

                if (this.reminderService != null)
                {
                    // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                    this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
                                                  this.providerManagerSystemTarget.SchedulingContext;
                    this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
                    .WaitWithThrow(this.initTimeout);
                    if (this.logger.IsVerbose)
                    {
                        this.logger.Verbose("Reminder service started successfully.");
                    }
                }

                this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>();
                this.scheduler.QueueTask(
                    () => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
                    this.providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(this.initTimeout);
                this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics

                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("App bootstrap calls done successfully.");
                }

                // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
                // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
                this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(this.initTimeout);
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Stream providers started successfully.");
                }

                // Now that we're active, we can start the gateway
                var mc = this.messageCenter as MessageCenter;
                mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Message gateway service started successfully.");
                }

                this.SystemStatus = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
                this.FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
                throw;
            }
            if (logger.IsVerbose)
            {
                logger.Verbose("Silo.Start complete: System status = {0}", this.SystemStatus);
            }
        }