예제 #1
0
 internal static void InitializeThread(TaskScheduler scheduler)
 {
     // There seems to be an implicit coupling of threads and contexts here that may be fragile. 
     // E.g. if InitializeThread() is mistakenly called on a wrong thread, would that thread be considered a worker pool thread from that point on? 
     // Is there a better/safer way to identify worker threads? 
     if (context != null && scheduler != null)
     {
         throw new InvalidOperationException("RuntimeContext.Current has already been initialized for this thread.");
     }
     context = new RuntimeContext {Scheduler = scheduler};
 }
예제 #2
0
 internal static void InitializeMainThread()
 {
     context = new RuntimeContext {Scheduler = null};
 }
예제 #3
0
        private void DoStart()
        {
            lock (lockable)
            {
                if (!this.SystemStatus.Equals(SystemStatus.Created))
                {
                    throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", this.SystemStatus));
                }

                this.SystemStatus = SystemStatus.Starting;
            }

            logger.Info(ErrorCode.SiloStarting, "Silo Start()");

            // Hook up to receive notification of process exit / Ctrl-C events
            AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
            if (GlobalConfig.FastKillOnCancelKeyPress)
            {
                Console.CancelKeyPress += HandleProcessExit;
            }

            ConfigureThreadPoolAndServicePointSettings();

            // This has to start first so that the directory system target factory gets loaded before we start the router.
            grainTypeManager.Start();
            runtimeClient.Start();

            // The order of these 4 is pretty much arbitrary.
            scheduler.Start();
            messageCenter.Start();
            incomingPingAgent.Start();
            incomingSystemAgent.Start();
            incomingAgent.Start();

            LocalGrainDirectory.Start();

            // Set up an execution context for this thread so that the target creation steps can use asynch values.
            RuntimeContext.InitializeMainThread();

            // Initialize the implicit stream subscribers table.
            var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();

            implicitStreamSubscriberTable.InitImplicitStreamSubscribers(this.grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());

            var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>();

            runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
            statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>();
            string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
                                       .WaitForResultWithThrow(initTimeout);

            if (statsProviderName != null)
            {
                LocalConfig.StatisticsProviderName = statsProviderName;
            }

            // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
            siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);
            siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);


            // This has to follow the above steps that start the runtime components
            CreateSystemTargets();

            InjectDependencies();

            // Validate the configuration.
            GlobalConfig.Application.ValidateConfiguration(logger);

            // Initialize storage providers once we have a basic silo runtime environment operating
            storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>();
            scheduler.QueueTask(
                () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);

            ITransactionAgent  transactionAgent        = this.Services.GetRequiredService <ITransactionAgent>();
            ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;

            scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
            .WaitWithThrow(initTimeout);

            var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore;

            versionStore?.SetStorageManager(storageProviderManager);
            if (logger.IsVerbose)
            {
                logger.Verbose("Storage provider manager created successfully.");
            }

            // Initialize log consistency providers once we have a basic silo runtime environment operating
            logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>();
            scheduler.QueueTask(
                () => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Log consistency provider manager created successfully.");
            }

            // Load and init stream providers before silo becomes active
            var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;

            scheduler.QueueTask(
                () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
            if (logger.IsVerbose)
            {
                logger.Verbose("Stream provider manager created successfully.");
            }

            // Load and init grain services before silo becomes active.
            CreateGrainServices(GlobalConfig.GrainServiceConfigurations);

            this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
                                           this.providerManagerSystemTarget.SchedulingContext;
            scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Local silo status oracle created successfully.");
            }
            scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Local silo status oracle became active successfully.");
            }
            scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
            .WaitWithThrow(this.initTimeout);

            //if running in multi cluster scenario, start the MultiClusterNetwork Oracle
            if (GlobalConfig.HasMultiClusterNetwork)
            {
                logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
                            GlobalConfig.ServiceId, GlobalConfig.ClusterId);

                this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
                                                 this.providerManagerSystemTarget.SchedulingContext;
                scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
                .WaitWithThrow(initTimeout);
                if (logger.IsVerbose)
                {
                    logger.Verbose("multicluster oracle created successfully.");
                }
            }

            try
            {
                this.siloStatistics.Start(this.LocalConfig);
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Silo statistics manager started successfully.");
                }

                // Finally, initialize the deployment load collector, for grains with load-based placement
                var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
                this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
                .WaitWithThrow(this.initTimeout);
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Silo deployment load publisher started successfully.");
                }

                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants);
                this.platformWatchdog.Start();
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Silo platform watchdog started successfully.");
                }

                if (this.reminderService != null)
                {
                    // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                    this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
                                                  this.providerManagerSystemTarget.SchedulingContext;
                    this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
                    .WaitWithThrow(this.initTimeout);
                    if (this.logger.IsVerbose)
                    {
                        this.logger.Verbose("Reminder service started successfully.");
                    }
                }

                this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>();
                this.scheduler.QueueTask(
                    () => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
                    this.providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(this.initTimeout);
                this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics

                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("App bootstrap calls done successfully.");
                }

                // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
                // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
                this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(this.initTimeout);
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Stream providers started successfully.");
                }

                // Now that we're active, we can start the gateway
                var mc = this.messageCenter as MessageCenter;
                mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Message gateway service started successfully.");
                }

                this.SystemStatus = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
                this.FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
                throw;
            }
            if (logger.IsVerbose)
            {
                logger.Verbose("Silo.Start complete: System status = {0}", this.SystemStatus);
            }
        }
예제 #4
0
 internal static void InitializeMainThread()
 {
     context = new RuntimeContext {
         Scheduler = null
     };
 }
예제 #5
0
파일: Silo.cs 프로젝트: uehara/orleans
        private void DoStart()
        {
            lock (lockable)
            {
                if (!SystemStatus.Current.Equals(SystemStatus.Created))
                {
                    throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", SystemStatus.Current));
                }

                SystemStatus.Current = SystemStatus.Starting;
            }

            logger.Info(ErrorCode.SiloStarting, "Silo Start()");

            // Hook up to receive notification of process exit / Ctrl-C events
            AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
            Console.CancelKeyPress += HandleProcessExit;

            ConfigureThreadPoolAndServicePointSettings();

            // This has to start first so that the directory system target factory gets loaded before we start the router.
            typeManager.Start();
            InsideRuntimeClient.Current.Start();

            // The order of these 4 is pretty much arbitrary.
            scheduler.Start();
            messageCenter.Start();
            incomingPingAgent.Start();
            incomingSystemAgent.Start();
            incomingAgent.Start();

            LocalGrainDirectory.Start();

            // Set up an execution context for this thread so that the target creation steps can use asynch values.
            RuntimeContext.InitializeMainThread();

            SiloProviderRuntime.Initialize(GlobalConfig, SiloIdentity, grainFactory);
            InsideRuntimeClient.Current.CurrentStreamProviderRuntime = SiloProviderRuntime.Instance;
            statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance);
            string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
                                       .WaitForResultWithThrow(initTimeout);

            if (statsProviderName != null)
            {
                LocalConfig.StatisticsProviderName = statsProviderName;
            }
            allSiloProviders.AddRange(statisticsProviderManager.GetProviders());

            // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
            siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
            siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);

            IMembershipTable membershipTable = membershipFactory.GetMembershipTable(GlobalConfig.LivenessType, GlobalConfig.MembershipTableAssembly);

            membershipOracle = membershipFactory.CreateMembershipOracle(this, membershipTable);

            // This has to follow the above steps that start the runtime components
            CreateSystemTargets();

            InjectDependencies();

            // Validate the configuration.
            GlobalConfig.Application.ValidateConfiguration(logger);

            // ensure this runs in the grain context, wait for it to complete
            scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("System grains created successfully.");
            }

            // Initialize storage providers once we have a basic silo runtime environment operating
            storageProviderManager = new StorageProviderManager(grainFactory);
            scheduler.QueueTask(
                () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            catalog.SetStorageManager(storageProviderManager);
            allSiloProviders.AddRange(storageProviderManager.GetProviders());
            if (logger.IsVerbose)
            {
                logger.Verbose("Storage provider manager created successfully.");
            }

            // Load and init stream providers before silo becomes active
            var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;

            scheduler.QueueTask(
                () => siloStreamProviderManager.LoadStreamProviders(this.GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager;
            allSiloProviders.AddRange(siloStreamProviderManager.GetProviders());
            if (logger.IsVerbose)
            {
                logger.Verbose("Stream provider manager created successfully.");
            }

            ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext;

            bool waitForPrimaryToStart = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary;

            if (waitForPrimaryToStart) // only in MembershipTableGrain case.
            {
                scheduler.QueueTask(() => membershipFactory.WaitForTableToInit(membershipTable), statusOracleContext)
                .WaitWithThrow(initTimeout);
            }
            scheduler.QueueTask(() => membershipTable.InitializeMembershipTable(this.GlobalConfig, true, TraceLogger.GetLogger(membershipTable.GetType().Name)), statusOracleContext)
            .WaitWithThrow(initTimeout);

            scheduler.QueueTask(() => LocalSiloStatusOracle.Start(), statusOracleContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Local silo status oracle created successfully.");
            }
            scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Local silo status oracle became active successfully.");
            }

            try
            {
                siloStatistics.Start(LocalConfig);
                if (logger.IsVerbose)
                {
                    logger.Verbose("Silo statistics manager started successfully.");
                }

                // Finally, initialize the deployment load collector, for grains with load-based placement
                scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext)
                .WaitWithThrow(initTimeout);
                if (logger.IsVerbose)
                {
                    logger.Verbose("Silo deployment load publisher started successfully.");
                }

                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants);
                platformWatchdog.Start();
                if (logger.IsVerbose)
                {
                    logger.Verbose("Silo platform watchdog started successfully.");
                }

                if (reminderService != null)
                {
                    // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                    scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext)
                    .WaitWithThrow(initTimeout);
                    if (logger.IsVerbose)
                    {
                        logger.Verbose("Reminder service started successfully.");
                    }
                }

                // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
                scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(initTimeout);
                if (logger.IsVerbose)
                {
                    logger.Verbose("Stream providers started successfully.");
                }

                var bootstrapProviderManager = new BootstrapProviderManager();
                scheduler.QueueTask(
                    () => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations),
                    providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(initTimeout);
                BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
                allSiloProviders.AddRange(BootstrapProviders);

                if (logger.IsVerbose)
                {
                    logger.Verbose("App bootstrap calls done successfully.");
                }

                // Now that we're active, we can start the gateway
                var mc = messageCenter as MessageCenter;
                if (mc != null)
                {
                    mc.StartGateway(clientRegistrar);
                }
                if (logger.IsVerbose)
                {
                    logger.Verbose("Message gateway service started successfully.");
                }

                scheduler.QueueTask(clientRegistrar.Start, clientRegistrar.SchedulingContext)
                .WaitWithThrow(initTimeout);
                if (logger.IsVerbose)
                {
                    logger.Verbose("Client registrar service started successfully.");
                }

                SystemStatus.Current = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc));
                FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
                throw;
            }
            if (logger.IsVerbose)
            {
                logger.Verbose("Silo.Start complete: System status = {0}", SystemStatus.Current);
            }
        }
예제 #6
0
파일: Silo.cs 프로젝트: suifing/orleans
        private async Task OnRuntimeServicesStart(CancellationToken ct)
        {
            //TODO: Setup all (or as many as possible) of the class started in this call to work directly with lifecyce

            // The order of these 4 is pretty much arbitrary.
            scheduler.Start();
            messageCenter.Start();
            incomingPingAgent.Start();
            incomingSystemAgent.Start();
            incomingAgent.Start();

            LocalGrainDirectory.Start();

            // Set up an execution context for this thread so that the target creation steps can use asynch values.
            RuntimeContext.InitializeMainThread();

            // Initialize the implicit stream subscribers table.
            var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();
            var grainTypeManager = Services.GetRequiredService <GrainTypeManager>();

            implicitStreamSubscriberTable.InitImplicitStreamSubscribers(grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());

            var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>();

            runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
            statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>();
            string statsProviderName = await statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
                                       .WithTimeout(initTimeout);

            if (statsProviderName != null)
            {
                LocalConfig.StatisticsProviderName = statsProviderName;
            }

            // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
            await siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WithTimeout(initTimeout);

            await siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WithTimeout(initTimeout);


            // This has to follow the above steps that start the runtime components
            CreateSystemTargets();

            await InjectDependencies();

            // Validate the configuration.
            GlobalConfig.Application.ValidateConfiguration(logger);

            // Initialize storage providers once we have a basic silo runtime environment operating
            storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>();
            await scheduler.QueueTask(
                () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WithTimeout(initTimeout);

            ITransactionAgent  transactionAgent        = this.Services.GetRequiredService <ITransactionAgent>();
            ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;
            await scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
            .WithTimeout(initTimeout);

            var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore;

            versionStore?.SetStorageManager(storageProviderManager);
            logger.Debug("Storage provider manager created successfully.");

            // Initialize log consistency providers once we have a basic silo runtime environment operating
            logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>();
            await scheduler.QueueTask(
                () => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WithTimeout(initTimeout);

            logger.Debug("Log consistency provider manager created successfully.");

            // Load and init stream providers before silo becomes active
            var siloStreamProviderManager = (StreamProviderManager)this.Services.GetRequiredService <IStreamProviderManager>();
            await scheduler.QueueTask(
                () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
                providerManagerSystemTarget.SchedulingContext)
            .WithTimeout(initTimeout);

            runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
            logger.Debug("Stream provider manager created successfully.");

            // Load and init grain services before silo becomes active.
            await CreateGrainServices(GlobalConfig.GrainServiceConfigurations);

            this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
                                           this.providerManagerSystemTarget.SchedulingContext;
            await scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
            .WithTimeout(initTimeout);

            logger.Debug("Local silo status oracle created successfully.");
            await scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
            .WithTimeout(initTimeout);

            logger.Debug("Local silo status oracle became active successfully.");
            await scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
            .WithTimeout(this.initTimeout);

            //if running in multi cluster scenario, start the MultiClusterNetwork Oracle
            if (GlobalConfig.HasMultiClusterNetwork)
            {
                logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
                            GlobalConfig.ServiceId, GlobalConfig.ClusterId);

                this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
                                                 this.providerManagerSystemTarget.SchedulingContext;
                await scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
                .WithTimeout(initTimeout);

                logger.Debug("multicluster oracle created successfully.");
            }

            try
            {
                this.siloStatistics.Start(this.LocalConfig);
                logger.Debug("Silo statistics manager started successfully.");

                // Finally, initialize the deployment load collector, for grains with load-based placement
                var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
                await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
                .WithTimeout(this.initTimeout);

                logger.Debug("Silo deployment load publisher started successfully.");

                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants, this.executorService, this.loggerFactory);
                this.platformWatchdog.Start();
                if (this.logger.IsEnabled(LogLevel.Debug))
                {
                    logger.Debug("Silo platform watchdog started successfully.");
                }

                if (this.reminderService != null)
                {
                    // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                    this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
                                                  this.providerManagerSystemTarget.SchedulingContext;
                    await this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
                    .WithTimeout(this.initTimeout);

                    this.logger.Debug("Reminder service started successfully.");
                }

                this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>();
                await this.scheduler.QueueTask(
                    () => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
                    this.providerManagerSystemTarget.SchedulingContext)
                .WithTimeout(this.initTimeout);

                this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics

                logger.Debug("App bootstrap calls done successfully.");

                // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
                // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
                await this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
                .WithTimeout(this.initTimeout);

                logger.Debug("Stream providers started successfully.");

                // Now that we're active, we can start the gateway
                var mc = this.messageCenter as MessageCenter;
                mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
                logger.Debug("Message gateway service started successfully.");

                this.SystemStatus = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
                throw;
            }
            if (logger.IsEnabled(LogLevel.Debug))
            {
                logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
            }
        }