Beispiel #1
0
        private async Task OnRuntimeGrainServicesStart(CancellationToken ct)
        {
            var stopWatch = Stopwatch.StartNew();

            // Load and init grain services before silo becomes active.
            await StartAsyncTaskWithPerfAnalysis("Init grain services",
                                                 () => CreateGrainServices(), stopWatch);

            var versionStore = Services.GetService <IVersionStore>();

            await StartAsyncTaskWithPerfAnalysis("Init type manager", () => scheduler
                                                 .QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager)
                                                 .WithTimeout(this.initTimeout, $"TypeManager Initializing failed due to timeout {initTimeout}"), stopWatch);

            try
            {
                StatisticsOptions statisticsOptions = Services.GetRequiredService <IOptions <StatisticsOptions> >().Value;
                StartTaskWithPerfAnalysis("Start silo statistics", () => this.siloStatistics.Start(statisticsOptions), stopWatch);
                logger.Debug("Silo statistics manager started successfully.");

                // Finally, initialize the deployment load collector, for grains with load-based placement
                await StartAsyncTaskWithPerfAnalysis("Start deployment load collector", StartDeploymentLoadCollector, stopWatch);

                async Task StartDeploymentLoadCollector()
                {
                    var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();

                    await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher)
                    .WithTimeout(this.initTimeout, $"Starting DeploymentLoadPublisher failed due to timeout {initTimeout}");

                    logger.Debug("Silo deployment load publisher started successfully.");
                }


                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                var healthCheckParticipants = this.Services.GetService <IEnumerable <IHealthCheckParticipant> >().ToList();
                this.platformWatchdog = new Watchdog(statisticsOptions.LogWriteInterval, healthCheckParticipants, this.loggerFactory.CreateLogger <Watchdog>());
                this.platformWatchdog.Start();
                if (this.logger.IsEnabled(LogLevel.Debug))
                {
                    logger.Debug("Silo platform watchdog started successfully.");
                }
            }
            catch (Exception exc)
            {
                this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
                throw;
            }
            if (logger.IsEnabled(LogLevel.Debug))
            {
                logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
            }
        }
Beispiel #2
0
        private async Task OnRuntimeServicesStart(CancellationToken ct)
        {
            //TODO: Setup all (or as many as possible) of the class started in this call to work directly with lifecyce

            // The order of these 4 is pretty much arbitrary.
            scheduler.Start();
            messageCenter.Start();
            incomingPingAgent.Start();
            incomingSystemAgent.Start();
            incomingAgent.Start();

            LocalGrainDirectory.Start();

            // Set up an execution context for this thread so that the target creation steps can use asynch values.
            RuntimeContext.InitializeMainThread();

            // Initialize the implicit stream subscribers table.
            var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();
            var grainTypeManager = Services.GetRequiredService <GrainTypeManager>();

            implicitStreamSubscriberTable.InitImplicitStreamSubscribers(grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());

            var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>();

            runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
            statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>();
            string statsProviderName = await statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
                                       .WithTimeout(initTimeout);

            if (statsProviderName != null)
            {
                LocalConfig.StatisticsProviderName = statsProviderName;
            }

            // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
            await siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WithTimeout(initTimeout);

            await siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WithTimeout(initTimeout);


            // This has to follow the above steps that start the runtime components
            CreateSystemTargets();

            await InjectDependencies();

            // Validate the configuration.
            GlobalConfig.Application.ValidateConfiguration(logger);

            // Initialize storage providers once we have a basic silo runtime environment operating
            storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>();
            await scheduler.QueueTask(
                () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WithTimeout(initTimeout);

            ITransactionAgent  transactionAgent        = this.Services.GetRequiredService <ITransactionAgent>();
            ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;
            await scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
            .WithTimeout(initTimeout);

            var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore;

            versionStore?.SetStorageManager(storageProviderManager);
            logger.Debug("Storage provider manager created successfully.");

            // Initialize log consistency providers once we have a basic silo runtime environment operating
            logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>();
            await scheduler.QueueTask(
                () => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WithTimeout(initTimeout);

            logger.Debug("Log consistency provider manager created successfully.");

            // Load and init stream providers before silo becomes active
            var siloStreamProviderManager = (StreamProviderManager)this.Services.GetRequiredService <IStreamProviderManager>();
            await scheduler.QueueTask(
                () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
                providerManagerSystemTarget.SchedulingContext)
            .WithTimeout(initTimeout);

            runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
            logger.Debug("Stream provider manager created successfully.");

            // Load and init grain services before silo becomes active.
            await CreateGrainServices(GlobalConfig.GrainServiceConfigurations);

            this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
                                           this.providerManagerSystemTarget.SchedulingContext;
            await scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
            .WithTimeout(initTimeout);

            logger.Debug("Local silo status oracle created successfully.");
            await scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
            .WithTimeout(initTimeout);

            logger.Debug("Local silo status oracle became active successfully.");
            await scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
            .WithTimeout(this.initTimeout);

            //if running in multi cluster scenario, start the MultiClusterNetwork Oracle
            if (GlobalConfig.HasMultiClusterNetwork)
            {
                logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
                            GlobalConfig.ServiceId, GlobalConfig.ClusterId);

                this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
                                                 this.providerManagerSystemTarget.SchedulingContext;
                await scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
                .WithTimeout(initTimeout);

                logger.Debug("multicluster oracle created successfully.");
            }

            try
            {
                this.siloStatistics.Start(this.LocalConfig);
                logger.Debug("Silo statistics manager started successfully.");

                // Finally, initialize the deployment load collector, for grains with load-based placement
                var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
                await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
                .WithTimeout(this.initTimeout);

                logger.Debug("Silo deployment load publisher started successfully.");

                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants, this.executorService, this.loggerFactory);
                this.platformWatchdog.Start();
                if (this.logger.IsEnabled(LogLevel.Debug))
                {
                    logger.Debug("Silo platform watchdog started successfully.");
                }

                if (this.reminderService != null)
                {
                    // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                    this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
                                                  this.providerManagerSystemTarget.SchedulingContext;
                    await this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
                    .WithTimeout(this.initTimeout);

                    this.logger.Debug("Reminder service started successfully.");
                }

                this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>();
                await this.scheduler.QueueTask(
                    () => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
                    this.providerManagerSystemTarget.SchedulingContext)
                .WithTimeout(this.initTimeout);

                this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics

                logger.Debug("App bootstrap calls done successfully.");

                // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
                // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
                await this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
                .WithTimeout(this.initTimeout);

                logger.Debug("Stream providers started successfully.");

                // Now that we're active, we can start the gateway
                var mc = this.messageCenter as MessageCenter;
                mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
                logger.Debug("Message gateway service started successfully.");

                this.SystemStatus = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
                throw;
            }
            if (logger.IsEnabled(LogLevel.Debug))
            {
                logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
            }
        }
Beispiel #3
0
        private void DoStart()
        {
            lock (lockable)
            {
                if (SystemStatus.Current != SystemStatus.Created)
                {
                    throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Start state. This silo is in the {0} state.", SystemStatus.Current));
                }

                SystemStatus.Current = SystemStatus.Starting;
            }

            logger.Info(ErrorCode.SiloStarting, "Silo Start()");

            // Hook up to receive notification of process exit / Ctrl-C events
            AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
            Console.CancelKeyPress += HandleProcessExit;

            ConfigureThreadPoolAndServicePointSettings();

            // This has to start first so that the directory system target factory gets loaded before we start the router.
            typeManager.Start();
            InsideRuntimeClient.Current.Start();

            // The order of these 4 is pretty much arbitrary.
            scheduler.Start();
            messageCenter.Start();
            incomingPingAgent.Start();
            incomingSystemAgent.Start();
            incomingAgent.Start();

            LocalGrainDirectory.Start();

            // Set up an execution context for this thread so that the target creation steps can use asynch values.
            RuntimeContext.InitializeMainThread();

            SiloProviderRuntime.Initialize(GlobalConfig);
            statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance);
            string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
                                       .WaitForResultWithThrow(initTimeout);

            if (statsProviderName != null)
            {
                LocalConfig.StatisticsProviderName = statsProviderName;
            }

            // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
            siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
            siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);

            membershipOracle = membershipFactory.CreateMembershipOracle(this).WaitForResultWithThrow(initTimeout);

            // This has to follow the above steps that start the runtime components
            CreateSystemTargets();

            InjectDependencies();

            // ensure this runs in the grain context, wait for it to complete
            scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext)
            .WaitWithThrow(initTimeout);

            // Initialize storage providers once we have a basic silo runtime environment operating
            storageProviderManager = new StorageProviderManager();
            scheduler.QueueTask(
                () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            catalog.SetStorageManager(storageProviderManager);

            // Load and init stream providers before silo becomes active
            var siloStreamProviderManager = new Orleans.Streams.StreamProviderManager();

            scheduler.QueueTask(
                () => siloStreamProviderManager.LoadStreamProviders(this.GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager;

            ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext;
            bool waitForPrimaryToStart             = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary;

            scheduler.QueueTask(() => LocalSiloStatusOracle.Start(waitForPrimaryToStart), statusOracleContext)
            .WaitWithThrow(initTimeout);
            scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext)
            .WaitWithThrow(initTimeout);

            try
            {
                siloStatistics.Start(LocalConfig);

                // Finally, initialize the deployment load collector, for grains with load-based placement
                scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext)
                .WaitWithThrow(initTimeout);

                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants);
                platformWatchdog.Start();

                // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext)
                .WaitWithThrow(initTimeout);

                // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
                scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(initTimeout);

                var bootstrapProviderManager = new BootstrapProviderManager();
                scheduler.QueueTask(
                    () => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations),
                    providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(initTimeout);
                BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics

                // Now that we're active, we can start the gateway
                var mc = messageCenter as MessageCenter;
                if (mc != null)
                {
                    mc.StartGateway();
                }

                SystemStatus.Current = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc));
                FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
                throw;
            }
        }
Beispiel #4
0
        private async Task OnRuntimeGrainServicesStart(CancellationToken ct)
        {
            var stopWatch = Stopwatch.StartNew();

            // Load and init grain services before silo becomes active.
            await StartAsyncTaskWithPerfAnalysis("Init grain services",
                                                 () => CreateGrainServices(), stopWatch);

            this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
                                           this.fallbackScheduler.SchedulingContext;

            await StartAsyncTaskWithPerfAnalysis("Starting local silo status oracle", StartMembershipOracle, stopWatch);

            async Task StartMembershipOracle()
            {
                await scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
                .WithTimeout(initTimeout, $"Starting MembershipOracle failed due to timeout {initTimeout}");

                logger.Debug("Local silo status oracle created successfully.");
            }

            var versionStore = Services.GetService <IVersionStore>();

            await StartAsyncTaskWithPerfAnalysis("Init type manager", () => scheduler
                                                 .QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
                                                 .WithTimeout(this.initTimeout, $"TypeManager Initializing failed due to timeout {initTimeout}"), stopWatch);

            //if running in multi cluster scenario, start the MultiClusterNetwork Oracle
            if (this.multiClusterOracle != null)
            {
                await StartAsyncTaskWithPerfAnalysis("Start multicluster oracle", StartMultiClusterOracle, stopWatch);

                async Task StartMultiClusterOracle()
                {
                    logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
                                this.clusterOptions.ServiceId, this.clusterOptions.ClusterId);

                    this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
                                                     this.fallbackScheduler.SchedulingContext;
                    await scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
                    .WithTimeout(initTimeout, $"Starting MultiClusterOracle failed due to timeout {initTimeout}");

                    logger.Debug("multicluster oracle created successfully.");
                }
            }

            try
            {
                StatisticsOptions statisticsOptions = Services.GetRequiredService <IOptions <StatisticsOptions> >().Value;
                StartTaskWithPerfAnalysis("Start silo statistics", () => this.siloStatistics.Start(statisticsOptions), stopWatch);
                logger.Debug("Silo statistics manager started successfully.");

                // Finally, initialize the deployment load collector, for grains with load-based placement
                await StartAsyncTaskWithPerfAnalysis("Start deployment load collector", StartDeploymentLoadCollector, stopWatch);

                async Task StartDeploymentLoadCollector()
                {
                    var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();

                    await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
                    .WithTimeout(this.initTimeout, $"Starting DeploymentLoadPublisher failed due to timeout {initTimeout}");

                    logger.Debug("Silo deployment load publisher started successfully.");
                }


                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                this.platformWatchdog = new Watchdog(statisticsOptions.LogWriteInterval, this.healthCheckParticipants, this.executorService, this.loggerFactory);
                this.platformWatchdog.Start();
                if (this.logger.IsEnabled(LogLevel.Debug))
                {
                    logger.Debug("Silo platform watchdog started successfully.");
                }
            }
            catch (Exception exc)
            {
                this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
                throw;
            }
            if (logger.IsEnabled(LogLevel.Debug))
            {
                logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
            }
        }
Beispiel #5
0
        private void DoStart()
        {
            lock (lockable)
            {
                if (!SystemStatus.Current.Equals(SystemStatus.Created))
                    throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", SystemStatus.Current));
                
                SystemStatus.Current = SystemStatus.Starting;
            }

            logger.Info(ErrorCode.SiloStarting, "Silo Start()");

            // Hook up to receive notification of process exit / Ctrl-C events
            AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
            Console.CancelKeyPress += HandleProcessExit;

            ConfigureThreadPoolAndServicePointSettings();

            // This has to start first so that the directory system target factory gets loaded before we start the router.
            grainTypeManager.Start();
            runtimeClient.Start();

            // The order of these 4 is pretty much arbitrary.
            scheduler.Start();
            messageCenter.Start();
            incomingPingAgent.Start();
            incomingSystemAgent.Start();
            incomingAgent.Start();

            LocalGrainDirectory.Start();

            // Set up an execution context for this thread so that the target creation steps can use asynch values.
            RuntimeContext.InitializeMainThread();
            
            // Initialize the implicit stream subscribers table.
            var implicitStreamSubscriberTable = Services.GetRequiredService<ImplicitStreamSubscriberTable>();
            implicitStreamSubscriberTable.InitImplicitStreamSubscribers(this.grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());

            var siloProviderRuntime = Services.GetRequiredService<SiloProviderRuntime>();
            runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
            statisticsProviderManager = new StatisticsProviderManager("Statistics", siloProviderRuntime);
            string statsProviderName =  statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
                .WaitForResultWithThrow(initTimeout);
            if (statsProviderName != null)
                LocalConfig.StatisticsProviderName = statsProviderName;
            allSiloProviders.AddRange(statisticsProviderManager.GetProviders());

            // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
            siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);
            siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);

            IMembershipTable membershipTable = Services.GetRequiredService<IMembershipTable>();
            
            multiClusterOracle =
                Services.GetRequiredService<MultiClusterOracleFactory>().CreateGossipOracle(this).WaitForResultWithThrow(initTimeout);

            // This has to follow the above steps that start the runtime components
            CreateSystemTargets();

            InjectDependencies();

            // Validate the configuration.
            GlobalConfig.Application.ValidateConfiguration(logger);

            // ensure this runs in the grain context, wait for it to complete
            scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext)
                .WaitWithThrow(initTimeout);
            if (logger.IsVerbose) {  logger.Verbose("System grains created successfully."); }

            // Initialize storage providers once we have a basic silo runtime environment operating
            storageProviderManager = new StorageProviderManager(grainFactory, Services, siloProviderRuntime);
            scheduler.QueueTask(
                () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
                    .WaitWithThrow(initTimeout);
            catalog.SetStorageManager(storageProviderManager);
            allSiloProviders.AddRange(storageProviderManager.GetProviders());
            if (logger.IsVerbose) { logger.Verbose("Storage provider manager created successfully."); }

            // Load and init stream providers before silo becomes active
            var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;
            scheduler.QueueTask(
                () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
                    providerManagerSystemTarget.SchedulingContext)
                        .WaitWithThrow(initTimeout);
            runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
            allSiloProviders.AddRange(siloStreamProviderManager.GetProviders());
            if (logger.IsVerbose) { logger.Verbose("Stream provider manager created successfully."); }

            ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext;

            bool waitForPrimaryToStart = GlobalConfig.PrimaryNodeIsRequired && Type != SiloType.Primary;
            if (waitForPrimaryToStart) // only in MembershipTableGrain case.
            {
                scheduler.QueueTask(() => membershipFactory.WaitForTableToInit(membershipTable), statusOracleContext)
                        .WaitWithThrow(initTimeout);
            }
            scheduler.QueueTask(() => membershipTable.InitializeMembershipTable(GlobalConfig, true, LogManager.GetLogger(membershipTable.GetType().Name)), statusOracleContext)
                .WaitWithThrow(initTimeout);
          
            scheduler.QueueTask(() => LocalSiloStatusOracle.Start(), statusOracleContext)
                .WaitWithThrow(initTimeout);
            if (logger.IsVerbose) { logger.Verbose("Local silo status oracle created successfully."); }
            scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext)
                .WaitWithThrow(initTimeout);
            if (logger.IsVerbose) { logger.Verbose("Local silo status oracle became active successfully."); }

            //if running in multi cluster scenario, start the MultiClusterNetwork Oracle
            if (GlobalConfig.HasMultiClusterNetwork) 
            {
                logger.Info("Creating multicluster oracle with my ServiceId={0} and ClusterId={1}.",
                    GlobalConfig.ServiceId, GlobalConfig.ClusterId);

                ISchedulingContext clusterStatusContext = ((SystemTarget) multiClusterOracle).SchedulingContext;
                scheduler.QueueTask(() => multiClusterOracle.Start(LocalSiloStatusOracle), clusterStatusContext)
                                    .WaitWithThrow(initTimeout);
                if (logger.IsVerbose) { logger.Verbose("multicluster oracle created successfully."); }
            }

            try
            {
                this.siloStatistics.Start(this.LocalConfig);
                if (this.logger.IsVerbose) { this.logger.Verbose("Silo statistics manager started successfully."); }

                // Finally, initialize the deployment load collector, for grains with load-based placement
                var deploymentLoadPublisher = Services.GetRequiredService<DeploymentLoadPublisher>();
                this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
                    .WaitWithThrow(this.initTimeout);
                if (this.logger.IsVerbose) { this.logger.Verbose("Silo deployment load publisher started successfully."); }

                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants);
                this.platformWatchdog.Start();
                if (this.logger.IsVerbose) { this.logger.Verbose("Silo platform watchdog started successfully."); }

                if (this.reminderService != null)
                {
                    // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                    this.scheduler.QueueTask(this.reminderService.Start, ((SystemTarget)this.reminderService).SchedulingContext)
                        .WaitWithThrow(this.initTimeout);
                    if (this.logger.IsVerbose)
                    {
                        this.logger.Verbose("Reminder service started successfully.");
                    }
                }

                this.bootstrapProviderManager = new BootstrapProviderManager();
                this.scheduler.QueueTask(
                    () => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
                    this.providerManagerSystemTarget.SchedulingContext)
                        .WaitWithThrow(this.initTimeout);
                this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
                this.allSiloProviders.AddRange(this.BootstrapProviders);

                if (this.logger.IsVerbose) { this.logger.Verbose("App bootstrap calls done successfully."); }

                // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
                // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
                this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
                    .WaitWithThrow(this.initTimeout);
                if (this.logger.IsVerbose) { this.logger.Verbose("Stream providers started successfully."); }

                // Now that we're active, we can start the gateway
                var mc = this.messageCenter as MessageCenter;
                mc?.StartGateway(this.Services.GetRequiredService<ClientObserverRegistrar>());
                if (this.logger.IsVerbose) { this.logger.Verbose("Message gateway service started successfully."); }

                SystemStatus.Current = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
                this.FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
                throw;
            }
            if (logger.IsVerbose) { logger.Verbose("Silo.Start complete: System status = {0}", SystemStatus.Current); }
        }
Beispiel #6
0
        private async Task OnRuntimeGrainServicesStart(CancellationToken ct)
        {
            var stopWatch = Stopwatch.StartNew();

            await StartAsyncTaskWithPerfAnalysis("Init transaction agent", InitTransactionAgent, stopWatch);

            async Task InitTransactionAgent()
            {
                ITransactionAgent  transactionAgent        = this.Services.GetRequiredService <ITransactionAgent>();
                ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;
                await scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
                .WithTimeout(initTimeout);
            }

            // Load and init grain services before silo becomes active.
            await StartAsyncTaskWithPerfAnalysis("Init grain services",
                                                 () => CreateGrainServices(GlobalConfig.GrainServiceConfigurations), stopWatch);

            this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
                                           this.fallbackScheduler.SchedulingContext;

            await StartAsyncTaskWithPerfAnalysis("Start local silo status oracle", StartMembershipOracle, stopWatch);

            async Task StartMembershipOracle()
            {
                await scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
                .WithTimeout(initTimeout);

                logger.Debug("Local silo status oracle created successfully.");
                await scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
                .WithTimeout(initTimeout);

                logger.Debug("Local silo status oracle became active successfully.");
            }

            var versionStore = Services.GetService <IVersionStore>();

            await StartAsyncTaskWithPerfAnalysis("Init type manager", () => scheduler
                                                 .QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
                                                 .WithTimeout(this.initTimeout), stopWatch);

            //if running in multi cluster scenario, start the MultiClusterNetwork Oracle
            if (this.multiClusterOracle != null)
            {
                await StartAsyncTaskWithPerfAnalysis("Start multicluster oracle", StartMultiClusterOracle, stopWatch);

                async Task StartMultiClusterOracle()
                {
                    logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
                                this.siloOptions.ServiceId, this.siloOptions.ClusterId);

                    this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
                                                     this.fallbackScheduler.SchedulingContext;
                    await scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
                    .WithTimeout(initTimeout);

                    logger.Debug("multicluster oracle created successfully.");
                }
            }

            try
            {
                StatisticsOptions statisticsOptions = Services.GetRequiredService <IOptions <StatisticsOptions> >().Value;
                StartTaskWithPerfAnalysis("Start silo statistics", () => this.siloStatistics.Start(statisticsOptions), stopWatch);
                logger.Debug("Silo statistics manager started successfully.");

                // Finally, initialize the deployment load collector, for grains with load-based placement
                await StartAsyncTaskWithPerfAnalysis("Start deployment load collector", StartDeploymentLoadCollector, stopWatch);

                async Task StartDeploymentLoadCollector()
                {
                    var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();

                    await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
                    .WithTimeout(this.initTimeout);

                    logger.Debug("Silo deployment load publisher started successfully.");
                }


                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants, this.executorService, this.loggerFactory);
                this.platformWatchdog.Start();
                if (this.logger.IsEnabled(LogLevel.Debug))
                {
                    logger.Debug("Silo platform watchdog started successfully.");
                }

                if (this.reminderService != null)
                {
                    await StartAsyncTaskWithPerfAnalysis("Start reminder service", StartReminderService, stopWatch);

                    async Task StartReminderService()
                    {
                        // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                        this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
                                                      this.fallbackScheduler.SchedulingContext;
                        await this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
                        .WithTimeout(this.initTimeout);

                        this.logger.Debug("Reminder service started successfully.");
                    }
                }

                StartTaskWithPerfAnalysis("Start gateway", StartGateway, stopWatch);
                void StartGateway()
                {
                    // Now that we're active, we can start the gateway
                    var mc = this.messageCenter as MessageCenter;

                    mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
                    logger.Debug("Message gateway service started successfully.");
                }
                this.SystemStatus = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
                throw;
            }
            if (logger.IsEnabled(LogLevel.Debug))
            {
                logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
            }
        }
Beispiel #7
0
        private void DoStart()
        {
            lock (lockable)
            {
                if (!this.SystemStatus.Equals(SystemStatus.Created))
                {
                    throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", this.SystemStatus));
                }

                this.SystemStatus = SystemStatus.Starting;
            }

            logger.Info(ErrorCode.SiloStarting, "Silo Start()");

            // Hook up to receive notification of process exit / Ctrl-C events
            AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
            if (GlobalConfig.FastKillOnCancelKeyPress)
            {
                Console.CancelKeyPress += HandleProcessExit;
            }

            ConfigureThreadPoolAndServicePointSettings();

            // This has to start first so that the directory system target factory gets loaded before we start the router.
            grainTypeManager.Start();
            runtimeClient.Start();

            // The order of these 4 is pretty much arbitrary.
            scheduler.Start();
            messageCenter.Start();
            incomingPingAgent.Start();
            incomingSystemAgent.Start();
            incomingAgent.Start();

            LocalGrainDirectory.Start();

            // Set up an execution context for this thread so that the target creation steps can use asynch values.
            RuntimeContext.InitializeMainThread();

            // Initialize the implicit stream subscribers table.
            var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();

            implicitStreamSubscriberTable.InitImplicitStreamSubscribers(this.grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());

            var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>();

            runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
            statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>();
            string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
                                       .WaitForResultWithThrow(initTimeout);

            if (statsProviderName != null)
            {
                LocalConfig.StatisticsProviderName = statsProviderName;
            }

            // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
            siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);
            siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);


            // This has to follow the above steps that start the runtime components
            CreateSystemTargets();

            InjectDependencies();

            // Validate the configuration.
            GlobalConfig.Application.ValidateConfiguration(logger);

            // Initialize storage providers once we have a basic silo runtime environment operating
            storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>();
            scheduler.QueueTask(
                () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);

            ITransactionAgent  transactionAgent        = this.Services.GetRequiredService <ITransactionAgent>();
            ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;

            scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
            .WaitWithThrow(initTimeout);

            var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore;

            versionStore?.SetStorageManager(storageProviderManager);
            if (logger.IsVerbose)
            {
                logger.Verbose("Storage provider manager created successfully.");
            }

            // Initialize log consistency providers once we have a basic silo runtime environment operating
            logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>();
            scheduler.QueueTask(
                () => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Log consistency provider manager created successfully.");
            }

            // Load and init stream providers before silo becomes active
            var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;

            scheduler.QueueTask(
                () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
            if (logger.IsVerbose)
            {
                logger.Verbose("Stream provider manager created successfully.");
            }

            // Load and init grain services before silo becomes active.
            CreateGrainServices(GlobalConfig.GrainServiceConfigurations);

            this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
                                           this.providerManagerSystemTarget.SchedulingContext;
            scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Local silo status oracle created successfully.");
            }
            scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Local silo status oracle became active successfully.");
            }
            scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
            .WaitWithThrow(this.initTimeout);

            //if running in multi cluster scenario, start the MultiClusterNetwork Oracle
            if (GlobalConfig.HasMultiClusterNetwork)
            {
                logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
                            GlobalConfig.ServiceId, GlobalConfig.ClusterId);

                this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
                                                 this.providerManagerSystemTarget.SchedulingContext;
                scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
                .WaitWithThrow(initTimeout);
                if (logger.IsVerbose)
                {
                    logger.Verbose("multicluster oracle created successfully.");
                }
            }

            try
            {
                this.siloStatistics.Start(this.LocalConfig);
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Silo statistics manager started successfully.");
                }

                // Finally, initialize the deployment load collector, for grains with load-based placement
                var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
                this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
                .WaitWithThrow(this.initTimeout);
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Silo deployment load publisher started successfully.");
                }

                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants);
                this.platformWatchdog.Start();
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Silo platform watchdog started successfully.");
                }

                if (this.reminderService != null)
                {
                    // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                    this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
                                                  this.providerManagerSystemTarget.SchedulingContext;
                    this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
                    .WaitWithThrow(this.initTimeout);
                    if (this.logger.IsVerbose)
                    {
                        this.logger.Verbose("Reminder service started successfully.");
                    }
                }

                this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>();
                this.scheduler.QueueTask(
                    () => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
                    this.providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(this.initTimeout);
                this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics

                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("App bootstrap calls done successfully.");
                }

                // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
                // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
                this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(this.initTimeout);
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Stream providers started successfully.");
                }

                // Now that we're active, we can start the gateway
                var mc = this.messageCenter as MessageCenter;
                mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
                if (this.logger.IsVerbose)
                {
                    this.logger.Verbose("Message gateway service started successfully.");
                }

                this.SystemStatus = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
                this.FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
                throw;
            }
            if (logger.IsVerbose)
            {
                logger.Verbose("Silo.Start complete: System status = {0}", this.SystemStatus);
            }
        }
Beispiel #8
0
        private void DoStart()
        {
            lock (lockable)
            {
                if (!SystemStatus.Current.Equals(SystemStatus.Created))
                {
                    throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", SystemStatus.Current));
                }

                SystemStatus.Current = SystemStatus.Starting;
            }

            logger.Info(ErrorCode.SiloStarting, "Silo Start()");

            // Hook up to receive notification of process exit / Ctrl-C events
            AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
            Console.CancelKeyPress += HandleProcessExit;

            ConfigureThreadPoolAndServicePointSettings();

            // This has to start first so that the directory system target factory gets loaded before we start the router.
            typeManager.Start();
            InsideRuntimeClient.Current.Start();

            // The order of these 4 is pretty much arbitrary.
            scheduler.Start();
            messageCenter.Start();
            incomingPingAgent.Start();
            incomingSystemAgent.Start();
            incomingAgent.Start();

            LocalGrainDirectory.Start();

            // Set up an execution context for this thread so that the target creation steps can use asynch values.
            RuntimeContext.InitializeMainThread();

            SiloProviderRuntime.Initialize(GlobalConfig, SiloIdentity, grainFactory, Services);
            InsideRuntimeClient.Current.CurrentStreamProviderRuntime = SiloProviderRuntime.Instance;
            statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance);
            string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
                                       .WaitForResultWithThrow(initTimeout);

            if (statsProviderName != null)
            {
                LocalConfig.StatisticsProviderName = statsProviderName;
            }
            allSiloProviders.AddRange(statisticsProviderManager.GetProviders());

            // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
            siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
            siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);

            IMembershipTable membershipTable = membershipFactory.GetMembershipTable(GlobalConfig.LivenessType, GlobalConfig.MembershipTableAssembly);

            membershipOracle   = membershipFactory.CreateMembershipOracle(this, membershipTable);
            multiClusterOracle = multiClusterFactory.CreateGossipOracle(this).WaitForResultWithThrow(initTimeout);

            // This has to follow the above steps that start the runtime components
            CreateSystemTargets();

            InjectDependencies();

            // Validate the configuration.
            GlobalConfig.Application.ValidateConfiguration(logger);

            // ensure this runs in the grain context, wait for it to complete
            scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("System grains created successfully.");
            }

            // Initialize storage providers once we have a basic silo runtime environment operating
            storageProviderManager = new StorageProviderManager(grainFactory, Services);
            scheduler.QueueTask(
                () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            catalog.SetStorageManager(storageProviderManager);
            allSiloProviders.AddRange(storageProviderManager.GetProviders());
            if (logger.IsVerbose)
            {
                logger.Verbose("Storage provider manager created successfully.");
            }

            // Load and init stream providers before silo becomes active
            var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;

            scheduler.QueueTask(
                () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance),
                providerManagerSystemTarget.SchedulingContext)
            .WaitWithThrow(initTimeout);
            InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager;
            allSiloProviders.AddRange(siloStreamProviderManager.GetProviders());
            if (logger.IsVerbose)
            {
                logger.Verbose("Stream provider manager created successfully.");
            }

            ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext;

            bool waitForPrimaryToStart = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary;

            if (waitForPrimaryToStart) // only in MembershipTableGrain case.
            {
                scheduler.QueueTask(() => membershipFactory.WaitForTableToInit(membershipTable), statusOracleContext)
                .WaitWithThrow(initTimeout);
            }
            scheduler.QueueTask(() => membershipTable.InitializeMembershipTable(GlobalConfig, true, LogManager.GetLogger(membershipTable.GetType().Name)), statusOracleContext)
            .WaitWithThrow(initTimeout);

            scheduler.QueueTask(() => LocalSiloStatusOracle.Start(), statusOracleContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Local silo status oracle created successfully.");
            }
            scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext)
            .WaitWithThrow(initTimeout);
            if (logger.IsVerbose)
            {
                logger.Verbose("Local silo status oracle became active successfully.");
            }

            //if running in multi cluster scenario, start the MultiClusterNetwork Oracle
            if (GlobalConfig.HasMultiClusterNetwork)
            {
                logger.Info("Creating multicluster oracle with my ServiceId={0} and ClusterId={1}.",
                            GlobalConfig.ServiceId, GlobalConfig.ClusterId);

                ISchedulingContext clusterStatusContext = ((SystemTarget)multiClusterOracle).SchedulingContext;
                scheduler.QueueTask(() => multiClusterOracle.Start(LocalSiloStatusOracle), clusterStatusContext)
                .WaitWithThrow(initTimeout);
                if (logger.IsVerbose)
                {
                    logger.Verbose("multicluster oracle created successfully.");
                }
            }

            try
            {
                siloStatistics.Start(LocalConfig);
                if (logger.IsVerbose)
                {
                    logger.Verbose("Silo statistics manager started successfully.");
                }

                // Finally, initialize the deployment load collector, for grains with load-based placement
                scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext)
                .WaitWithThrow(initTimeout);
                if (logger.IsVerbose)
                {
                    logger.Verbose("Silo deployment load publisher started successfully.");
                }

                // Start background timer tick to watch for platform execution stalls, such as when GC kicks in
                platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants);
                platformWatchdog.Start();
                if (logger.IsVerbose)
                {
                    logger.Verbose("Silo platform watchdog started successfully.");
                }

                if (reminderService != null)
                {
                    // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
                    scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext)
                    .WaitWithThrow(initTimeout);
                    if (logger.IsVerbose)
                    {
                        logger.Verbose("Reminder service started successfully.");
                    }
                }

                bootstrapProviderManager = new BootstrapProviderManager();
                scheduler.QueueTask(
                    () => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations),
                    providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(initTimeout);
                BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
                allSiloProviders.AddRange(BootstrapProviders);

                if (logger.IsVerbose)
                {
                    logger.Verbose("App bootstrap calls done successfully.");
                }

                // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
                // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
                scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(initTimeout);
                if (logger.IsVerbose)
                {
                    logger.Verbose("Stream providers started successfully.");
                }

                // Now that we're active, we can start the gateway
                var mc = messageCenter as MessageCenter;
                if (mc != null)
                {
                    mc.StartGateway(clientRegistrar);
                }
                if (logger.IsVerbose)
                {
                    logger.Verbose("Message gateway service started successfully.");
                }

                SystemStatus.Current = SystemStatus.Running;
            }
            catch (Exception exc)
            {
                SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc));
                FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
                throw;
            }
            if (logger.IsVerbose)
            {
                logger.Verbose("Silo.Start complete: System status = {0}", SystemStatus.Current);
            }
        }