Example #1
0
        private async Task OnRuntimeServicesStop(CancellationToken cancellationToken)
        {
            bool   gracefully = !cancellationToken.IsCancellationRequested;
            string operation  = gracefully ? "Shutdown()" : "Stop()";

            try
            {
                if (gracefully)
                {
                    logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
                    // 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone
                    await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext)
                    .WithTimeout(stopTimeout);
                }
                else
                {
                    logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()");
                    // 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone
                    await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext)
                    .WithTimeout(stopTimeout);
                }
            }
            catch (Exception exc)
            {
                logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc);
                return; // will go to finally
            }

            if (reminderService != null)
            {
                // 2: Stop reminder service
                await scheduler.QueueTask(reminderService.Stop, this.reminderServiceContext)
                .WithTimeout(stopTimeout);
            }

            if (gracefully)
            {
                // 3: Deactivate all grains
                SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout));
            }

            // 3: Stop the gateway
            SafeExecute(messageCenter.StopAcceptingClientMessages);

            // 4: Start rejecting all silo to silo application messages
            SafeExecute(messageCenter.BlockApplicationMessages);

            // 5: Stop scheduling/executing application turns
            SafeExecute(scheduler.StopApplicationTurns);

            // 6: Directory: Speed up directory handoff
            // will be started automatically when directory receives SiloStatusChangeNotification(Stopping)

            // 7:
            SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout));

            // The order of closing providers might be importan: Stats, streams, boostrap, storage.
            // Stats first since no one depends on it.
            // Storage should definitely be last since other providers ma ybe using it, potentilay indirectly.
            // Streams and Bootstrap - the order is less clear. Seems like Bootstrap may indirecly depend on Streams, but not the other way around.
            // 8:
            SafeExecute(() =>
            {
                scheduler.QueueTask(() => statisticsProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(initTimeout);
            });
            // 9:
            SafeExecute(() =>
            {
                var siloStreamProviderManager = (StreamProviderManager)StreamProviderManager;
                scheduler.QueueTask(() => siloStreamProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(initTimeout);
            });
            // 10:
            SafeExecute(() =>
            {
                scheduler.QueueTask(() => bootstrapProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(initTimeout);
            });
            // 11:
            SafeExecute(() =>
            {
                scheduler.QueueTask(() => storageProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
                .WaitWithThrow(initTimeout);
            });
        }
Example #2
0
        /// <summary>
        /// Gracefully stop the run time system only, but not the application.
        /// Applications requests would be abruptly terminated, while the internal system state gracefully stopped and saved as much as possible.
        /// </summary>
        private void Terminate(bool gracefully)
        {
            string operation             = gracefully ? "Shutdown()" : "Stop()";
            bool   stopAlreadyInProgress = false;

            lock (lockable)
            {
                if (this.SystemStatus.Equals(SystemStatus.Stopping) ||
                    this.SystemStatus.Equals(SystemStatus.ShuttingDown) ||
                    this.SystemStatus.Equals(SystemStatus.Terminated))
                {
                    stopAlreadyInProgress = true;
                    // Drop through to wait below
                }
                else if (!this.SystemStatus.Equals(SystemStatus.Running))
                {
                    throw new InvalidOperationException(String.Format("Calling Silo.{0} on a silo which is not in the Running state. This silo is in the {1} state.", operation, this.SystemStatus));
                }
                else
                {
                    if (gracefully)
                    {
                        this.SystemStatus = SystemStatus.ShuttingDown;
                    }
                    else
                    {
                        this.SystemStatus = SystemStatus.Stopping;
                    }
                }
            }

            if (stopAlreadyInProgress)
            {
                logger.Info(ErrorCode.SiloStopInProgress, "Silo termination is in progress - Will wait for it to finish");
                var pause = TimeSpan.FromSeconds(1);
                while (!this.SystemStatus.Equals(SystemStatus.Terminated))
                {
                    logger.Info(ErrorCode.WaitingForSiloStop, "Waiting {0} for termination to complete", pause);
                    Thread.Sleep(pause);
                }
                return;
            }

            try
            {
                try
                {
                    if (gracefully)
                    {
                        logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
                        // 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone
                        scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext)
                        .WaitWithThrow(stopTimeout);
                    }
                    else
                    {
                        logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()");
                        // 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone
                        scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext)
                        .WaitWithThrow(stopTimeout);
                    }
                }
                catch (Exception exc)
                {
                    logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc);
                    return; // will go to finally
                }

                if (reminderService != null)
                {
                    // 2: Stop reminder service
                    scheduler.QueueTask(reminderService.Stop, this.reminderServiceContext)
                    .WaitWithThrow(stopTimeout);
                }

                if (gracefully)
                {
                    // 3: Deactivate all grains
                    SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout));
                }

                // 3: Stop the gateway
                SafeExecute(messageCenter.StopAcceptingClientMessages);

                // 4: Start rejecting all silo to silo application messages
                SafeExecute(messageCenter.BlockApplicationMessages);

                // 5: Stop scheduling/executing application turns
                SafeExecute(scheduler.StopApplicationTurns);

                // 6: Directory: Speed up directory handoff
                // will be started automatically when directory receives SiloStatusChangeNotification(Stopping)

                // 7:
                SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout));

                // The order of closing providers might be importan: Stats, streams, boostrap, storage.
                // Stats first since no one depends on it.
                // Storage should definitely be last since other providers ma ybe using it, potentilay indirectly.
                // Streams and Bootstrap - the order is less clear. Seems like Bootstrap may indirecly depend on Streams, but not the other way around.
                // 8:
                SafeExecute(() =>
                {
                    scheduler.QueueTask(() => statisticsProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
                    .WaitWithThrow(initTimeout);
                });
                // 9:
                SafeExecute(() =>
                {
                    var siloStreamProviderManager = (StreamProviderManager)StreamProviderManager;
                    scheduler.QueueTask(() => siloStreamProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
                    .WaitWithThrow(initTimeout);
                });
                // 10:
                SafeExecute(() =>
                {
                    scheduler.QueueTask(() => bootstrapProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
                    .WaitWithThrow(initTimeout);
                });
                // 11:
                SafeExecute(() =>
                {
                    scheduler.QueueTask(() => storageProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
                    .WaitWithThrow(initTimeout);
                });
            }
            finally
            {
                // 10, 11, 12: Write Dead in the table, Drain scheduler, Stop msg center, ...
                logger.Info(ErrorCode.SiloStopped, "Silo is Stopped()");
                FastKill();
            }
        }