private async Task OnRuntimeServicesStop(CancellationToken cancellationToken) { bool gracefully = !cancellationToken.IsCancellationRequested; string operation = gracefully ? "Shutdown()" : "Stop()"; try { if (gracefully) { logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()"); // 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext) .WithTimeout(stopTimeout); } else { logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()"); // 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext) .WithTimeout(stopTimeout); } } catch (Exception exc) { logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc); return; // will go to finally } if (reminderService != null) { // 2: Stop reminder service await scheduler.QueueTask(reminderService.Stop, this.reminderServiceContext) .WithTimeout(stopTimeout); } if (gracefully) { // 3: Deactivate all grains SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout)); } // 3: Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); // 4: Start rejecting all silo to silo application messages SafeExecute(messageCenter.BlockApplicationMessages); // 5: Stop scheduling/executing application turns SafeExecute(scheduler.StopApplicationTurns); // 6: Directory: Speed up directory handoff // will be started automatically when directory receives SiloStatusChangeNotification(Stopping) // 7: SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout)); // The order of closing providers might be importan: Stats, streams, boostrap, storage. // Stats first since no one depends on it. // Storage should definitely be last since other providers ma ybe using it, potentilay indirectly. // Streams and Bootstrap - the order is less clear. Seems like Bootstrap may indirecly depend on Streams, but not the other way around. // 8: SafeExecute(() => { scheduler.QueueTask(() => statisticsProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 9: SafeExecute(() => { var siloStreamProviderManager = (StreamProviderManager)StreamProviderManager; scheduler.QueueTask(() => siloStreamProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 10: SafeExecute(() => { scheduler.QueueTask(() => bootstrapProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 11: SafeExecute(() => { scheduler.QueueTask(() => storageProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); }
/// <summary> /// Gracefully stop the run time system only, but not the application. /// Applications requests would be abruptly terminated, while the internal system state gracefully stopped and saved as much as possible. /// </summary> private void Terminate(bool gracefully) { string operation = gracefully ? "Shutdown()" : "Stop()"; bool stopAlreadyInProgress = false; lock (lockable) { if (this.SystemStatus.Equals(SystemStatus.Stopping) || this.SystemStatus.Equals(SystemStatus.ShuttingDown) || this.SystemStatus.Equals(SystemStatus.Terminated)) { stopAlreadyInProgress = true; // Drop through to wait below } else if (!this.SystemStatus.Equals(SystemStatus.Running)) { throw new InvalidOperationException(String.Format("Calling Silo.{0} on a silo which is not in the Running state. This silo is in the {1} state.", operation, this.SystemStatus)); } else { if (gracefully) { this.SystemStatus = SystemStatus.ShuttingDown; } else { this.SystemStatus = SystemStatus.Stopping; } } } if (stopAlreadyInProgress) { logger.Info(ErrorCode.SiloStopInProgress, "Silo termination is in progress - Will wait for it to finish"); var pause = TimeSpan.FromSeconds(1); while (!this.SystemStatus.Equals(SystemStatus.Terminated)) { logger.Info(ErrorCode.WaitingForSiloStop, "Waiting {0} for termination to complete", pause); Thread.Sleep(pause); } return; } try { try { if (gracefully) { logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()"); // 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext) .WaitWithThrow(stopTimeout); } else { logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()"); // 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext) .WaitWithThrow(stopTimeout); } } catch (Exception exc) { logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc); return; // will go to finally } if (reminderService != null) { // 2: Stop reminder service scheduler.QueueTask(reminderService.Stop, this.reminderServiceContext) .WaitWithThrow(stopTimeout); } if (gracefully) { // 3: Deactivate all grains SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout)); } // 3: Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); // 4: Start rejecting all silo to silo application messages SafeExecute(messageCenter.BlockApplicationMessages); // 5: Stop scheduling/executing application turns SafeExecute(scheduler.StopApplicationTurns); // 6: Directory: Speed up directory handoff // will be started automatically when directory receives SiloStatusChangeNotification(Stopping) // 7: SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout)); // The order of closing providers might be importan: Stats, streams, boostrap, storage. // Stats first since no one depends on it. // Storage should definitely be last since other providers ma ybe using it, potentilay indirectly. // Streams and Bootstrap - the order is less clear. Seems like Bootstrap may indirecly depend on Streams, but not the other way around. // 8: SafeExecute(() => { scheduler.QueueTask(() => statisticsProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 9: SafeExecute(() => { var siloStreamProviderManager = (StreamProviderManager)StreamProviderManager; scheduler.QueueTask(() => siloStreamProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 10: SafeExecute(() => { scheduler.QueueTask(() => bootstrapProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 11: SafeExecute(() => { scheduler.QueueTask(() => storageProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); } finally { // 10, 11, 12: Write Dead in the table, Drain scheduler, Stop msg center, ... logger.Info(ErrorCode.SiloStopped, "Silo is Stopped()"); FastKill(); } }