private async Task OnBecomeActiveStop(CancellationToken ct) { if (this.isFastKilledNeeded) { return; } bool gracefully = !ct.IsCancellationRequested; string operation = gracefully ? "Shutdown()" : "Stop()"; try { if (gracefully) { logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()"); //Stop LocalGrainDirectory await scheduler.QueueTask(() => localGrainDirectory.Stop(true), localGrainDirectory.CacheValidator.SchedulingContext) .WithCancellation(ct, "localGrainDirectory Stop failed because the task was cancelled"); SafeExecute(() => catalog.DeactivateAllActivations().Wait(ct)); //wait for all queued message sent to OutboundMessageQueue before MessageCenter stop and OutboundMessageQueue stop. await Task.Delay(WaitForMessageToBeQueuedForOutbound); } } catch (Exception exc) { logger.Error(ErrorCode.SiloFailedToStopMembership, $"Failed to {operation}. About to FastKill this silo.", exc); this.isFastKilledNeeded = true; } // Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); }
private async Task OnBecomeActiveStop(CancellationToken ct) { bool gracefully = !ct.IsCancellationRequested; string operation = gracefully ? "Shutdown()" : "Stop()"; try { if (gracefully) { logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()"); // Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext) .WithTimeout(stopTimeout, $"MembershipOracle Shutting down failed due to timeout {stopTimeout}"); // Deactivate all grains SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout)); } else { logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()"); // Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext) .WithTimeout(stopTimeout, $"Stopping MembershipOracle faield due to timeout {stopTimeout}"); } } catch (Exception exc) { logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc); return; // will go to finally } // Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); }
private async Task OnBecomeActiveStop(CancellationToken ct) { if (this.isFastKilledNeeded) { return; } bool gracefully = !ct.IsCancellationRequested; try { if (gracefully) { // Stop LocalGrainDirectory await localGrainDirectory.CacheValidator.QueueActionAsync(() => localGrainDirectory.Stop()); SafeExecute(() => catalog.DeactivateAllActivations().Wait(ct)); // Wait for all queued message sent to OutboundMessageQueue before MessageCenter stop and OutboundMessageQueue stop. await Task.Delay(waitForMessageToBeQueuedForOutbound); } } catch (Exception exc) { logger.LogError( (int)ErrorCode.SiloFailedToStopMembership, exc, "Failed to shutdown gracefully. About to terminate ungracefully"); this.isFastKilledNeeded = true; } // Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); }
private async Task OnBecomeActiveStop(CancellationToken ct) { if (this.isFastKilledNeeded) { return; } bool gracefully = !ct.IsCancellationRequested; try { if (gracefully) { // Stop LocalGrainDirectory var resolver = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); localGrainDirectory.CacheValidator.WorkItemGroup.QueueAction(() => { try { localGrainDirectory.Stop(); resolver.TrySetResult(true); } catch (Exception exc) { resolver.TrySetException(exc); } }); await resolver.Task; try { await catalog.DeactivateAllActivations().WithCancellation(ct); } catch (Exception exception) { logger.LogError(exception, "Error deactivating activations"); } // Wait for all queued message sent to OutboundMessageQueue before MessageCenter stop and OutboundMessageQueue stop. await Task.WhenAny(Task.Delay(waitForMessageToBeQueuedForOutbound), ct.WhenCancelled()); } } catch (Exception exc) { logger.LogError( (int)ErrorCode.SiloFailedToStopMembership, exc, "Failed to shutdown gracefully. About to terminate ungracefully"); this.isFastKilledNeeded = true; } // Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); }
private async Task OnBecomeActiveStop(CancellationToken ct) { if (this.isFastKilledNeeded) { return; } bool gracefully = !ct.IsCancellationRequested; string operation = gracefully ? "Shutdown()" : "Stop()"; try { if (gracefully) { logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()"); // Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone //We do a 30 ms wait on the async silo status gossip in shutdown senario. await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext) .WithCancellation(ct, "MembershipOracle Shutting down failed because the task was cancelled"); //Stop LocalGrainDirectory await scheduler.QueueTask(() => localGrainDirectory.Stop(true), localGrainDirectory.CacheValidator.SchedulingContext) .WithCancellation(ct, "localGrainDirectory Stop failed because the task was cancelled"); SafeExecute(() => catalog.DeactivateAllActivations().Wait(ct)); //wait for all queued message sent to OutboundMessageQueue before MessageCenter stop and OutboundMessageQueue stop. await Task.Delay(WaitForMessageToBeQueuedForOutbound); } else { logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()"); // TODO: we have to do this, even on non graceful shutdown, but the timeout should be smaller // Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext) .WithTimeout(stopTimeout, $"Stopping MembershipOracle faield due to timeout {stopTimeout}"); } } catch (Exception exc) { logger.Error(ErrorCode.SiloFailedToStopMembership, $"Failed to {operation} membership oracle. About to FastKill this silo.", exc); this.isFastKilledNeeded = true; } // Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); }
private Task OnRuntimeServicesStop(CancellationToken cancellationToken) { if (!cancellationToken.IsCancellationRequested) { // Deactivate all grains SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout)); } // Start rejecting all silo to silo application messages SafeExecute(messageCenter.BlockApplicationMessages); // Stop scheduling/executing application turns SafeExecute(scheduler.StopApplicationTurns); // Directory: Speed up directory handoff // will be started automatically when directory receives SiloStatusChangeNotification(Stopping) SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout)); return(Task.CompletedTask); }
private async Task OnBecomeActiveStop(CancellationToken ct) { if (this.isFastKilledNeeded) { return; } bool gracefully = !ct.IsCancellationRequested; string operation = gracefully ? "Shutdown()" : "Stop()"; try { if (gracefully) { logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()"); // Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext) .WithCancellation(ct, "MembershipOracle Shutting down failed because the task was cancelled"); // Deactivate all grains SafeExecute(() => catalog.DeactivateAllActivations().Wait(ct)); } else { logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()"); // TODO: we have to do this, even on non graceful shutdown, but the timeout should be smaller // Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext) .WithTimeout(stopTimeout, $"Stopping MembershipOracle faield due to timeout {stopTimeout}"); } } catch (Exception exc) { logger.Error(ErrorCode.SiloFailedToStopMembership, $"Failed to {operation} membership oracle. About to FastKill this silo.", exc); this.isFastKilledNeeded = true; } // Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); }
private async Task OnRuntimeServicesStop(CancellationToken cancellationToken) { bool gracefully = !cancellationToken.IsCancellationRequested; string operation = gracefully ? "Shutdown()" : "Stop()"; try { if (gracefully) { logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()"); // 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext) .WithTimeout(stopTimeout); } else { logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()"); // 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext) .WithTimeout(stopTimeout); } } catch (Exception exc) { logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc); return; // will go to finally } if (reminderService != null) { // 2: Stop reminder service await scheduler.QueueTask(reminderService.Stop, this.reminderServiceContext) .WithTimeout(stopTimeout); } if (gracefully) { // 3: Deactivate all grains SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout)); } // 3: Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); // 4: Start rejecting all silo to silo application messages SafeExecute(messageCenter.BlockApplicationMessages); // 5: Stop scheduling/executing application turns SafeExecute(scheduler.StopApplicationTurns); // 6: Directory: Speed up directory handoff // will be started automatically when directory receives SiloStatusChangeNotification(Stopping) // 7: SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout)); // The order of closing providers might be importan: Stats, streams, boostrap, storage. // Stats first since no one depends on it. // Storage should definitely be last since other providers ma ybe using it, potentilay indirectly. // Streams and Bootstrap - the order is less clear. Seems like Bootstrap may indirecly depend on Streams, but not the other way around. // 8: SafeExecute(() => { scheduler.QueueTask(() => statisticsProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 9: SafeExecute(() => { var siloStreamProviderManager = (StreamProviderManager)StreamProviderManager; scheduler.QueueTask(() => siloStreamProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 10: SafeExecute(() => { scheduler.QueueTask(() => bootstrapProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 11: SafeExecute(() => { scheduler.QueueTask(() => storageProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); }
/// <summary> /// Gracefully stop the run time system only, but not the application. /// Applications requests would be abruptly terminated, while the internal system state gracefully stopped and saved as much as possible. /// </summary> private void Terminate(bool gracefully) { string operation = gracefully ? "Shutdown()" : "Stop()"; bool stopAlreadyInProgress = false; lock (lockable) { if (SystemStatus.Current.Equals(SystemStatus.Stopping) || SystemStatus.Current.Equals(SystemStatus.ShuttingDown) || SystemStatus.Current.Equals(SystemStatus.Terminated)) { stopAlreadyInProgress = true; // Drop through to wait below } else if (!SystemStatus.Current.Equals(SystemStatus.Running)) { throw new InvalidOperationException(String.Format("Calling Silo.{0} on a silo which is not in the Running state. This silo is in the {1} state.", operation, SystemStatus.Current)); } else { if (gracefully) { SystemStatus.Current = SystemStatus.ShuttingDown; } else { SystemStatus.Current = SystemStatus.Stopping; } } } if (stopAlreadyInProgress) { logger.Info(ErrorCode.SiloStopInProgress, "Silo termination is in progress - Will wait for it to finish"); var pause = TimeSpan.FromSeconds(1); while (!SystemStatus.Current.Equals(SystemStatus.Terminated)) { logger.Info(ErrorCode.WaitingForSiloStop, "Waiting {0} for termination to complete", pause); Thread.Sleep(pause); } return; } try { try { if (gracefully) { logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()"); // 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone scheduler.QueueTask(LocalSiloStatusOracle.ShutDown, ((SystemTarget)LocalSiloStatusOracle).SchedulingContext) .WaitWithThrow(stopTimeout); } else { logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()"); // 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone scheduler.QueueTask(LocalSiloStatusOracle.Stop, ((SystemTarget)LocalSiloStatusOracle).SchedulingContext) .WaitWithThrow(stopTimeout); } } catch (Exception exc) { logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} LocalSiloStatusOracle. About to FastKill this silo.", operation), exc); return; // will go to finally } if (reminderService != null) { // 2: Stop reminder service scheduler.QueueTask(reminderService.Stop, ((SystemTarget)reminderService).SchedulingContext) .WaitWithThrow(stopTimeout); } if (gracefully) { // 3: Deactivate all grains SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout)); } // 3: Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); // 4: Start rejecting all silo to silo application messages SafeExecute(messageCenter.BlockApplicationMessages); // 5: Stop scheduling/executing application turns SafeExecute(scheduler.StopApplicationTurns); // 6: Directory: Speed up directory handoff // will be started automatically when directory receives SiloStatusChangeNotification(Stopping) // 7: SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout)); // 8: SafeExecute(() => { var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager; scheduler.QueueTask(() => siloStreamProviderManager.StopStreamProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 9: SafeExecute(storageProviderManager.UnloadStorageProviders); } finally { // 10, 11, 12: Write Dead in the table, Drain scheduler, Stop msg center, ... FastKill(); logger.Info(ErrorCode.SiloStopped, "Silo is Stopped()"); } }
/// <summary> /// Gracefully stop the run time system only, but not the application. /// Applications requests would be abruptly terminated, while the internal system state gracefully stopped and saved as much as possible. /// </summary> private void Terminate(bool gracefully) { string operation = gracefully ? "Shutdown()" : "Stop()"; bool stopAlreadyInProgress = false; lock (lockable) { if (this.SystemStatus.Equals(SystemStatus.Stopping) || this.SystemStatus.Equals(SystemStatus.ShuttingDown) || this.SystemStatus.Equals(SystemStatus.Terminated)) { stopAlreadyInProgress = true; // Drop through to wait below } else if (!this.SystemStatus.Equals(SystemStatus.Running)) { throw new InvalidOperationException(String.Format("Calling Silo.{0} on a silo which is not in the Running state. This silo is in the {1} state.", operation, this.SystemStatus)); } else { if (gracefully) { this.SystemStatus = SystemStatus.ShuttingDown; } else { this.SystemStatus = SystemStatus.Stopping; } } } if (stopAlreadyInProgress) { logger.Info(ErrorCode.SiloStopInProgress, "Silo termination is in progress - Will wait for it to finish"); var pause = TimeSpan.FromSeconds(1); while (!this.SystemStatus.Equals(SystemStatus.Terminated)) { logger.Info(ErrorCode.WaitingForSiloStop, "Waiting {0} for termination to complete", pause); Thread.Sleep(pause); } return; } try { try { if (gracefully) { logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()"); // 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext) .WaitWithThrow(stopTimeout); } else { logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()"); // 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext) .WaitWithThrow(stopTimeout); } } catch (Exception exc) { logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc); return; // will go to finally } if (reminderService != null) { // 2: Stop reminder service scheduler.QueueTask(reminderService.Stop, this.reminderServiceContext) .WaitWithThrow(stopTimeout); } if (gracefully) { // 3: Deactivate all grains SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout)); } // 3: Stop the gateway SafeExecute(messageCenter.StopAcceptingClientMessages); // 4: Start rejecting all silo to silo application messages SafeExecute(messageCenter.BlockApplicationMessages); // 5: Stop scheduling/executing application turns SafeExecute(scheduler.StopApplicationTurns); // 6: Directory: Speed up directory handoff // will be started automatically when directory receives SiloStatusChangeNotification(Stopping) // 7: SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout)); // The order of closing providers might be importan: Stats, streams, boostrap, storage. // Stats first since no one depends on it. // Storage should definitely be last since other providers ma ybe using it, potentilay indirectly. // Streams and Bootstrap - the order is less clear. Seems like Bootstrap may indirecly depend on Streams, but not the other way around. // 8: SafeExecute(() => { scheduler.QueueTask(() => statisticsProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 9: SafeExecute(() => { var siloStreamProviderManager = (StreamProviderManager)StreamProviderManager; scheduler.QueueTask(() => siloStreamProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 10: SafeExecute(() => { scheduler.QueueTask(() => bootstrapProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); // 11: SafeExecute(() => { scheduler.QueueTask(() => storageProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); }); } finally { // 10, 11, 12: Write Dead in the table, Drain scheduler, Stop msg center, ... logger.Info(ErrorCode.SiloStopped, "Silo is Stopped()"); FastKill(); } }