private async Task OnBecomeActiveStop(CancellationToken ct)
{
if (this.isFastKilledNeeded)
{
return;
}
bool gracefully = !ct.IsCancellationRequested;
string operation = gracefully ? "Shutdown()" : "Stop()";
try
{
if (gracefully)
{
logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
//Stop LocalGrainDirectory
await scheduler.QueueTask(() => localGrainDirectory.Stop(true), localGrainDirectory.CacheValidator.SchedulingContext)
.WithCancellation(ct, "localGrainDirectory Stop failed because the task was cancelled");
SafeExecute(() => catalog.DeactivateAllActivations().Wait(ct));
//wait for all queued message sent to OutboundMessageQueue before MessageCenter stop and OutboundMessageQueue stop.
await Task.Delay(WaitForMessageToBeQueuedForOutbound);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.SiloFailedToStopMembership,
$"Failed to {operation}. About to FastKill this silo.", exc);
this.isFastKilledNeeded = true;
}
// Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
}
private async Task OnBecomeActiveStop(CancellationToken ct)
{
bool gracefully = !ct.IsCancellationRequested;
string operation = gracefully ? "Shutdown()" : "Stop()";
try
{
if (gracefully)
{
logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
// Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone
await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext)
.WithTimeout(stopTimeout, $"MembershipOracle Shutting down failed due to timeout {stopTimeout}");
// Deactivate all grains
SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout));
}
else
{
logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()");
// Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone
await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext)
.WithTimeout(stopTimeout, $"Stopping MembershipOracle faield due to timeout {stopTimeout}");
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc);
return; // will go to finally
}
// Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
}
private async Task OnBecomeActiveStop(CancellationToken ct)
{
if (this.isFastKilledNeeded)
{
return;
}
bool gracefully = !ct.IsCancellationRequested;
try
{
if (gracefully)
{
// Stop LocalGrainDirectory
await localGrainDirectory.CacheValidator.QueueActionAsync(() => localGrainDirectory.Stop());
SafeExecute(() => catalog.DeactivateAllActivations().Wait(ct));
// Wait for all queued message sent to OutboundMessageQueue before MessageCenter stop and OutboundMessageQueue stop.
await Task.Delay(waitForMessageToBeQueuedForOutbound);
}
}
catch (Exception exc)
{
logger.LogError(
(int)ErrorCode.SiloFailedToStopMembership,
exc,
"Failed to shutdown gracefully. About to terminate ungracefully");
this.isFastKilledNeeded = true;
}
// Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
}
private async Task OnBecomeActiveStop(CancellationToken ct)
{
if (this.isFastKilledNeeded)
{
return;
}
bool gracefully = !ct.IsCancellationRequested;
try
{
if (gracefully)
{
// Stop LocalGrainDirectory
var resolver = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
localGrainDirectory.CacheValidator.WorkItemGroup.QueueAction(() =>
{
try
{
localGrainDirectory.Stop();
resolver.TrySetResult(true);
}
catch (Exception exc)
{
resolver.TrySetException(exc);
}
});
await resolver.Task;
try
{
await catalog.DeactivateAllActivations().WithCancellation(ct);
}
catch (Exception exception)
{
logger.LogError(exception, "Error deactivating activations");
}
// Wait for all queued message sent to OutboundMessageQueue before MessageCenter stop and OutboundMessageQueue stop.
await Task.WhenAny(Task.Delay(waitForMessageToBeQueuedForOutbound), ct.WhenCancelled());
}
}
catch (Exception exc)
{
logger.LogError(
(int)ErrorCode.SiloFailedToStopMembership,
exc,
"Failed to shutdown gracefully. About to terminate ungracefully");
this.isFastKilledNeeded = true;
}
// Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
}
private async Task OnBecomeActiveStop(CancellationToken ct)
{
if (this.isFastKilledNeeded)
{
return;
}
bool gracefully = !ct.IsCancellationRequested;
string operation = gracefully ? "Shutdown()" : "Stop()";
try
{
if (gracefully)
{
logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
// Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone
//We do a 30 ms wait on the async silo status gossip in shutdown senario.
await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext)
.WithCancellation(ct, "MembershipOracle Shutting down failed because the task was cancelled");
//Stop LocalGrainDirectory
await scheduler.QueueTask(() => localGrainDirectory.Stop(true), localGrainDirectory.CacheValidator.SchedulingContext)
.WithCancellation(ct, "localGrainDirectory Stop failed because the task was cancelled");
SafeExecute(() => catalog.DeactivateAllActivations().Wait(ct));
//wait for all queued message sent to OutboundMessageQueue before MessageCenter stop and OutboundMessageQueue stop.
await Task.Delay(WaitForMessageToBeQueuedForOutbound);
}
else
{
logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()");
// TODO: we have to do this, even on non graceful shutdown, but the timeout should be smaller
// Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone
await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext)
.WithTimeout(stopTimeout, $"Stopping MembershipOracle faield due to timeout {stopTimeout}");
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.SiloFailedToStopMembership,
$"Failed to {operation} membership oracle. About to FastKill this silo.", exc);
this.isFastKilledNeeded = true;
}
// Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
}
private Task OnRuntimeServicesStop(CancellationToken cancellationToken)
{
if (!cancellationToken.IsCancellationRequested)
{
// Deactivate all grains
SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout));
}
// Start rejecting all silo to silo application messages
SafeExecute(messageCenter.BlockApplicationMessages);
// Stop scheduling/executing application turns
SafeExecute(scheduler.StopApplicationTurns);
// Directory: Speed up directory handoff
// will be started automatically when directory receives SiloStatusChangeNotification(Stopping)
SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout));
return(Task.CompletedTask);
}
private async Task OnBecomeActiveStop(CancellationToken ct)
{
if (this.isFastKilledNeeded)
{
return;
}
bool gracefully = !ct.IsCancellationRequested;
string operation = gracefully ? "Shutdown()" : "Stop()";
try
{
if (gracefully)
{
logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
// Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone
await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext)
.WithCancellation(ct, "MembershipOracle Shutting down failed because the task was cancelled");
// Deactivate all grains
SafeExecute(() => catalog.DeactivateAllActivations().Wait(ct));
}
else
{
logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()");
// TODO: we have to do this, even on non graceful shutdown, but the timeout should be smaller
// Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone
await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext)
.WithTimeout(stopTimeout, $"Stopping MembershipOracle faield due to timeout {stopTimeout}");
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.SiloFailedToStopMembership,
$"Failed to {operation} membership oracle. About to FastKill this silo.", exc);
this.isFastKilledNeeded = true;
}
// Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
}
private async Task OnRuntimeServicesStop(CancellationToken cancellationToken)
{
bool gracefully = !cancellationToken.IsCancellationRequested;
string operation = gracefully ? "Shutdown()" : "Stop()";
try
{
if (gracefully)
{
logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
// 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone
await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext)
.WithTimeout(stopTimeout);
}
else
{
logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()");
// 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone
await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext)
.WithTimeout(stopTimeout);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc);
return; // will go to finally
}
if (reminderService != null)
{
// 2: Stop reminder service
await scheduler.QueueTask(reminderService.Stop, this.reminderServiceContext)
.WithTimeout(stopTimeout);
}
if (gracefully)
{
// 3: Deactivate all grains
SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout));
}
// 3: Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
// 4: Start rejecting all silo to silo application messages
SafeExecute(messageCenter.BlockApplicationMessages);
// 5: Stop scheduling/executing application turns
SafeExecute(scheduler.StopApplicationTurns);
// 6: Directory: Speed up directory handoff
// will be started automatically when directory receives SiloStatusChangeNotification(Stopping)
// 7:
SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout));
// The order of closing providers might be importan: Stats, streams, boostrap, storage.
// Stats first since no one depends on it.
// Storage should definitely be last since other providers ma ybe using it, potentilay indirectly.
// Streams and Bootstrap - the order is less clear. Seems like Bootstrap may indirecly depend on Streams, but not the other way around.
// 8:
SafeExecute(() =>
{
scheduler.QueueTask(() => statisticsProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 9:
SafeExecute(() =>
{
var siloStreamProviderManager = (StreamProviderManager)StreamProviderManager;
scheduler.QueueTask(() => siloStreamProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 10:
SafeExecute(() =>
{
scheduler.QueueTask(() => bootstrapProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 11:
SafeExecute(() =>
{
scheduler.QueueTask(() => storageProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
}
/// <summary>
/// Gracefully stop the run time system only, but not the application.
/// Applications requests would be abruptly terminated, while the internal system state gracefully stopped and saved as much as possible.
/// </summary>
private void Terminate(bool gracefully)
{
string operation = gracefully ? "Shutdown()" : "Stop()";
bool stopAlreadyInProgress = false;
lock (lockable)
{
if (SystemStatus.Current.Equals(SystemStatus.Stopping) ||
SystemStatus.Current.Equals(SystemStatus.ShuttingDown) ||
SystemStatus.Current.Equals(SystemStatus.Terminated))
{
stopAlreadyInProgress = true;
// Drop through to wait below
}
else if (!SystemStatus.Current.Equals(SystemStatus.Running))
{
throw new InvalidOperationException(String.Format("Calling Silo.{0} on a silo which is not in the Running state. This silo is in the {1} state.", operation, SystemStatus.Current));
}
else
{
if (gracefully)
{
SystemStatus.Current = SystemStatus.ShuttingDown;
}
else
{
SystemStatus.Current = SystemStatus.Stopping;
}
}
}
if (stopAlreadyInProgress)
{
logger.Info(ErrorCode.SiloStopInProgress, "Silo termination is in progress - Will wait for it to finish");
var pause = TimeSpan.FromSeconds(1);
while (!SystemStatus.Current.Equals(SystemStatus.Terminated))
{
logger.Info(ErrorCode.WaitingForSiloStop, "Waiting {0} for termination to complete", pause);
Thread.Sleep(pause);
}
return;
}
try
{
try
{
if (gracefully)
{
logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
// 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone
scheduler.QueueTask(LocalSiloStatusOracle.ShutDown, ((SystemTarget)LocalSiloStatusOracle).SchedulingContext)
.WaitWithThrow(stopTimeout);
}
else
{
logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()");
// 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone
scheduler.QueueTask(LocalSiloStatusOracle.Stop, ((SystemTarget)LocalSiloStatusOracle).SchedulingContext)
.WaitWithThrow(stopTimeout);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} LocalSiloStatusOracle. About to FastKill this silo.", operation), exc);
return; // will go to finally
}
if (reminderService != null)
{
// 2: Stop reminder service
scheduler.QueueTask(reminderService.Stop, ((SystemTarget)reminderService).SchedulingContext)
.WaitWithThrow(stopTimeout);
}
if (gracefully)
{
// 3: Deactivate all grains
SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout));
}
// 3: Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
// 4: Start rejecting all silo to silo application messages
SafeExecute(messageCenter.BlockApplicationMessages);
// 5: Stop scheduling/executing application turns
SafeExecute(scheduler.StopApplicationTurns);
// 6: Directory: Speed up directory handoff
// will be started automatically when directory receives SiloStatusChangeNotification(Stopping)
// 7:
SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout));
// 8:
SafeExecute(() =>
{
var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;
scheduler.QueueTask(() => siloStreamProviderManager.StopStreamProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 9:
SafeExecute(storageProviderManager.UnloadStorageProviders);
}
finally
{
// 10, 11, 12: Write Dead in the table, Drain scheduler, Stop msg center, ...
FastKill();
logger.Info(ErrorCode.SiloStopped, "Silo is Stopped()");
}
}
/// <summary>
/// Gracefully stop the run time system only, but not the application.
/// Applications requests would be abruptly terminated, while the internal system state gracefully stopped and saved as much as possible.
/// </summary>
private void Terminate(bool gracefully)
{
string operation = gracefully ? "Shutdown()" : "Stop()";
bool stopAlreadyInProgress = false;
lock (lockable)
{
if (this.SystemStatus.Equals(SystemStatus.Stopping) ||
this.SystemStatus.Equals(SystemStatus.ShuttingDown) ||
this.SystemStatus.Equals(SystemStatus.Terminated))
{
stopAlreadyInProgress = true;
// Drop through to wait below
}
else if (!this.SystemStatus.Equals(SystemStatus.Running))
{
throw new InvalidOperationException(String.Format("Calling Silo.{0} on a silo which is not in the Running state. This silo is in the {1} state.", operation, this.SystemStatus));
}
else
{
if (gracefully)
{
this.SystemStatus = SystemStatus.ShuttingDown;
}
else
{
this.SystemStatus = SystemStatus.Stopping;
}
}
}
if (stopAlreadyInProgress)
{
logger.Info(ErrorCode.SiloStopInProgress, "Silo termination is in progress - Will wait for it to finish");
var pause = TimeSpan.FromSeconds(1);
while (!this.SystemStatus.Equals(SystemStatus.Terminated))
{
logger.Info(ErrorCode.WaitingForSiloStop, "Waiting {0} for termination to complete", pause);
Thread.Sleep(pause);
}
return;
}
try
{
try
{
if (gracefully)
{
logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
// 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone
scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext)
.WaitWithThrow(stopTimeout);
}
else
{
logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()");
// 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone
scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext)
.WaitWithThrow(stopTimeout);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc);
return; // will go to finally
}
if (reminderService != null)
{
// 2: Stop reminder service
scheduler.QueueTask(reminderService.Stop, this.reminderServiceContext)
.WaitWithThrow(stopTimeout);
}
if (gracefully)
{
// 3: Deactivate all grains
SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout));
}
// 3: Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
// 4: Start rejecting all silo to silo application messages
SafeExecute(messageCenter.BlockApplicationMessages);
// 5: Stop scheduling/executing application turns
SafeExecute(scheduler.StopApplicationTurns);
// 6: Directory: Speed up directory handoff
// will be started automatically when directory receives SiloStatusChangeNotification(Stopping)
// 7:
SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout));
// The order of closing providers might be importan: Stats, streams, boostrap, storage.
// Stats first since no one depends on it.
// Storage should definitely be last since other providers ma ybe using it, potentilay indirectly.
// Streams and Bootstrap - the order is less clear. Seems like Bootstrap may indirecly depend on Streams, but not the other way around.
// 8:
SafeExecute(() =>
{
scheduler.QueueTask(() => statisticsProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 9:
SafeExecute(() =>
{
var siloStreamProviderManager = (StreamProviderManager)StreamProviderManager;
scheduler.QueueTask(() => siloStreamProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 10:
SafeExecute(() =>
{
scheduler.QueueTask(() => bootstrapProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 11:
SafeExecute(() =>
{
scheduler.QueueTask(() => storageProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
}
finally
{
// 10, 11, 12: Write Dead in the table, Drain scheduler, Stop msg center, ...
logger.Info(ErrorCode.SiloStopped, "Silo is Stopped()");
FastKill();
}
}