private async Task OnRuntimeServicesStop(CancellationToken cancellationToken)
{
bool gracefully = !cancellationToken.IsCancellationRequested;
string operation = gracefully ? "Shutdown()" : "Stop()";
try
{
if (gracefully)
{
logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
// 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone
await scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext)
.WithTimeout(stopTimeout);
}
else
{
logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()");
// 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone
await scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext)
.WithTimeout(stopTimeout);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc);
return; // will go to finally
}
if (reminderService != null)
{
// 2: Stop reminder service
await scheduler.QueueTask(reminderService.Stop, this.reminderServiceContext)
.WithTimeout(stopTimeout);
}
if (gracefully)
{
// 3: Deactivate all grains
SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout));
}
// 3: Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
// 4: Start rejecting all silo to silo application messages
SafeExecute(messageCenter.BlockApplicationMessages);
// 5: Stop scheduling/executing application turns
SafeExecute(scheduler.StopApplicationTurns);
// 6: Directory: Speed up directory handoff
// will be started automatically when directory receives SiloStatusChangeNotification(Stopping)
// 7:
SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout));
// The order of closing providers might be importan: Stats, streams, boostrap, storage.
// Stats first since no one depends on it.
// Storage should definitely be last since other providers ma ybe using it, potentilay indirectly.
// Streams and Bootstrap - the order is less clear. Seems like Bootstrap may indirecly depend on Streams, but not the other way around.
// 8:
SafeExecute(() =>
{
scheduler.QueueTask(() => statisticsProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 9:
SafeExecute(() =>
{
var siloStreamProviderManager = (StreamProviderManager)StreamProviderManager;
scheduler.QueueTask(() => siloStreamProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 10:
SafeExecute(() =>
{
scheduler.QueueTask(() => bootstrapProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 11:
SafeExecute(() =>
{
scheduler.QueueTask(() => storageProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
}
/// <summary>
/// Gracefully stop the run time system only, but not the application.
/// Applications requests would be abruptly terminated, while the internal system state gracefully stopped and saved as much as possible.
/// </summary>
private void Terminate(bool gracefully)
{
string operation = gracefully ? "Shutdown()" : "Stop()";
bool stopAlreadyInProgress = false;
lock (lockable)
{
if (this.SystemStatus.Equals(SystemStatus.Stopping) ||
this.SystemStatus.Equals(SystemStatus.ShuttingDown) ||
this.SystemStatus.Equals(SystemStatus.Terminated))
{
stopAlreadyInProgress = true;
// Drop through to wait below
}
else if (!this.SystemStatus.Equals(SystemStatus.Running))
{
throw new InvalidOperationException(String.Format("Calling Silo.{0} on a silo which is not in the Running state. This silo is in the {1} state.", operation, this.SystemStatus));
}
else
{
if (gracefully)
{
this.SystemStatus = SystemStatus.ShuttingDown;
}
else
{
this.SystemStatus = SystemStatus.Stopping;
}
}
}
if (stopAlreadyInProgress)
{
logger.Info(ErrorCode.SiloStopInProgress, "Silo termination is in progress - Will wait for it to finish");
var pause = TimeSpan.FromSeconds(1);
while (!this.SystemStatus.Equals(SystemStatus.Terminated))
{
logger.Info(ErrorCode.WaitingForSiloStop, "Waiting {0} for termination to complete", pause);
Thread.Sleep(pause);
}
return;
}
try
{
try
{
if (gracefully)
{
logger.Info(ErrorCode.SiloShuttingDown, "Silo starting to Shutdown()");
// 1: Write "ShutDown" state in the table + broadcast gossip msgs to re-read the table to everyone
scheduler.QueueTask(this.membershipOracle.ShutDown, this.membershipOracleContext)
.WaitWithThrow(stopTimeout);
}
else
{
logger.Info(ErrorCode.SiloStopping, "Silo starting to Stop()");
// 1: Write "Stopping" state in the table + broadcast gossip msgs to re-read the table to everyone
scheduler.QueueTask(this.membershipOracle.Stop, this.membershipOracleContext)
.WaitWithThrow(stopTimeout);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.SiloFailedToStopMembership, String.Format("Failed to {0} membership oracle. About to FastKill this silo.", operation), exc);
return; // will go to finally
}
if (reminderService != null)
{
// 2: Stop reminder service
scheduler.QueueTask(reminderService.Stop, this.reminderServiceContext)
.WaitWithThrow(stopTimeout);
}
if (gracefully)
{
// 3: Deactivate all grains
SafeExecute(() => catalog.DeactivateAllActivations().WaitWithThrow(stopTimeout));
}
// 3: Stop the gateway
SafeExecute(messageCenter.StopAcceptingClientMessages);
// 4: Start rejecting all silo to silo application messages
SafeExecute(messageCenter.BlockApplicationMessages);
// 5: Stop scheduling/executing application turns
SafeExecute(scheduler.StopApplicationTurns);
// 6: Directory: Speed up directory handoff
// will be started automatically when directory receives SiloStatusChangeNotification(Stopping)
// 7:
SafeExecute(() => LocalGrainDirectory.StopPreparationCompletion.WaitWithThrow(stopTimeout));
// The order of closing providers might be importan: Stats, streams, boostrap, storage.
// Stats first since no one depends on it.
// Storage should definitely be last since other providers ma ybe using it, potentilay indirectly.
// Streams and Bootstrap - the order is less clear. Seems like Bootstrap may indirecly depend on Streams, but not the other way around.
// 8:
SafeExecute(() =>
{
scheduler.QueueTask(() => statisticsProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 9:
SafeExecute(() =>
{
var siloStreamProviderManager = (StreamProviderManager)StreamProviderManager;
scheduler.QueueTask(() => siloStreamProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 10:
SafeExecute(() =>
{
scheduler.QueueTask(() => bootstrapProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
// 11:
SafeExecute(() =>
{
scheduler.QueueTask(() => storageProviderManager.CloseProviders(), providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
});
}
finally
{
// 10, 11, 12: Write Dead in the table, Drain scheduler, Stop msg center, ...
logger.Info(ErrorCode.SiloStopped, "Silo is Stopped()");
FastKill();
}
}