// TODO move this logic in the LocalGrainDirectory
private void OnSiloStatusChange(SiloAddress updatedSilo, SiloStatus status)
{
// ignore joining events and also events on myself.
if (updatedSilo.Equals(LocalSilo))
{
return;
}
// We deactivate those activations when silo goes either of ShuttingDown/Stopping/Dead states,
// since this is what Directory is doing as well. Directory removes a silo based on all those 3 statuses,
// thus it will only deliver a "remove" notification for a given silo once to us. Therefore, we need to react the fist time we are notified.
// We may review the directory behavior in the future and treat ShuttingDown differently ("drain only") and then this code will have to change a well.
if (!status.IsTerminating())
{
return;
}
if (status == SiloStatus.Dead)
{
this.RuntimeClient.BreakOutstandingMessagesToDeadSilo(updatedSilo);
}
var activationsToShutdown = new List <IGrainContext>();
try
{
// scan all activations in activation directory and deactivate the ones that the removed silo is their primary partition owner.
lock (activations)
{
foreach (var activation in activations)
{
try
{
var activationData = activation.Value;
if (!activationData.PlacementStrategy.IsUsingGrainDirectory || grainDirectoryResolver.HasNonDefaultDirectory(activationData.GrainId.Type))
{
continue;
}
if (!updatedSilo.Equals(directory.GetPrimaryForGrain(activationData.GrainId)))
{
continue;
}
activationsToShutdown.Add(activationData);
}
catch (Exception exc)
{
logger.LogError(
(int)ErrorCode.Catalog_SiloStatusChangeNotification_Exception,
exc,
"Catalog has thrown an exception while handling removal of silo {Silo}", updatedSilo.ToStringWithHashCode());
}
}
}
logger.LogInformation(
(int)ErrorCode.Catalog_SiloStatusChangeNotification,
"Catalog is deactivating {Count} activations due to a failure of silo {Silo}, since it is a primary directory partition to these grain ids.",
activationsToShutdown.Count,
updatedSilo.ToStringWithHashCode());
}
finally
{
// outside the lock.
if (activationsToShutdown.Count > 0)
{
DeactivateActivations(activationsToShutdown).Ignore();
}
}
}