private async Task MonitorOrleansClustering()
{
var previous = _clusterMembershipService.CurrentSnapshot;
while (!_shutdownToken.IsCancellationRequested)
{
try
{
await foreach (var update in _clusterMembershipService.MembershipUpdates.WithCancellation(_shutdownToken.Token))
{
// Determine which silos should be monitoring Kubernetes
var chosenSilos = _clusterMembershipService.CurrentSnapshot.Members.Values
.Where(s => s.Status == SiloStatus.Active)
.OrderBy(s => s.SiloAddress)
.Take(_options.CurrentValue.MaxAgents)
.ToList();
if (!_enableMonitoring && chosenSilos.Any(s => s.SiloAddress.Equals(_localSiloDetails.SiloAddress)))
{
_enableMonitoring = true;
_pauseMonitoringSemaphore.Release(1);
}
else if (_enableMonitoring)
{
_enableMonitoring = false;
}
if (_enableMonitoring && _options.CurrentValue.DeleteDefunctSiloPods)
{
var delta = update.CreateUpdate(previous);
foreach (var change in delta.Changes)
{
if (change.SiloAddress.Equals(_localSiloDetails.SiloAddress))
{
// Ignore all changes for this silo
continue;
}
if (change.Status == SiloStatus.Dead)
{
try
{
if (_logger.IsEnabled(LogLevel.Information))
{
_logger.LogInformation("Silo {SiloAddress} is dead, proceeding to delete the corresponding pod, {PodName}, in namespace {PodNamespace}", change.SiloAddress, change.Name, _podNamespace);
}
await _client.DeleteNamespacedPodAsync(change.Name, _podNamespace);
}
catch (Exception exception)
{
_logger.LogError(exception, "Error deleting pod {PodName} in namespace {PodNamespace} corresponding to defunct silo {SiloAddress}", change.Name, _podNamespace, change.SiloAddress);
}
}
}
}
previous = update;
}
}
catch (Exception exception) when(!(_shutdownToken.IsCancellationRequested && (exception is TaskCanceledException || exception is OperationCanceledException)))
{
_logger.LogError(exception, "Error monitoring cluster changes");
if (!_shutdownToken.IsCancellationRequested)
{
await Task.Delay(5000);
}
}
}
}