private async Task MonitorOrleansClustering() { var previous = _clusterMembershipService.CurrentSnapshot; while (!_shutdownToken.IsCancellationRequested) { try { await foreach (var update in _clusterMembershipService.MembershipUpdates.WithCancellation(_shutdownToken.Token)) { // Determine which silos should be monitoring Kubernetes var chosenSilos = _clusterMembershipService.CurrentSnapshot.Members.Values .Where(s => s.Status == SiloStatus.Active) .OrderBy(s => s.SiloAddress) .Take(_options.CurrentValue.MaxAgents) .ToList(); if (!_enableMonitoring && chosenSilos.Any(s => s.SiloAddress.Equals(_localSiloDetails.SiloAddress))) { _enableMonitoring = true; _pauseMonitoringSemaphore.Release(1); } else if (_enableMonitoring) { _enableMonitoring = false; } if (_enableMonitoring && _options.CurrentValue.DeleteDefunctSiloPods) { var delta = update.CreateUpdate(previous); foreach (var change in delta.Changes) { if (change.SiloAddress.Equals(_localSiloDetails.SiloAddress)) { // Ignore all changes for this silo continue; } if (change.Status == SiloStatus.Dead) { try { if (_logger.IsEnabled(LogLevel.Information)) { _logger.LogInformation("Silo {SiloAddress} is dead, proceeding to delete the corresponding pod, {PodName}, in namespace {PodNamespace}", change.SiloAddress, change.Name, _podNamespace); } await _client.DeleteNamespacedPodAsync(change.Name, _podNamespace); } catch (Exception exception) { _logger.LogError(exception, "Error deleting pod {PodName} in namespace {PodNamespace} corresponding to defunct silo {SiloAddress}", change.Name, _podNamespace, change.SiloAddress); } } } } previous = update; } } catch (Exception exception) when(!(_shutdownToken.IsCancellationRequested && (exception is TaskCanceledException || exception is OperationCanceledException))) { _logger.LogError(exception, "Error monitoring cluster changes"); if (!_shutdownToken.IsCancellationRequested) { await Task.Delay(5000); } } } }