private async Task MonitorKubernetesPods()
{
while (!_shutdownToken.IsCancellationRequested)
{
try
{
if (!_enableMonitoring)
{
// Pulse the semaphore to avoid spinning in a tight loop.
await _pauseMonitoringSemaphore.WaitAsync();
continue;
}
if (_shutdownToken.IsCancellationRequested)
{
break;
}
var pods = await _client.ListNamespacedPodWithHttpMessagesAsync(
namespaceParameter : _podNamespace,
labelSelector : _podLabelSelector,
watch : true,
cancellationToken : _shutdownToken.Token);
await foreach (var(eventType, pod) in pods.WatchAsync <V1PodList, V1Pod>(_shutdownToken.Token))
{
if (!_enableMonitoring || _shutdownToken.IsCancellationRequested)
{
break;
}
if (string.Equals(pod.Metadata.Name, _podName, StringComparison.Ordinal))
{
// Never declare ourselves dead this way.
continue;
}
if (eventType == WatchEventType.Modified)
{
// TODO: Remember silo addresses for pods are restarting/terminating
}
if (eventType == WatchEventType.Deleted)
{
if (this.TryMatchSilo(pod, out var member) && member.Status != SiloStatus.Dead)
{
if (_logger.IsEnabled(LogLevel.Information))
{
_logger.LogInformation("Declaring server {Silo} dead since its corresponding pod, {Pod}, has been deleted", member.SiloAddress, pod.Metadata.Name);
}
await _clusterMembershipService.TryKill(member.SiloAddress);
}
}
}
if (_enableMonitoring && !_shutdownToken.IsCancellationRequested)
{
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.LogDebug("Unexpected end of stream from Kubernetes API. Will try again.");
}
await Task.Delay(5000);
}
}
catch (Exception exception)
{
_logger.LogError(exception, "Error monitoring Kubernetes pods");
if (!_shutdownToken.IsCancellationRequested)
{
await Task.Delay(5000);
}
}
}
}
private async Task MonitorKubernetesPods()
{
var jsonSettings = new JsonSerializerSettings
{
DefaultValueHandling = DefaultValueHandling.Ignore,
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
};
while (!_shutdownToken.IsCancellationRequested)
{
try
{
if (!_enableMonitoring)
{
// Pulse the semaphore to avoid spinning in a tight loop.
_logger.LogInformation("Waiting for Kubernetes monitoring to be enabled");
await _pauseMonitoringSemaphore.WaitAsync();
_logger.LogInformation("Woke up after slumber");
continue;
}
if (_shutdownToken.IsCancellationRequested)
{
_logger.LogInformation("Shutdown1");
break;
}
var pods = await _client.ListNamespacedPodWithHttpMessagesAsync(
namespaceParameter : _podNamespace,
labelSelector : _podLabelSelector,
watch : true,
cancellationToken : _shutdownToken.Token);
await foreach (var(eventType, pod) in pods.WatchAsync <V1PodList, V1Pod>(_shutdownToken.Token))
{
if (!_enableMonitoring || _shutdownToken.IsCancellationRequested)
{
_logger.LogInformation("Break loop");
break;
}
#if false
_logger.LogInformation(
"Event: {Event} Pod: {Pod}",
eventType.ToString(),
JsonConvert.SerializeObject(pod, jsonSettings));
#endif
_logger.LogInformation(
"Event: {Event} PodName: {PodName}",
eventType.ToString(),
pod.Metadata.Name);
if (string.Equals(pod.Metadata.Name, _podName, StringComparison.Ordinal))
{
// Never declare ourselves dead this way.
continue;
}
if (eventType == WatchEventType.Modified)
{
// TODO: Remember silo addresses for pods are restarting/terminating
}
if (eventType == WatchEventType.Deleted)
{
if (this.TryMatchSilo(pod, out var member) && member.Status != SiloStatus.Dead)
{
_logger.LogInformation("Declaring server {Silo} dead since its corresponding pod, {Pod}, has been deleted", member.SiloAddress, pod.Metadata.Name);
await _clusterMembershipService.TryKill(member.SiloAddress);
}
}
}
if (_enableMonitoring && !_shutdownToken.IsCancellationRequested)
{
_logger.LogDebug("Unexpected end of stream from Kubernetes API. Will try again.");
await Task.Delay(5000);
}
}
catch (Exception exception)
{
_logger.LogError(exception, "Error monitoring Kubernetes pods");
if (!_shutdownToken.IsCancellationRequested)
{
await Task.Delay(5000);
}
}
}
}
