internal static void MapCustomUIPaths(this V1Container container, HealthCheckResource resource, OperatorDiagnostics diagnostics)
{
var uiPath = resource.Spec.UiPath ?? Constants.DEFAULT_UI_PATH;
container.Env.Add(new V1EnvVar("ui_path", uiPath));
diagnostics.UiPathConfigured(nameof(resource.Spec.UiPath), uiPath);
if (!string.IsNullOrEmpty(resource.Spec.UiApiPath))
{
container.Env.Add(new V1EnvVar("ui_api_path", resource.Spec.UiApiPath));
diagnostics.UiPathConfigured(nameof(resource.Spec.UiApiPath), resource.Spec.UiApiPath);
}
if (!string.IsNullOrEmpty(resource.Spec.UiResourcesPath))
{
container.Env.Add(new V1EnvVar("ui_resources_path", resource.Spec.UiResourcesPath));
diagnostics.UiPathConfigured(nameof(resource.Spec.UiResourcesPath), resource.Spec.UiResourcesPath);
}
if (!string.IsNullOrEmpty(resource.Spec.UiWebhooksPath))
{
container.Env.Add(new V1EnvVar("ui_webhooks_path", resource.Spec.UiWebhooksPath));
diagnostics.UiPathConfigured(nameof(resource.Spec.UiWebhooksPath), resource.Spec.UiWebhooksPath);
}
if (resource.Spec.UiNoRelativePaths.HasValue)
{
var noRelativePaths = resource.Spec.UiNoRelativePaths.Value.ToString();
container.Env.Add(new V1EnvVar("ui_no_relative_paths", noRelativePaths));
diagnostics.UiPathConfigured(nameof(resource.Spec.UiNoRelativePaths), noRelativePaths);
}
}
public async Task <V1Deployment> GetOrCreateAsync(HealthCheckResource resource)
{
var deployment = await Get(resource);
if (deployment != null)
{
return(deployment);
}
try
{
var deploymentResource = Build(resource);
var response =
await _client.CreateNamespacedDeploymentWithHttpMessagesAsync(deploymentResource,
resource.Metadata.NamespaceProperty);
deployment = response.Body;
_logger.LogInformation("Deployment {deployment} has been created", deployment.Metadata.Name);
}
catch (Exception ex)
{
_logger.LogError("Error creating deployment: {error}", ex.Message);
}
return(deployment);
}
public static string CreateAddress(V1Service service, HealthCheckResource resource)
{
var defaultPort = int.Parse(resource.Spec.PortNumber ?? Constants.DEFAULT_PORT);
var port = GetServicePort(service)?.Port ?? defaultPort;
var address = service.Spec.ClusterIP;
string healthScheme = resource.Spec.HealthChecksScheme;
if (service.Metadata.Annotations?.ContainsKey(Constants.HEALTH_CHECK_SCHEME_ANNOTATION) ?? false)
{
healthScheme = service.Metadata.Annotations[Constants.HEALTH_CHECK_SCHEME_ANNOTATION];
}
if (healthScheme.IsEmpty())
{
healthScheme = Constants.DEFAULT_SCHEME;
}
if (address.Contains(":"))
{
return($"{healthScheme}://[{address}]:{port}");
}
else
{
return($"{healthScheme}://{address}:{port}");
}
}
public V1Service Build(HealthCheckResource resource)
{
var meta = new V1ObjectMeta
{
Name = $"{resource.Spec.Name}-svc",
OwnerReferences = new List <V1OwnerReference> {
resource.CreateOwnerReference()
},
Labels = new Dictionary <string, string>
{
["app"] = resource.Spec.Name
},
};
var spec = new V1ServiceSpec
{
Selector = new Dictionary <string, string>
{
["app"] = resource.Spec.Name
},
Type = resource.Spec.ServiceType ?? Constants.DefaultServiceType,
Ports = new List <V1ServicePort> {
new V1ServicePort {
Name = "httport",
Port = int.Parse(resource.Spec.PortNumber ?? Constants.DefaultPort),
TargetPort = 80
}
}
};
return(new V1Service(metadata: meta, spec: spec));
}
public async Task <V1Deployment> GetOrCreateAsync(HealthCheckResource resource)
{
var deployment = await Get(resource);
if (deployment != null)
{
return(deployment);
}
try
{
var deploymentResource = Build(resource);
var response =
await _client.CreateNamespacedDeploymentWithHttpMessagesAsync(deploymentResource,
resource.Metadata.NamespaceProperty);
deployment = response.Body;
_operatorDiagnostics.DeploymentCreated(deployment.Metadata.Name);
}
catch (Exception ex)
{
_operatorDiagnostics.DeploymentOperationError(deployment.Metadata.Name, Deployment.Operation.ADD, ex.Message);
}
return(deployment);
}
public static string CreateAddress(V1Service service, HealthCheckResource resource)
{
var defaultPort = int.Parse(resource.Spec.PortNumber ?? Constants.DefaultPort);
var port = GetServicePort(service)?.Port ?? defaultPort;
var address = service.Spec.ClusterIP;
string healthScheme = resource.Spec.HealthChecksScheme;
if (service.Metadata.Annotations?.ContainsKey(Constants.HealthCheckSchemeAnnotation) ?? false)
{
healthScheme = service.Metadata.Annotations[Constants.HealthCheckSchemeAnnotation];
}
if (string.IsNullOrEmpty(healthScheme))
{
healthScheme = Constants.DefaultScheme;
}
if (address.Contains(":"))
{
return($"{healthScheme}://[{address}]:{port}");
}
else
{
return($"{healthScheme}://{address}:{port}");
}
}
public async Task <V1Secret> GetOrCreateAsync(HealthCheckResource resource)
{
var secret = await Get(resource);
if (secret != null)
{
return(secret);
}
try
{
var secretResource = Build(resource);
secret = await _client.CreateNamespacedSecretAsync(secretResource,
resource.Metadata.NamespaceProperty);
_logger.LogInformation("Secret {name} has been created", secret.Metadata.Name);
}
catch (Exception ex)
{
_logger.LogError("Error creating Secret: {message}", ex.Message);
throw;
}
return(secret);
}
internal Task Watch(HealthCheckResource resource, CancellationToken token)
{
Func <HealthCheckResource, bool> filter = (k) => k.Metadata.NamespaceProperty == resource.Metadata.NamespaceProperty;
if (!_watchers.Keys.Any(filter))
{
var response = _client.ListNamespacedServiceWithHttpMessagesAsync(
namespaceParameter: resource.Metadata.NamespaceProperty,
labelSelector: $"{resource.Spec.ServicesLabel}",
watch: true,
cancellationToken: token);
var watcher = response.Watch <V1Service, V1ServiceList>(
onEvent: async(type, item) => await _notificationHandler.NotifyDiscoveredServiceAsync(type, item, resource),
onError: e =>
{
_diagnostics.ServiceWatcherThrow(e);
Watch(resource, token);
}
);
_diagnostics.ServiceWatcherStarting(resource.Metadata.NamespaceProperty);
_watchers.Add(resource, watcher);
}
return(Task.CompletedTask);
}
public async Task DeleteDeploymentAsync(HealthCheckResource resource)
{
_logger.LogInformation("Deleting healthchecks deployment {name}", resource.Spec.Name);
await _secretHandler.Delete(resource);
await _deploymentHandler.Delete(resource);
await _serviceHandler.Delete(resource);
}
public static V1OwnerReference CreateOwnerReference(this HealthCheckResource resource)
{
return(new V1OwnerReference
{
Name = resource.Spec.Name,
ApiVersion = resource.ApiVersion,
Uid = resource.Metadata.Uid,
Kind = resource.Kind,
Controller = true
});
}
public async Task DeleteAsync(HealthCheckResource resource)
{
try
{
await _client.DeleteNamespacedSecretAsync($"{resource.Spec.Name}-secret", resource.Metadata.NamespaceProperty);
}
catch (Exception ex)
{
_logger.LogError("Error deleting secret for hc resource {name} : {message}", resource.Spec.Name, ex.Message);
}
}
public async Task DeleteAsync(HealthCheckResource resource)
{
try
{
await _client.DeleteNamespacedDeploymentAsync($"{resource.Spec.Name}-deploy",
resource.Metadata.NamespaceProperty);
}
catch (Exception ex)
{
_operatorDiagnostics.DeploymentOperationError(resource.Spec.Name, Deployment.Operation.DELETE, ex.Message);
}
}
public async Task Delete(HealthCheckResource resource)
{
try
{
await _client.DeleteNamespacedDeploymentAsync($"{resource.Spec.Name}-deploy",
resource.Metadata.NamespaceProperty);
}
catch (Exception ex)
{
_logger.LogError("Error deleting deployment for hc resource: {name} - err: {error}", resource.Spec.Name, ex.Message);
}
}
private void StopServiceWatcher(HealthCheckResource resource)
{
Action stopWatcher = () => _serviceWatcher.Stopwatch(resource);
Action stopClusterWatcher = () => _clusterServiceWatcher.Stopwatch(resource);
var stop = resource.Spec.Scope switch
{
Deployment.Scope.Namespaced => stopWatcher,
Deployment.Scope.Cluster => stopClusterWatcher,
_ => throw new ArgumentOutOfRangeException(nameof(resource.Spec.Scope))
};
stop();
}
internal void Stopwatch(HealthCheckResource resource)
{
Func <HealthCheckResource, bool> filter = (k) => k.Metadata.NamespaceProperty == resource.Metadata.NamespaceProperty;
if (_watchers.Keys.Any(filter))
{
var svcResource = _watchers.Keys.FirstOrDefault(filter);
if (svcResource != null)
{
_diagnostics.ServiceWatcherStopped(resource.Metadata.NamespaceProperty);
_watchers[svcResource]?.Dispose();
_watchers.Remove(svcResource);
}
}
}
/// <inheritdoc />
public async Task <HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = new CancellationToken())
{
var result = new HealthCheckResource();
var timer = Stopwatch.StartNew();
foreach (var healthTest in _healthTests)
{
result.Tests.Add(await healthTest.RunTestAsync());
}
timer.Stop();
result.ReportAsOf = DateTime.UtcNow;
result.ReportDuration = $"{timer.Elapsed.TotalSeconds} seconds";
_healthStatusFlag.SetHealthStatus(result.Tests.Any(t => t.TestResult == TestResult.Failed) ? HealthStatus.Unhealthy : HealthStatus.Healthy);
return(new HealthCheckResult(_healthStatusFlag.CurrentHealth, data: JObject.FromObject(result).ToObject <Dictionary <string, object> >()));
}
internal void Stopwatch(HealthCheckResource resource)
{
Func <HealthCheckResource, bool> filter = (k) => k.Metadata.NamespaceProperty == resource.Metadata.NamespaceProperty;
if (_watchers.Keys.Any(filter))
{
var svcResource = _watchers.Keys.FirstOrDefault(filter);
if (svcResource != null)
{
_logger.LogInformation("Stopping services watcher for namespace {namespace}", resource.Metadata.NamespaceProperty);
_watchers[svcResource].Dispose();
_watchers.Remove(svcResource);
}
}
}
private async Task StartServiceWatcher(HealthCheckResource resource)
{
Func <Task> startWatcher = async() => await _serviceWatcher.Watch(resource, _operatorCts.Token);
Func <Task> startClusterWatcher = async() => await _clusterServiceWatcher.Watch(resource, _operatorCts.Token);
var start = resource.Spec.Scope switch
{
Deployment.Scope.Namespaced => startWatcher,
Deployment.Scope.Cluster => startClusterWatcher,
_ => throw new ArgumentOutOfRangeException(nameof(resource.Spec.Scope))
};
await start();
}
private async Task OnEventHandlerAsync(WatchEventType type, HealthCheckResource item, CancellationToken token)
{
if (type == WatchEventType.Added)
{
await _controller.DeployAsync(item);
_serviceWatcher.Watch(item, token);
}
if (type == WatchEventType.Deleted)
{
await _controller.DeleteDeploymentAsync(item);
_serviceWatcher.Stopwatch(item);
}
}
internal Task Watch(HealthCheckResource resource, CancellationToken token)
{
var response = _client.ListServiceForAllNamespacesWithHttpMessagesAsync(
labelSelector: $"{resource.Spec.ServicesLabel}",
watch: true,
cancellationToken: token);
_watcher = response.Watch <V1Service, V1ServiceList>(
onEvent: async(type, item) => await _notificationHandler.NotifyDiscoveredServiceAsync(type, item, resource),
onError: e => _diagnostics.ServiceWatcherThrow(e)
);
_diagnostics.ServiceWatcherStarting("All");
return(Task.CompletedTask);
}
public async Task <DeploymentResult> DeployAsync(HealthCheckResource resource)
{
_logger.LogInformation("Creating secret for hc resource - namespace {namespace}", resource.Metadata.NamespaceProperty);
var secret = await _secretHandler.GetOrCreate(resource);
_logger.LogInformation("Creating deployment for hc resource - namespace {namespace}", resource.Metadata.NamespaceProperty);
var deployment = await _deploymentHandler.GetOrCreateAsync(resource);
_logger.LogInformation("Creating service for hc resource - namespace {namespace}", resource.Metadata.NamespaceProperty);
var service = await _serviceHandler.GetOrCreateAsync(resource);
return(DeploymentResult.Create(deployment, service, secret));
}
public static async Task PushNotification(
WatchEventType eventType,
HealthCheckResource resource,
V1Service uiService,
V1Service notificationService,
V1Secret endpointSecret,
ILogger <K8sOperator> logger,
IHttpClientFactory httpClientFactory)
{
var address = KubernetesAddressFactory.CreateHealthAddress(notificationService, resource);
var uiAddress = KubernetesAddressFactory.CreateAddress(uiService, resource);
dynamic healthCheck = new
{
Type = eventType,
notificationService.Metadata.Name,
Uri = address
};
var client = httpClientFactory.CreateClient();
try
{
string type = healthCheck.Type.ToString();
string name = healthCheck.Name;
string uri = healthCheck.Uri;
logger.LogInformation("[PushService] Namespace {Namespace} - Sending Type: {type} - Service {name} with uri : {uri} to ui endpoint: {address}", resource.Metadata.NamespaceProperty, type, name, uri, uiAddress);
var key = Encoding.UTF8.GetString(endpointSecret.Data["key"]);
var response = await client.PostAsync($"{uiAddress}{Constants.PushServicePath}?{Constants.PushServiceAuthKey}={key}",
new StringContent(JsonSerializer.Serialize(healthCheck, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
}), Encoding.UTF8, "application/json"));
logger.LogInformation("[PushService] Notification result for {name} - status code: {statuscode}", notificationService.Metadata.Name, response.StatusCode);
}
catch (Exception ex)
{
logger.LogError("Error notifying healthcheck service: {message}", ex.Message);
}
}
public static string CreateHealthAddress(V1Service service, HealthCheckResource resource)
{
var address = CreateAddress(service, resource);
string healthPath = resource.Spec.HealthChecksPath;
if (service.Metadata.Annotations?.ContainsKey(Constants.HEALTH_CHECK_PATH_ANNOTATION) ?? false)
{
healthPath = service.Metadata.Annotations[Constants.HEALTH_CHECK_PATH_ANNOTATION];
}
if (healthPath.IsEmpty())
{
healthPath = Constants.DEFAULT_HEALTH_PATH;
}
return($"{address}/{ healthPath.TrimStart('/')}");
}
public static string CreateHealthAddress(V1Service service, HealthCheckResource resource)
{
var address = CreateAddress(service, resource);
string healthPath = resource.Spec.HealthChecksPath;
if (service.Metadata.Annotations?.ContainsKey(Constants.HealthCheckPathAnnotation) ?? false)
{
healthPath = service.Metadata.Annotations[Constants.HealthCheckPathAnnotation];
}
if (string.IsNullOrEmpty(healthPath))
{
healthPath = Constants.DefaultHealthPath;
}
return($"{address}/{ healthPath.TrimStart('/')}");
}
public V1ConfigMap Build(HealthCheckResource resource)
{
return(new V1ConfigMap
{
BinaryData = new Dictionary <string, byte[]>
{
[Constants.STYLE_SHEET_NAME] = Encoding.UTF8.GetBytes(resource.Spec.StylesheetContent)
},
Metadata = new V1ObjectMeta
{
OwnerReferences = new List <V1OwnerReference>
{
resource.CreateOwnerReference(),
},
NamespaceProperty = resource.Metadata.NamespaceProperty,
Name = $"{resource.Spec.Name}-config"
}
});
}
private async Task WaitForAvailableReplicas(HealthCheckResource resource)
{
int retries = 1;
int availableReplicas = 0;
while (retries <= WaitForReplicaRetries && availableReplicas == 0)
{
var deployment = await _client.ListNamespacedOwnedDeploymentAsync(resource.Metadata.NamespaceProperty, resource.Metadata.Uid);
availableReplicas = deployment.Status.AvailableReplicas ?? 0;
if (availableReplicas == 0)
{
_logger.LogInformation("The UI replica {Name} in {Namespace} is not available yet, retrying...{Retries}/{MaxRetries}", deployment.Metadata.Name, resource.Metadata.NamespaceProperty, retries, WaitForReplicaRetries);
await Task.Delay(WaitForReplicaDelay);
retries++;
}
}
}
public static string CreateAddress(V1Service service, HealthCheckResource resource)
{
var defaultPort = int.Parse(resource.Spec.PortNumber ?? Constants.DefaultPort);
var port = service.Spec.Type switch
{
ServiceType.LoadBalancer => GetServicePort(service)?.Port ?? defaultPort,
ServiceType.ClusterIP => GetServicePort(service)?.Port ?? defaultPort,
ServiceType.NodePort => GetServicePort(service)?.NodePort ?? defaultPort,
_ => throw new NotSupportedException($"{service.Spec.Type} port type not supported")
};
var address = service.Spec.Type switch
{
ServiceType.LoadBalancer => GetLoadBalancerAddress(service),
ServiceType.NodePort => GetLoadBalancerAddress(service),
ServiceType.ClusterIP => service.Spec.ClusterIP,
_ => throw new NotSupportedException($"{service.Spec.Type} port type not supported")
};
string healthScheme = resource.Spec.HealthChecksScheme;
if (service.Metadata.Annotations?.ContainsKey(Constants.HealthCheckSchemeAnnotation) ?? false)
{
healthScheme = service.Metadata.Annotations[Constants.HealthCheckSchemeAnnotation];
}
if (string.IsNullOrEmpty(healthScheme))
{
healthScheme = Constants.DefaultScheme;
}
if (address.Contains(":"))
{
return($"{healthScheme}://[{address}]:{port}");
}
else
{
return($"{healthScheme}://{address}:{port}");
}
}
public V1Secret Build(HealthCheckResource resource)
{
return(new V1Secret
{
Metadata = new V1ObjectMeta
{
Name = $"{resource.Spec.Name}-secret",
NamespaceProperty = resource.Metadata.NamespaceProperty,
OwnerReferences = new List <V1OwnerReference> {
resource.CreateOwnerReference()
},
Labels = new Dictionary <string, string>
{
["app"] = resource.Spec.Name
}
},
Data = new Dictionary <string, byte[]>
{
["key"] = Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())
}
});
}
public V1Service Build(HealthCheckResource resource)
{
var meta = new V1ObjectMeta
{
Name = $"{resource.Spec.Name}-svc",
OwnerReferences = new List <V1OwnerReference> {
resource.CreateOwnerReference()
},
Annotations = new Dictionary <string, string>(),
Labels = new Dictionary <string, string>
{
["app"] = resource.Spec.Name
},
};
var spec = new V1ServiceSpec
{
Selector = new Dictionary <string, string>
{
["app"] = resource.Spec.Name
},
Type = resource.Spec.ServiceType ?? Constants.DefaultServiceType,
Ports = new List <V1ServicePort> {
new V1ServicePort {
Name = "httport",
Port = int.Parse(resource.Spec.PortNumber ?? Constants.DefaultPort),
TargetPort = 80
}
}
};
foreach (var annotation in resource.Spec.ServiceAnnotations)
{
_logger.LogInformation("Adding annotation {Annotation} to ui service with value {AnnotationValue}", annotation.Name, annotation.Value);
meta.Annotations.Add(annotation.Name, annotation.Value);
}
return(new V1Service(metadata: meta, spec: spec));
}
public async Task <V1ConfigMap> GetOrCreateAsync(HealthCheckResource resource)
{
var configMap = await Get(resource);
if (configMap != null)
{
return(configMap);
}
try
{
var configMapResource = Build(resource);
configMap = await _client.CreateNamespacedConfigMapAsync(configMapResource, resource.Metadata.NamespaceProperty);
_logger.LogInformation("Config Map {name} has been created", configMap.Metadata.Name);
}
catch (Exception ex)
{
_logger.LogError("Error creating config map for hc resource {name} : {message}", resource.Spec.Name, ex.Message);
}
return(configMap);
}