public override void HandleItem(WatchEventType eventType, DatabaseResource item)
{
Logger.LogDebug("Recieved new Database object (v {ResourceVersion})", item.Metadata.ResourceVersion);
try {
var servers = GetServerResources(item.Metadata.NamespaceProperty, item.Spec.DatabaseSelector);
foreach (var server in servers)
{
if (eventType == WatchEventType.Added)
{
if (sqlService.DoesDatabaseExist(server.Spec, item.Metadata.Name))
{
Logger.LogInformation("Database {database} already exists on server {server}", item.Metadata.Name, server.Metadata.Name);
k8sService.UpdateDatabaseStatus(item, "Available", "Database already exists", DateTimeOffset.Now);
continue;
}
if (item.Spec.BackupFiles?.Any() ?? false)
{
sqlService.RestoreDatabase(server.Spec, item);
k8sService.UpdateDatabaseStatus(item, "Available", "Database restored", DateTimeOffset.Now);
}
else
{
sqlService.CreateDatabase(server.Spec, item);
k8sService.UpdateDatabaseStatus(item, "Available", "Database created", DateTimeOffset.Now);
}
Logger.LogInformation("Created database {database}", item.Metadata.Name);
}
else if (eventType == WatchEventType.Deleted)
{
if (item.Spec.GCStrategy == GarbageCollectionStrategy.Delete)
{
sqlService.DeleteDatabase(server.Spec, item.Metadata.Name);
}
}
}
}
catch (Exception ex)
{
Logger.LogError(ex, "An error occurred during message processing");
try
{
k8sService.UpdateDatabaseStatus(item, "Failed", ex.GetType().Name, DateTimeOffset.Now);
eventRecorder.Record("CreateDatabase",
"Failed",
ex.Message,
new V1ObjectReference(
item.ApiVersion,
kind: item.Kind,
name: item.Metadata.Name,
namespaceProperty: item.Metadata.NamespaceProperty)
);
}
catch (HttpOperationException httpEx)
{
Logger.LogError(httpEx, "An error occurred logging this info to Kubernetes");
Logger.LogDebug(httpEx.Response.Content);
}
catch (Exception) {
throw;
}
}
}
public override bool HandleDequeuedItem(WatchEventType eventType, DeploymentScriptResource item, int previousExecutionCount)
{
Metrics.Measure.Counter.Increment(options);
if (eventType != WatchEventType.Deleted)
{
try
{
int executionCount = 0;
var databases = k8sService.GetDatabases(item.Metadata.NamespaceProperty, item.Spec.DatabaseSelector);
foreach (var database in databases.Items)
{
if (database.Status.Reason == "Available")
{
var servers = k8sService.GetDatabaseServer(item.Metadata.NamespaceProperty, database.Spec.DatabaseServerSelector);
foreach (var server in servers.Items)
{
rehydrator.Rehydrate(server);
sqlService.ExecuteScript(server.Spec, database, item.Spec.Script);
executionCount++;
}
}
}
if (executionCount > 0)
{
k8sService.UpdateDeploymentScriptStatus(item, "Executed", "Successfully executed", DateTimeOffset.Now);
return(true);
}
}
catch (Exception ex)
{
Logger.LogError(ex, "An error occurred during message processing");
try
{
k8sService.UpdateDeploymentScriptStatus(item, "Failed", ex.GetType().Name, DateTimeOffset.Now);
eventRecorder.Record("ExecuteDeploymentScript",
"Failed",
ex.Message,
new V1ObjectReference(
item.ApiVersion,
kind: item.Kind,
name: item.Metadata.Name,
namespaceProperty: item.Metadata.NamespaceProperty)
);
}
catch (HttpOperationException httpEx)
{
Logger.LogError(httpEx, "An error occurred logging this info to Kubernetes");
Logger.LogDebug(httpEx.Response.Content);
}
catch (Exception)
{
throw;
}
}
return(false);
}
return(true);
}
public override void HandleItem(WatchEventType eventType, ExampleResource item)
{
HandledItems++;
}
/// <summary>
/// Called by the informer with real-time resource updates.
/// </summary>
/// <param name="eventType">Indicates if the resource new, updated, or deleted.</param>
/// <param name="resource">The information as provided by the Kubernets API server.</param>
private void Notification(WatchEventType eventType, V1Service resource)
{
_cache.Update(eventType, resource);
_queue.Add(new QueueItem(NamespacedName.From(resource), null));
}
public static void PodStatus(WatchEventType type, string podname)
{
Log.LogDebug((int)EventIds.PodStatus, $"Pod '{podname}', status'{type}'");
}
public CustomResourceEvent(WatchEventType type, CustomResource resource)
{
Type = type;
Resource = resource;
}
public AeTitleUpdatedEventArgs(WatchEventType eventType) => EventType = eventType;
public bool Execute(WatchEventType eventType, TCustomResource resource)
{
return(predicate(resource));
}
/// <summary>
/// Called by the informer with real-time resource updates.
/// </summary>
/// <param name="eventType">Indicates if the resource new, updated, or deleted.</param>
/// <param name="resource">The information as provided by the Kubernets API server.</param>
private void Notification(WatchEventType eventType, V1Ingress resource)
{
_cache.Update(eventType, resource);
_queue.Add(_ingressChangeQueueItem);
}
public async Task ProcessEventAsync_DuplicateGenerationsAreDiscardedByDefault(WatchEventType eventType)
{
// Arrange
var resource_v1 = CreateCustomResource(generation: 1);
var resource_v2 = CreateCustomResource(generation: 2);
// Act
await _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v1), DUMMY_TOKEN);
await _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v1), DUMMY_TOKEN); // duplicate generation
await _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v2), DUMMY_TOKEN);
// Assert
VerifyAddOrModifyIsCalledWith(_controller, resource_v1, resource_v2);
VerifyDeleteIsNotCalled(_controller);
VerifyNoOtherApiIsCalled();
}
public async Task ProcessEventAsync_DuplicateGenerationsAreProcessedIfForcedByConfiguration(WatchEventType eventType)
{
// Arrange
var controller = new TestableController(new OperatorConfiguration()
{
DiscardDuplicateSpecGenerations = false
}, _client);
var resource_v1 = CreateCustomResource(generation: 1);
var resource_v2 = CreateCustomResource(generation: 2);
// Act
await controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v1), DUMMY_TOKEN);
await controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v1), DUMMY_TOKEN); // duplicate generation
await controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v2), DUMMY_TOKEN);
// Assert
VerifyAddOrModifyIsCalledWith(controller, resource_v1, resource_v1, resource_v2);
VerifyDeleteIsNotCalled(controller);
VerifyNoOtherApiIsCalled();
}
public void ProcessEventAsync_EventsForDifferentResourceAreProcessedConcurrently(WatchEventType eventType1, bool delete1, WatchEventType eventType2, bool delete2)
{
var resource1 = CreateCustomResource(uid: "1", deletionTimeStamp: delete1 ? DateTime.Now : (DateTime?)null);
var resource2 = CreateCustomResource(uid: "2", deletionTimeStamp: delete2 ? DateTime.Now : (DateTime?)null);
// Send 2 updates in a row for the different resources
// Update #1
var token1 = _controller.BlockNextEvent();
var task1 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType1, resource1), DUMMY_TOKEN);
// Update #2
var token2 = _controller.BlockNextEvent();
var task2 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType2, resource2), DUMMY_TOKEN);
// Updates are processed concurrently
VerifyCalledEvents(_controller, (resource1, delete1), (resource2, delete2));
// Updates end
_controller.UnblockEvent(token1);
_controller.UnblockEvent(token2);
Task.WaitAll(task2, task1);
}
public void ProcessEventAsync_EventsForSameResourceAreProcessedSerially(WatchEventType eventType1, bool delete1, WatchEventType eventType2, bool delete2)
{
var resource_v1 = CreateCustomResource(generation: 1, deletionTimeStamp: delete1 ? DateTime.Now : (DateTime?)null);
var resource_v2 = CreateCustomResource(generation: 2, deletionTimeStamp: delete2 ? DateTime.Now : (DateTime?)null);
// Send 2 updates in a row for the same resource
// Update #1
var token1 = _controller.BlockNextEvent();
var task1 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType1, resource_v1), DUMMY_TOKEN);
// Update #2
var token2 = _controller.BlockNextEvent();
var task2 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType2, resource_v2), DUMMY_TOKEN);
// Update #1 starts, update #2 is waiting
VerifyCalledEvents(_controller, (resource_v1, delete1));
// Update #1 ends, update #2 starts
_controller.UnblockEvent(token1);
VerifyCalledEvents(_controller, (resource_v1, delete1), (resource_v2, delete2));
// Update #2 ends
_controller.UnblockEvent(token2);
Task.WaitAll(task2, task1);
}
public ImmutableList <string> Update(WatchEventType eventType, V1Endpoints endpoints)
{
throw new NotImplementedException();
}
private void Notification(WatchEventType eventType, V1ConfigMap configMap)
{
}
private async Task ProcessEvent(WatchEventType eventType, T item)
{
var metadata = GetMetadata(item);
if (metadata.Annotations == null)
{
return;
}
if (!metadata.Annotations.ContainsKey(Annotations.Reflection.Reflects) &&
!metadata.Annotations.ContainsKey(Annotations.Reflection.Allowed))
{
return;
}
Logger.LogDebug("{kind} {ns}/{name} was {eventType}", item.Kind, metadata.NamespaceProperty, metadata.Name,
eventType);
var key = new NamespacedResource(metadata);
switch (eventType)
{
case WatchEventType.Added:
case WatchEventType.Modified:
if (metadata.Annotations.TryGetValue(Annotations.Reflection.Reflects, out var reflects))
{
foreach (var mirror in _mirrors)
{
mirror.Value.RemoveAll(s => NamespacedResource.Comparer.Equals(s, key));
}
var source = new NamespacedResource(reflects);
if (!_mirrors.TryGetValue(source, out var mirrorList))
{
mirrorList = new List <NamespacedResource>();
_mirrors.AddOrUpdate(source, mirrorList, (_, __) => mirrorList);
}
if (!mirrorList.Contains(key, NamespacedResource.Comparer))
{
mirrorList.Add(key);
}
T sourceSecret = null;
try
{
sourceSecret = await GetRequest(ApiClient, source.Name, source.Namespace);
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
Logger.LogWarning("{kind} {ns}/{name} cannot reflect {kind} {sourceNs}/{sourceName}." +
" Source {kind} {sourceNs}/{sourceName} could not be found.",
item.Kind, key.Namespace, key.Name,
item.Kind, source.Namespace, source.Name,
item.Kind, source.Namespace, source.Name);
}
if (sourceSecret != null)
{
await Mirror(sourceSecret, item);
}
}
if (metadata.ReflectionAllowed())
{
if (!_mirrors.TryGetValue(key, out var mirrorList))
{
mirrorList = new List <NamespacedResource>();
_mirrors.AddOrUpdate(key, mirrorList, (_, __) => mirrorList);
}
foreach (var mirror in mirrorList)
{
var targetSecret =
await GetRequest(ApiClient, mirror.Name, mirror.Namespace);
await Mirror(item, targetSecret);
}
}
break;
case WatchEventType.Deleted:
{
_mirrors.TryRemove(key, out _);
foreach (var mirror in _mirrors)
{
if (!mirror.Value.Any(s => NamespacedResource.Comparer.Equals(s, key)))
{
continue;
}
Logger.LogDebug("Removing {kind} {ns}/{name} from {kind} {sourceNs}/{sourceName} mirror list",
item.Kind, key.Namespace, key.Name,
item.Kind, mirror.Key.Namespace, mirror.Key.Name);
mirror.Value.RemoveAll(s =>
NamespacedResourceEqualityComparer.Instance.Equals(s, key));
}
}
break;
case WatchEventType.Error:
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private void Notification(WatchEventType eventType, V1Pod pod)
{
}
public abstract bool HandleDequeuedItem(WatchEventType type, TCustomResource item, int executionCount);
public override void HandleItem(WatchEventType eventType, DatabaseServerResource item)
{
}
/// <summary>
/// Simulates an incoming event for a given controller
/// </summary>
public void SimulateEvent(IController controller, WatchEventType eventType, CustomResource resource)
{
_watchers.Single(x => x.Controller == controller).OnIncomingEvent(eventType, resource);
}
public static void PodStatus(WatchEventType type, V1Pod pod)
{
Log.LogDebug((int)EventIds.PodStatus, $"Pod '{pod.Metadata.Name}', status'{type}'");
}
/// <summary>
/// Called by the informer with real-time resource updates.
/// </summary>
/// <param name="eventType">Indicates if the resource new, updated, or deleted.</param>
/// <param name="resource">The information as provided by the Kubernetes API server.</param>
private void Notification(WatchEventType eventType, V1IngressClass resource)
{
_cache.Update(eventType, resource);
}
public static void DeploymentStatus(WatchEventType type, ResourceName name)
{
Log.LogDebug((int)EventIds.DeploymentStatus, $"Deployment '{name}', status'{type}'");
}
public static Watcher <T> .WatchEvent ToWatchEvent <T>(this T obj, WatchEventType eventType)
{
return(new Watcher <T> .WatchEvent {
Type = eventType, Object = obj
});
}
public void Update(WatchEventType eventType, V1Ingress ingress)
{
if (ingress is null)
{
throw new ArgumentNullException(nameof(ingress));
}
var serviceNames = ImmutableList <string> .Empty;
if (eventType == WatchEventType.Added || eventType == WatchEventType.Modified)
{
// If the ingress exists, list out the related services
var spec = ingress.Spec;
var defaultBackend = spec?.DefaultBackend;
var defaultService = defaultBackend?.Service;
if (!string.IsNullOrEmpty(defaultService?.Name))
{
serviceNames = serviceNames.Add(defaultService.Name);
}
foreach (var rule in spec.Rules ?? Enumerable.Empty <V1IngressRule>())
{
var http = rule.Http;
foreach (var path in http.Paths ?? Enumerable.Empty <V1HTTPIngressPath>())
{
var backend = path.Backend;
var service = backend.Service;
if (!serviceNames.Contains(service.Name))
{
serviceNames = serviceNames.Add(service.Name);
}
}
}
}
var ingressName = ingress.Name();
lock (_sync)
{
var serviceNamesPrevious = ImmutableList <string> .Empty;
if (eventType == WatchEventType.Added || eventType == WatchEventType.Modified)
{
// If the ingress exists then remember details
_ingressData[ingressName] = new IngressData(ingress);
if (_ingressToServiceNames.TryGetValue(ingressName, out serviceNamesPrevious))
{
_ingressToServiceNames[ingressName] = serviceNames;
}
else
{
serviceNamesPrevious = ImmutableList <string> .Empty;
_ingressToServiceNames.Add(ingressName, serviceNames);
}
}
else if (eventType == WatchEventType.Deleted)
{
// otherwise clear out details
_ingressData.Remove(ingressName);
if (_ingressToServiceNames.TryGetValue(ingressName, out serviceNamesPrevious))
{
_ingressToServiceNames.Remove(ingressName);
}
}
// update cross-reference for new ingress-to-services linkage not previously known
foreach (var serviceName in serviceNames)
{
if (!serviceNamesPrevious.Contains(serviceName))
{
if (_serviceToIngressNames.TryGetValue(serviceName, out var ingressNamesPrevious))
{
_serviceToIngressNames[serviceName] = _serviceToIngressNames[serviceName].Add(ingressName);
}
else
{
_serviceToIngressNames.Add(serviceName, ImmutableList <string> .Empty.Add(ingressName));
}
}
}
// remove cross-reference for previous ingress-to-services linkage no longer present
foreach (var serviceName in serviceNamesPrevious)
{
if (!serviceNames.Contains(serviceName))
{
_serviceToIngressNames[serviceName] = _serviceToIngressNames[serviceName].Remove(ingressName);
}
}
}
}
private async Task HandleKubernetesEvent(WatchEventType type, V1Event kubernetesEvent)
{
var cloudEvent = _cloudEventsSchematizer.GenerateFromKubernetesEvent(kubernetesEvent);
await _cloudEventsPublisher.Publish(cloudEvent);
}
public ImmutableList <string> Update(WatchEventType eventType, V1Endpoints endpoints)
{
return(Namespace(endpoints.Namespace()).Update(eventType, endpoints));
}
/// <summary>
/// Initializes a new <see cref="ResourceEvent{TResource}"/>
/// </summary>
/// <param name="type">The event's type</param>
/// <param name="resource">The custom resource that has produced the event</param>
public ResourceEvent(WatchEventType type, TResource resource)
{
this.Type = type;
this.Resource = resource;
}
