private async Task CheckAutoReflections(TResource item) { var metadata = item.Metadata(); var id = KubernetesObjectId.For(metadata); var namespaces = new List <string>(); if (metadata.ReflectionAllowed() && metadata.ReflectionAutoEnabled()) { _autoReflections[id] = metadata.ReflectionAutoNamespaces(); var namespaceList = await _client.ListNamespaceAsync(); namespaces = namespaceList.Items .Select(s => s.Metadata.Name) .Where(s => metadata.ReflectionAutoNamespacesMatch(s) && metadata.ReflectionAllowedNamespacesMatch(s)) .ToList(); } else { _autoReflections.TryRemove(id, out _); namespaces.Clear(); } namespaces.Remove(metadata.NamespaceProperty); await UpdateAutoReflections(item, namespaces); }
private async Task OnEvent(WatcherEvent e) { var id = KubernetesObjectId.For(e.Item.Metadata()); _logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, id); await _mediator.Publish(e); }
private async Task OnEventHandlingError(WatcherEvent e, Exception ex) { var id = KubernetesObjectId.For(e.Item.Metadata()); if (ex is Microsoft.Rest.HttpOperationException httpEx) { _logger.LogError($"Microsoft.Rest.HttpOperationException response: {httpEx.Response.Content}"); } _logger.LogError(ex, "Failed to process {eventType} {kind} {@id} due to exception", e.Type, e.Item.Kind, id); await _secretsWatcher.Stop(); foreach (var certificatesWatcher in _certificatesWatchers.Values) { await certificatesWatcher.Stop(); } _eventQueue.Clear(); _logger.LogTrace("Watchers restarting"); await _secretsWatcher.Start(); foreach (var certificatesWatcher in _certificatesWatchers.Values) { await certificatesWatcher.Start(); } _logger.LogTrace("Watchers restarted"); }
public async Task Handle(InternalSecretWatcherEvent notification, CancellationToken cancellationToken) { if (notification.Type != WatchEventType.Added && notification.Type != WatchEventType.Modified) { return; } var secret = notification.Item; var secretId = KubernetesObjectId.For(secret.Metadata); var metadata = secret.Metadata; if (metadata.Labels == null) { return; } if (metadata.Labels.TryGetValue(CertManagerConstants.CertificateNameLabel, out var certificateName)) { var certificateId = new KubernetesObjectId(secretId.Namespace, certificateName); _logger.LogDebug("{kind} {@certificateId} has secret {@secretId}", CertManagerConstants.CertificateKind, certificateId, secretId); Certificate certificate = null; foreach (var certificateResourceDefinitionVersion in notification.CertificateResourceDefinitionVersions) { try { var certificateJObject = await _client.GetNamespacedCustomObjectAsync( CertManagerConstants.CrdGroup, certificateResourceDefinitionVersion, metadata.NamespaceProperty, CertManagerConstants.CertificatePlural, certificateName, cancellationToken); certificate = ((JObject)certificateJObject).ToObject <Certificate>(); } catch (HttpOperationException exception) when(exception.Response.StatusCode == HttpStatusCode.NotFound) { } } if (certificate != null) { await Annotate(secret, certificate); } else { _logger.LogDebug("Could not find {kind} {@id}", CertManagerConstants.CertificateKind, certificateId); } } }
protected async Task OnResourceWatcherEvent(WatcherEvent <TResource> request) { var item = request.Item; var eventType = request.Type; var metadata = item.Metadata(); var id = KubernetesObjectId.For(metadata); if (await OnResourceIgnoreCheck(item)) { return; } switch (eventType) { case WatchEventType.Added: case WatchEventType.Modified: { //Check if the source for auto reflection is still valid. if (await CheckAutoReflectionSource(item)) { return; } //Ensure auto reflections await CheckAutoReflections(item); //Update current item. If updated, return (Update will be picked up by watcher) if (await ReflectSourceToSelf(item)) { return; } //Update all child reflections await ReflectSelfToReflections(item); } break; case WatchEventType.Deleted: { _reflections.TryRemove(id, out _); _autoReflections.TryRemove(id, out _); //Remove all auto-reflections await UpdateAutoReflections(item, new List <string>()); } break; case WatchEventType.Error: break; default: throw new ArgumentOutOfRangeException(); } }
private async Task OnEventHandlingError(WatcherEvent e, Exception ex) { var id = KubernetesObjectId.For(e.Item.Metadata()); _logger.LogError(ex, "Failed to process {eventType} {kind} {@id} due to exception", e.Type, e.Item.Kind, id); await WatchersStop(); _eventQueue.Clear(); _logger.LogInformation("Restarting watchers "); await WatchersStart(); }
private async Task UpdateAutoReflections(TResource item, List <string> namespaces) { var metadata = item.Metadata(); var id = KubernetesObjectId.For(metadata); var autoReflections = await FindAutoReflections(id); var reflectionsToCreate = namespaces .Where(s => !autoReflections.Any(m => m.NamespaceProperty.Equals(s))).ToList(); var reflectionsToRemove = autoReflections .Where(s => !namespaces.Contains(s.NamespaceProperty)).ToList(); foreach (var reflectionNamespace in reflectionsToCreate) { var reflectionId = new KubernetesObjectId(reflectionNamespace, id.Name); try { await OnResourceAutoReflect(_client, item, reflectionNamespace); _logger.LogInformation( "Created reflection {kind} for {@id} in namespace {namespace}.", item.Kind, id, reflectionId.Namespace); } catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.Conflict) { _logger.LogWarning( "Cannot create reflection {kind} {@reflectionId} for {@id}. " + $"Found conflicting {{kind}} with the same name which was not created by {nameof(Reflector)}", item.Kind, reflectionId, id, item.Kind); } catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.Forbidden) { _logger.LogWarning( "Cannot create reflection {kind} {@reflectionId} for {@id}. " + "Access to namespace forbidden.", item.Kind, reflectionId, id, item.Kind); } } foreach (var reflection in reflectionsToRemove) { await OnResourceDelete(_client, reflection.Name, reflection.NamespaceProperty); _reflections.TryRemove(KubernetesObjectId.For(reflection), out _); _logger.LogInformation( "Deleted reflection {kind} for {@id} in namespace {targetNamespace}", item.Kind, id, reflection.NamespaceProperty); } }
private async Task OnEventHandlingError(WatcherEvent <V1Secret> e, Exception ex) { var id = KubernetesObjectId.For(e.Item.Metadata()); _logger.LogError(ex, "Failed to process {eventType} {kind} {@id} due to exception", e.Type, e.Item.Kind, id); await _secretsWatcher.Stop(); _eventQueue.Clear(); _logger.LogTrace("Watchers restarting"); await _secretsWatcher.Start(); _logger.LogTrace("Watchers restarted"); }
private async Task OnEventHandlingError(WatcherEvent e, Exception ex) { var id = KubernetesObjectId.For(e.Item.Metadata()); if (ex is Microsoft.Rest.HttpOperationException httpEx) { _logger.LogError($"Microsoft.Rest.HttpOperationException response: {httpEx.Response.Content}"); } _logger.LogError(ex, "Failed to process {eventType} {kind} {@id} due to exception", e.Type, e.Item.Kind, id); await WatchersStop(); _eventQueue.Clear(); _logger.LogInformation("Restarting watchers "); await WatchersStart(); }
private async Task OnEvent(WatcherEvent e) { var id = KubernetesObjectId.For(e.Item.Metadata()); _logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, id); switch (e) { case WatcherEvent <TResource> resourceEvent: await OnResourceWatcherEvent(resourceEvent); break; case WatcherEvent <V1Namespace> namespaceEvent: await OnNamespaceWatcherEvent(namespaceEvent); break; } }
private async Task <bool> CheckAutoReflectionSource(TResource item) { var metadata = item.Metadata(); var id = KubernetesObjectId.For(metadata); var autoReflectionSource = metadata.AutoReflects(); if (autoReflectionSource.Equals(KubernetesObjectId.Empty)) { return(false); } var delete = false; try { var source = await OnResourceGet(_client, autoReflectionSource.Name, autoReflectionSource.Namespace); var sourceMeta = source.Metadata(); if (!sourceMeta.ReflectionAllowed() || !sourceMeta.ReflectionAutoEnabled() || !sourceMeta.ReflectionAllowedNamespacesMatch(id.Namespace) || !sourceMeta.ReflectionAutoNamespacesMatch(id.Namespace)) { delete = true; } } catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound) { delete = true; } if (!delete) { return(false); } await OnResourceDelete(_client, id.Name, id.Namespace); _logger.LogInformation( "Deleted reflection {kind} for {@id} in namespace {targetNamespace}", item.Kind, autoReflectionSource, id.Namespace); return(true); }
private async Task ReflectSelfToReflections(TResource item) { var metadata = item.Metadata(); var id = KubernetesObjectId.For(metadata); var reflections = _reflections.Where(s => s.Value.Equals(id)).Select(s => s.Key).ToList(); foreach (var reflectionId in reflections) { try { var target = await OnResourceGet(_client, reflectionId.Name, reflectionId.Namespace); await Reflect(item, target); } catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound) { _logger.LogWarning( "Could not reflect {@sourceId} to {@targetId}. Reflection {@targetId} not found.", id, reflectionId, reflectionId); _reflections.TryRemove(reflectionId, out _); } }
protected override async Task <V1Secret> OnResourceAutoReflect(IKubernetes client, V1Secret item, string ns) { return(await client.CreateNamespacedSecretAsync(new V1Secret { ApiVersion = item.ApiVersion, Kind = item.Kind, Type = item.Type, Data = item.Data, Metadata = new V1ObjectMeta { Name = item.Metadata.Name, NamespaceProperty = ns, Annotations = new Dictionary <string, string> { [Annotations.Reflection.AutoReflects] = KubernetesObjectId.For(item.Metadata).ToString(), [Annotations.Reflection.Reflects] = KubernetesObjectId.For(item.Metadata).ToString(), [Annotations.Reflection.ReflectedVersion] = item.Metadata.ResourceVersion, [Annotations.Reflection.ReflectedAt] = JsonConvert.SerializeObject(DateTimeOffset.UtcNow) } } }, ns)); }
public async Task Handle(InternalCertificateWatcherEvent notification, CancellationToken cancellationToken) { if (notification.Type != WatchEventType.Added && notification.Type != WatchEventType.Modified) { return; } if (notification.Item.Metadata == null) { return; } if (notification.Item.Spec == null) { return; } var certificate = notification.Item; var certificateId = KubernetesObjectId.For(certificate.Metadata); var secretId = new KubernetesObjectId(certificateId.Namespace, certificate.Spec.SecretName); _logger.LogDebug("{kind} {@certificateId} has secret {@secretId}", certificate.Kind, certificateId, secretId); V1Secret secret = null; try { secret = await _client.ReadNamespacedSecretAsync(certificate.Spec.SecretName, certificate.Metadata.NamespaceProperty, cancellationToken : cancellationToken); } catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound) { _logger.LogDebug("Could not find matching secret {@secretId}", secretId); } if (secret != null) { await Annotate(secret, certificate); } }
private async Task <bool> ReflectSourceToSelf(TResource item) { var metadata = item.Metadata(); var id = KubernetesObjectId.For(metadata); var sourceId = metadata.Reflects(); if (sourceId.Equals(KubernetesObjectId.Empty)) { _reflections.TryRemove(id, out _); } else { _reflections[id] = sourceId; TResource source = null; try { source = await OnResourceGet(_client, sourceId.Name, sourceId.Namespace); } catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound) { _logger.LogWarning("{kind} {@id} cannot be reflected." + " Source {@sourceId} could not be found", item.Kind, id, sourceId); } if (source != null) { await Reflect(source, item); } return(true); } return(false); }
private async Task OnEvent(WatcherEvent <V1Secret> e) { if (e.Item.Type.StartsWith("helm.sh")) { return; } var secretId = KubernetesObjectId.For(e.Item.Metadata()); var item = e.Item; _logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, secretId); if (e.Type != WatchEventType.Added && e.Type != WatchEventType.Modified) { return; } if (!e.Item.Metadata.ReflectionAllowed() || !e.Item.Metadata.UbiquitiReflectionEnabled()) { return; } if (!e.Item.Type.Equals("kubernetes.io/tls", StringComparison.InvariantCultureIgnoreCase)) { return; } _logger.LogDebug("Ubiquiti enabled using host secret {secretId}.", secretId); var tlsCrt = Encoding.Default.GetString(item.Data["tls.crt"]); var tlsKey = Encoding.Default.GetString(item.Data["tls.key"]); var hostSecretIds = item.Metadata.UbiquitiReflectionHosts() .Select(s => new KubernetesObjectId(s)) .ToList(); foreach (var hostSecretId in hostSecretIds) { _logger.LogDebug( "Reflecting {secretId} to Ubiquiti device using host secret {hostSecretId}.", secretId, hostSecretId, hostSecretId); string hostAddress; string username; string password; try { var hostSecret = await _apiClient.ReadNamespacedSecretAsync(hostSecretId.Name, string.IsNullOrWhiteSpace(hostSecretId.Namespace) ?e.Item.Metadata.NamespaceProperty : hostSecretId.Namespace); if (hostSecret.Data is null || !hostSecret.Data.Keys.Any()) { _logger.LogWarning("Cannot reflect {secretId} to {hostSecretId}. " + "Host secret {hostSecretId} has no data.", secretId, hostSecretId, hostSecretId); continue; } hostAddress = hostSecret.Data.ContainsKey("host") ? Encoding.Default.GetString(hostSecret.Data["host"]) : null; username = hostSecret.Data.ContainsKey("username") ? Encoding.Default.GetString(hostSecret.Data["username"]) : null; password = hostSecret.Data.ContainsKey("password") ? Encoding.Default.GetString(hostSecret.Data["password"]) : null; } catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound) { _logger.LogWarning( "Cannot reflect {secretId} to {hostSecretId}. Host secret {hostSecretId} not found.", secretId, hostSecretId, hostSecretId); continue; } if (string.IsNullOrWhiteSpace(hostAddress) || string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password)) { _logger.LogWarning( "Cannot reflect {secretId} to Ubiquiti device using host secret {hostSecretId}. " + "Host secret {hostSecretId} must contain 'host', 'username' and 'password' values.", secretId, hostSecretId, hostSecretId); continue; } var hostParts = hostAddress.Split(new[] { ":" }, StringSplitOptions.RemoveEmptyEntries) .Select(s => s.Trim()) .Where(s => !string.IsNullOrWhiteSpace(s)).ToList(); if (!hostParts.Any() || hostParts.Count > 2) { _logger.LogWarning( "Cannot reflect {secretId} to Ubiquiti device using host secret {hostSecretId}. " + "Host secret {hostSecretId} contains invalid 'host' data. " + "'host' must be in the format 'host:port' where port is optional.", secretId, hostSecretId, hostSecretId); } var host = hostParts.First(); var port = hostParts.Count < 2 ? 22 : int.TryParse(hostParts.Last(), out var p) ? p : 22; // SSH using var client = new SshClient(host, port, username, password); client.ErrorOccurred += delegate(object sender, ExceptionEventArgs exceptionEventArgs) { _logger.LogError(exceptionEventArgs.Exception, "Cannot reflect {secretId} to Ubiquiti device using host secret {hostSecretId} due to exception.", secretId, hostSecretId); }; _logger.LogDebug("Connecting to Ubiquiti device at {host}", hostAddress); client.Connect(); _logger.LogDebug("Check certificate on Ubiquiti device at {host}", hostAddress); var catCommand = client.RunCommand("cat /etc/ssl/private/cloudkey.crt"); if (catCommand.Result.Contains(tlsCrt)) { _logger.LogDebug( "Skip reflecting {secretId} to Ubiquiti device using host secret {hostSecretId}. Already exists.", secretId, hostSecretId); return; } _logger.LogDebug("Configuring new Let's Encrypt certs on Ubiquiti device at {host}", hostAddress); client.RunCommand($"echo \"{tlsCrt}\" > /etc/ssl/private/cloudkey.crt"); client.RunCommand($"echo \"{tlsKey}\" > /etc/ssl/private/cloudkey.key"); client.RunCommand( "rm -f /etc/ssl/private/cert.tar /etc/ssl/private/unifi.keystore.jks /etc/ssl/private/ssl-cert-snakeoil.key /etc/ssl/private/fullchain.pem"); client.RunCommand( "openssl pkcs12 -export -in /etc/ssl/private/cloudkey.crt -inkey /etc/ssl/private/cloudkey.key -out /etc/ssl/private/cloudkey.p12 -name unifi -password pass:aircontrolenterprise"); client.RunCommand( "keytool -importkeystore -deststorepass aircontrolenterprise -destkeypass aircontrolenterprise -destkeystore /usr/lib/unifi/data/keystore -srckeystore /etc/ssl/private/cloudkey.p12 -srcstoretype PKCS12 -srcstorepass aircontrolenterprise -alias unifi"); client.RunCommand("rm -f /etc/ssl/private/cloudkey.p12"); client.RunCommand("tar -cvf /etc/ssl/private/cert.tar /etc/ssl/private/*"); client.RunCommand("chown root:ssl-cert /etc/ssl/private/*"); client.RunCommand("chmod 640 /etc/ssl/private/*"); _logger.LogDebug("Restarting on Ubiquiti device at {host}", hostAddress); client.RunCommand("systemctl restart nginx; systemctl restart unifi"); client.Disconnect(); _logger.LogInformation("Reflected {secretId} to Ubiquiti device using host secret {hostSecretId}.", secretId, hostSecretId); } }
private async Task Annotate(V1Secret secret, Certificate certificate) { var secretId = KubernetesObjectId.For(secret.Metadata); var certificateId = KubernetesObjectId.For(certificate.Metadata); var secretAnnotations = new Dictionary <string, string>(secret.Metadata.Annotations ?? new Dictionary <string, string>()); var original = secretAnnotations.ToDictionary(s => s.Key, s => s.Value); var certAnnotations = new Dictionary <string, string>(certificate.Metadata.Annotations ?? new Dictionary <string, string>()); void MatchAnnotations(Dictionary <string, string> pairs) { foreach (var pair in pairs) { if (certAnnotations.TryGetValue(pair.Key, out var value)) { secretAnnotations[pair.Value] = value; } else { secretAnnotations.Remove(pair.Value); } } } MatchAnnotations(new Dictionary <string, string> { { Annotations.CertManagerCertificate.SecretReflectionAllowed, Annotations.Reflection.Allowed }, { Annotations.CertManagerCertificate.SecretReflectionAllowedNamespaces, Annotations.Reflection.AllowedNamespaces }, { Annotations.CertManagerCertificate.SecretReflectionAutoEnabled, Annotations.Reflection.AutoEnabled }, { Annotations.CertManagerCertificate.SecretReflectionAutoNamespaces, Annotations.Reflection.AutoNamespaces }, { Annotations.CertManagerCertificate.SecretFortiEnabled, Annotations.Reflection.FortiEnabled }, { Annotations.CertManagerCertificate.SecretFortiHosts, Annotations.Reflection.FortiHosts }, { Annotations.CertManagerCertificate.SecretFortiCertificate, Annotations.Reflection.FortiCertificate } }); if (secretAnnotations.Count == original.Count && secretAnnotations.Keys.All(s => original.ContainsKey(s)) && secretAnnotations.All(s => original[s.Key] == s.Value)) { _logger.LogDebug( "{secretKind} {@secretId} matches {certificateKind} {@certificateId} annotations", secret.Kind, secretId, certificate.Kind, certificateId, Annotations.Prefix); return; } _logger.LogTrace( "Patching {secretKind} {@secretId} to match {certificateKind} {@certificateId} {reflectorPrefix} annotations", secret.Kind, secretId, certificate.Kind, certificateId, Annotations.Prefix); var patch = new JsonPatchDocument <V1Secret>(); patch.Replace(e => e.Metadata.Annotations, secretAnnotations); await _client.PatchNamespacedSecretWithHttpMessagesAsync(new V1Patch(patch), secret.Metadata.Name, secret.Metadata.NamespaceProperty); _logger.LogInformation( "Patched {secretKind} {@secretId} to match {certificateKind} {@certificateId} {reflectorPrefix} annotations", secret.Kind, secretId, certificate.Kind, certificateId, Annotations.Prefix); }
private async Task OnEvent(WatcherEvent <V1Secret> e) { if (e.Item.Type.StartsWith("helm.sh")) { return; } var secretId = KubernetesObjectId.For(e.Item.Metadata()); var item = e.Item; _logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, secretId); if (e.Type != WatchEventType.Added && e.Type != WatchEventType.Modified) { return; } if (!e.Item.Metadata.ReflectionAllowed() || !e.Item.Metadata.VMwareReflectionEnabled()) { return; } if (!e.Item.Type.Equals("kubernetes.io/tls", StringComparison.InvariantCultureIgnoreCase)) { return; } _logger.LogDebug("VMware enabled using host secret {secretId}.", secretId); var tlsCrt = Encoding.Default.GetString(item.Data["tls.crt"]); var tlsKey = Encoding.Default.GetString(item.Data["tls.key"]); var hostSecretIds = item.Metadata.VMwareReflectionHosts() .Select(s => new KubernetesObjectId(s)) .ToList(); foreach (var hostSecretId in hostSecretIds) { _logger.LogDebug( "Reflecting {secretId} to VMware ESXi device using host secret {hostSecretId}.", secretId, hostSecretId, hostSecretId); string hostAddress; string username; string password; try { var hostSecret = await _apiClient.ReadNamespacedSecretAsync(hostSecretId.Name, string.IsNullOrWhiteSpace(hostSecretId.Namespace) ?e.Item.Metadata.NamespaceProperty : hostSecretId.Namespace); if (hostSecret.Data is null || !hostSecret.Data.Keys.Any()) { _logger.LogWarning("Cannot reflect {secretId} to {hostSecretId}. " + "Host secret {hostSecretId} has no data.", secretId, hostSecretId, hostSecretId); continue; } hostAddress = hostSecret.Data.ContainsKey("host") ? Encoding.Default.GetString(hostSecret.Data["host"]) : null; username = hostSecret.Data.ContainsKey("username") ? Encoding.Default.GetString(hostSecret.Data["username"]) : null; password = hostSecret.Data.ContainsKey("password") ? Encoding.Default.GetString(hostSecret.Data["password"]) : null; } catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound) { _logger.LogWarning( "Cannot reflect {secretId} to {hostSecretId}. Host secret {hostSecretId} not found.", secretId, hostSecretId, hostSecretId); continue; } if (string.IsNullOrWhiteSpace(hostAddress) || string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password)) { _logger.LogWarning( "Cannot reflect {secretId} to VMware ESXi device using host secret {hostSecretId}. " + "Host secret {hostSecretId} must contain 'host', 'username' and 'password' values.", secretId, hostSecretId, hostSecretId); continue; } var hostParts = hostAddress.Split(new[] { ":" }, StringSplitOptions.RemoveEmptyEntries) .Select(s => s.Trim()) .Where(s => !string.IsNullOrWhiteSpace(s)).ToList(); if (!hostParts.Any() || hostParts.Count > 2) { _logger.LogWarning( "Cannot reflect {secretId} to VMware ESXi device using host secret {hostSecretId}. " + "Host secret {hostSecretId} contains invalid 'host' data. " + "'host' must be in the format 'host:port' where port is optional.", secretId, hostSecretId, hostSecretId); } var host = hostParts.First(); var port = hostParts.Count < 2 ? 22 : int.TryParse(hostParts.Last(), out var p) ? p : 22; // SSH void HandleKeyEvent(object sender, AuthenticationPromptEventArgs eventArgs) { foreach (var prompt in eventArgs.Prompts) { if (prompt.Request.IndexOf("Password:"******"Cannot reflect {secretId} to VMware ESXi device using host secret {hostSecretId} due to exception.", secretId, hostSecretId); }; _logger.LogDebug("Connecting to VMware ESXi device at {host}", hostAddress); client.Connect(); _logger.LogDebug("Check certificate on VMware ESXi device at {host}", hostAddress); var catCommand = client.RunCommand("cat /etc/vmware/ssl/rui.crt"); if (catCommand.Result.Contains(tlsCrt)) { _logger.LogDebug( "Skip reflecting {secretId} to VMware ESXi device using host secret {hostSecretId}. Already exists.", secretId, hostSecretId); return; } _logger.LogDebug("Configuring new Let's Encrypt certs on VMware ESXi device at {host}", hostAddress); client.RunCommand($"echo \"{tlsCrt}\" > /etc/vmware/ssl/rui.crt"); client.RunCommand($"echo \"{tlsKey}\" > /etc/vmware/ssl/rui.key"); _logger.LogDebug("Restarting on VMware ESXi device at {host}", hostAddress); client.RunCommand("services.sh restart"); client.Disconnect(); _logger.LogInformation("Reflected {secretId} to VMware ESXi device using host secret {hostSecretId}.", secretId, hostSecretId); } }
private async Task OnEvent(WatcherEvent <V1Secret> e) { var id = KubernetesObjectId.For(e.Item.Metadata()); var item = e.Item; _logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, id); if (e.Type != WatchEventType.Added && e.Type != WatchEventType.Modified) { return; } if (!e.Item.Metadata.ReflectionAllowed() || !e.Item.Metadata.FortiReflectionEnabled()) { return; } if (!e.Item.Type.Equals("kubernetes.io/tls", StringComparison.InvariantCultureIgnoreCase)) { return; } var caCrt = Encoding.Default.GetString(item.Data["ca.crt"]); var tlsCrt = Encoding.Default.GetString(item.Data["tls.crt"]); var tlsKey = Encoding.Default.GetString(item.Data["tls.key"]); var tlsCerts = tlsCrt.Split(new[] { "-----END CERTIFICATE-----" }, StringSplitOptions.RemoveEmptyEntries) .Select(s => s.TrimStart()) .Where(s => !string.IsNullOrWhiteSpace(s)) .Select(s => $"{s}-----END CERTIFICATE-----") .ToList(); var hostSecretIds = item.Metadata.FortiReflectionHosts().Select(s => new KubernetesObjectId(s)).ToList(); var fortiCertName = item.Metadata.FortiCertificate(); var fortiCertId = !string.IsNullOrWhiteSpace(fortiCertName) ? fortiCertName : item.Metadata.Name.Substring(0, Math.Min(item.Metadata.Name.Length, 30)); foreach (var hostSecretId in hostSecretIds) { _logger.LogDebug( "Reflecting {secretId} to FortiOS device using host secret {hostSecretId}.", id, hostSecretId, hostSecretId); string fortiHost; string fortiUsername; string fortiPassword; try { var hostSecret = await _apiClient.ReadNamespacedSecretAsync(hostSecretId.Name, hostSecretId.Namespace); if (hostSecret.Data is null || !hostSecret.Data.Keys.Any()) { _logger.LogWarning("Cannot reflect {secretId} to {hostSecretId}. " + "Host secret {hostSecretId} has no data.", id, hostSecretId, hostSecretId); continue; } fortiHost = hostSecret.Data.ContainsKey("host") ? Encoding.Default.GetString(hostSecret.Data["host"]) : null; fortiUsername = hostSecret.Data.ContainsKey("username") ? Encoding.Default.GetString(hostSecret.Data["username"]) : null; fortiPassword = hostSecret.Data.ContainsKey("password") ? Encoding.Default.GetString(hostSecret.Data["password"]) : null; } catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound) { _logger.LogWarning("Cannot reflect {secretId} to {hostSecretId}. Host secret {hostSecretId} not found.", id, hostSecretId, hostSecretId); continue; } if (string.IsNullOrWhiteSpace(fortiHost) || string.IsNullOrWhiteSpace(fortiUsername) || string.IsNullOrWhiteSpace(fortiPassword)) { _logger.LogWarning( "Cannot reflect {secretId} to FortiOS device using host secret {hostSecretId}. " + "Host secret {hostSecretId} must contain 'host', 'username' and 'password' values.", id, hostSecretId, hostSecretId); continue; } var hostParts = fortiHost.Split(new[] { ":" }, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()) .Where(s => !string.IsNullOrWhiteSpace(s)).ToList(); if (!hostParts.Any() || hostParts.Count > 2) { _logger.LogWarning( "Cannot reflect {secretId} to FortiOS device using host secret {hostSecretId}. " + "Host secret {hostSecretId} contains invalid 'host' data. " + "'host' must be in the format 'host:port' where port is optional.", id, hostSecretId, hostSecretId); } var hostPort = hostParts.Count < 2 ? 22 : int.TryParse(hostParts.Last(), out var port) ? port : 22; using var client = new SshClient(hostParts.First(), hostPort, fortiUsername, fortiPassword) { ConnectionInfo = { Timeout = TimeSpan.FromSeconds(10) } }; try { _logger.LogDebug("Connecting for FortiOS device at {host}", fortiHost); client.Connect(); } catch (Exception exception) { _logger.LogError(exception, "Cannot reflect {secretId} to FortiOS device using host secret {hostSecretId} due to exception.", id, hostSecretId); continue; } _logger.LogDebug("Checking for certificate {certId} on FortiOS device {host}", fortiCertId, fortiHost); string checkOutput; await using (var shell = client.CreateShellStream(nameof(FortiMirror), 128, 1024, 800, 600, 1024)) { var lastLine = string.Empty; var outputBuilder = new StringBuilder(); shell.WriteLine("config vpn certificate local"); shell.WriteLine($"edit {fortiCertId}"); shell.WriteLine("show full"); shell.WriteLine("end"); while (!(lastLine.Contains("#") && lastLine.Contains("end"))) { lastLine = shell.ReadLine(); outputBuilder.AppendLine(lastLine); } shell.Close(); checkOutput = outputBuilder.ToString(); } var localCertificateUpToDate = true; if (!string.IsNullOrWhiteSpace(tlsKey)) { if (checkOutput.Contains("unset private-key")) { localCertificateUpToDate = false; } } if (!checkOutput.Contains("set certificate", StringComparison.InvariantCultureIgnoreCase)) { localCertificateUpToDate = false; } else { var remoteCertificate = checkOutput.Substring(checkOutput.IndexOf("set certificate \"", StringComparison.InvariantCultureIgnoreCase)); remoteCertificate = remoteCertificate.Replace("set certificate \"", string.Empty, StringComparison.InvariantCultureIgnoreCase); remoteCertificate = remoteCertificate.Substring(0, remoteCertificate.IndexOf("\"", StringComparison.InvariantCultureIgnoreCase)); remoteCertificate = remoteCertificate.Replace("\r", string.Empty); if (!tlsCerts.Select(s => s.Replace("\r", string.Empty)).Contains(remoteCertificate)) { localCertificateUpToDate = false; } } var success = true; if (!localCertificateUpToDate) { var commandBuilder = new StringBuilder(); commandBuilder.AppendLine("config vpn certificate local"); commandBuilder.AppendLine($"edit {fortiCertId}"); commandBuilder.AppendLine($"set private-key \"{tlsKey}\""); commandBuilder.AppendLine($"set certificate \"{tlsCrt}\""); commandBuilder.AppendLine("end"); var command = client.RunCommand(commandBuilder.ToString()); if (command.ExitStatus != 0 || !string.IsNullOrWhiteSpace(command.Error)) { _logger.LogWarning( "Checking for certificate {certId} could not be installed on FortiOS device {host} due to error: {error}", id, fortiHost, command.Error); success = false; } } if (!localCertificateUpToDate && success) { var caCerts = tlsCerts.ToList(); if (!string.IsNullOrWhiteSpace(caCrt)) { caCerts.Add(caCrt); } var caId = 0; for (var i = 0; i < caCerts.Count; i++) { _logger.LogDebug("Installing CA certificate {index} for {certId} on FortiOS device {host}", i + 1, id, fortiHost); var commandBuilder = new StringBuilder(); commandBuilder.AppendLine("config vpn certificate ca"); commandBuilder.AppendLine($"edit {fortiCertId}_CA{(caId == 0 ? string.Empty : caId.ToString())}"); commandBuilder.AppendLine($"set ca \"{tlsCerts[i]}\""); commandBuilder.AppendLine("end"); var command = client.RunCommand(commandBuilder.ToString()); if (command.ExitStatus == 0 && string.IsNullOrWhiteSpace(command.Error)) { caId++; continue; } if (command.Error.Contains("This CA certificate is duplicated.")) { _logger.LogWarning("Skipping CA certificate {index} since it is duplicated by another certificate.", i + i); } else if (command.Error.Contains("Input is not a valid CA certificate.")) { _logger.LogDebug("Skipping CA certificate {index} since it is not a valid CA certificate.", i + i); } else { _logger.LogWarning("Could not install CA {index} certificate due to error: {response}", i + 1, command.Result); success = false; } } } if (!success) { _logger.LogError("Reflecting {secretId} to FortiOS device using host secret {hostSecretId} completed with errors.", id, hostSecretId); } else if (!localCertificateUpToDate) { _logger.LogInformation("Reflected {secretId} to FortiOS device using host secret {hostSecretId}.", id, hostSecretId); } } }
private async Task OnEvent(WatcherEvent <V1Secret> e) { if (e.Item.Type.StartsWith("helm.sh")) { return; } var secretId = KubernetesObjectId.For(e.Item.Metadata()); var item = e.Item; _logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, secretId); if (e.Type != WatchEventType.Added && e.Type != WatchEventType.Modified) { return; } if (!e.Item.Metadata.ReflectionAllowed() || !e.Item.Metadata.FreeNasReflectionEnabled()) { return; } if (!e.Item.Type.Equals("kubernetes.io/tls", StringComparison.InvariantCultureIgnoreCase)) { return; } _logger.LogDebug("FreeNas enabled using host secret {secretId}.", secretId); var tlsCrt = Encoding.Default.GetString(item.Data["tls.crt"]); var tlsKey = Encoding.Default.GetString(item.Data["tls.key"]); var hostSecretIds = item.Metadata.FreeNasReflectionHosts() .Select(s => new KubernetesObjectId(s)) .ToList(); var certName = item.Metadata.FreeNasCertificate(); foreach (var hostSecretId in hostSecretIds) { _logger.LogDebug( "Reflecting {secretId} to FreeNas device using host secret {hostSecretId}.", secretId, hostSecretId, hostSecretId); string hostAddress; string username; string password; try { var hostSecret = await _apiClient.ReadNamespacedSecretAsync(hostSecretId.Name, string.IsNullOrWhiteSpace(hostSecretId.Namespace) ?e.Item.Metadata.NamespaceProperty : hostSecretId.Namespace); if (hostSecret.Data is null || !hostSecret.Data.Keys.Any()) { _logger.LogWarning("Cannot reflect {secretId} to {hostSecretId}. " + "Host secret {hostSecretId} has no data.", secretId, hostSecretId, hostSecretId); continue; } hostAddress = hostSecret.Data.ContainsKey("host") ? Encoding.Default.GetString(hostSecret.Data["host"]) : null; username = hostSecret.Data.ContainsKey("username") ? Encoding.Default.GetString(hostSecret.Data["username"]) : null; password = hostSecret.Data.ContainsKey("password") ? Encoding.Default.GetString(hostSecret.Data["password"]) : null; } catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound) { _logger.LogWarning( "Cannot reflect {secretId} to {hostSecretId}. Host secret {hostSecretId} not found.", secretId, hostSecretId, hostSecretId); continue; } if (string.IsNullOrWhiteSpace(hostAddress) || string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password)) { _logger.LogWarning( "Cannot reflect {secretId} to FreeNas device using host secret {hostSecretId}. " + "Host secret {hostSecretId} must contain 'host', 'username' and 'password' values.", secretId, hostSecretId, hostSecretId); continue; } var hostParts = hostAddress.Split(new[] { ":" }, StringSplitOptions.RemoveEmptyEntries) .Select(s => s.Trim()) .Where(s => !string.IsNullOrWhiteSpace(s)).ToList(); if (!hostParts.Any() || hostParts.Count > 2) { _logger.LogWarning( "Cannot reflect {secretId} to FreeNas device using host secret {hostSecretId}. " + "Host secret {hostSecretId} contains invalid 'host' data. " + "'host' must be in the format 'host:port' where port is optional.", secretId, hostSecretId, hostSecretId); } var host = hostParts.First(); // Check if certificate is the same var client = _clientFactory.CreateClient(); client.BaseAddress = new Uri($"https://{host}/api/v2.0/"); client.DefaultRequestHeaders.Clear(); client.DefaultRequestHeaders.ConnectionClose = true; client.DefaultRequestHeaders .Accept .Add(new MediaTypeWithQualityHeaderValue("application/json")); //ACCEPT header client.DefaultRequestHeaders.Add("User-Agent", "Emberstack/Reflector"); var authenticationString = $"{username}:{password}"; var base64EncodedAuthenticationString = Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationString)); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodedAuthenticationString); var response = await client.GetAsync(CertificateUri); var options = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; await using var responseStream = await response.Content.ReadAsStreamAsync(); using var responseStr = response.Content.ReadAsStringAsync(); var certificates = await JsonSerializer.DeserializeAsync <List <FreeNasCertificate> >(responseStream, options); // var bytes = Encoding.ASCII.GetBytes(tlsCrt); // var x509Certificate2 = new X509Certificate2(bytes); // // var name = $"{certName}-{x509Certificate2.NotAfter:s}"; var name = certName; var cert = certificates .SingleOrDefault(x => x.Name == name); var certExists = !(cert is null); if (certExists) { if (tlsCrt.Contains(cert.Certificate)) { _logger.LogDebug( "Skip reflecting {secretId} to FreeNas device using host secret {hostSecretId}. Already exists.", secretId, hostSecretId); return; } } // Create the certificate var bodyCreate = JsonSerializer.Serialize(new FreeNasCertificateCreateImported { Name = name, Certificate = tlsCrt, Privatekey = tlsKey }, options); await client.PostAsync(CertificateUri, new StringContent(bodyCreate)); // Set the certificate as default var certId = certExists ? cert.Id : certificates.Single(x => x.Name == name).Id; var bodyGeneral = JsonSerializer.Serialize(new FreeNasSystemGeneral { UiCertificate = certId }, options); await client.PutAsync("system/general/", new StringContent(bodyGeneral)); _logger.LogInformation("Reflected {secretId} to FreeNas device using host secret {hostSecretId}.", secretId, hostSecretId); } }