private async Task CheckAutoReflections(TResource item)
{
var metadata = item.Metadata();
var id = KubernetesObjectId.For(metadata);
var namespaces = new List <string>();
if (metadata.ReflectionAllowed() && metadata.ReflectionAutoEnabled())
{
_autoReflections[id] = metadata.ReflectionAutoNamespaces();
var namespaceList = await _client.ListNamespaceAsync();
namespaces = namespaceList.Items
.Select(s => s.Metadata.Name)
.Where(s => metadata.ReflectionAutoNamespacesMatch(s) &&
metadata.ReflectionAllowedNamespacesMatch(s))
.ToList();
}
else
{
_autoReflections.TryRemove(id, out _);
namespaces.Clear();
}
namespaces.Remove(metadata.NamespaceProperty);
await UpdateAutoReflections(item, namespaces);
}
private async Task OnEvent(WatcherEvent e)
{
var id = KubernetesObjectId.For(e.Item.Metadata());
_logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, id);
await _mediator.Publish(e);
}
private async Task OnEventHandlingError(WatcherEvent e, Exception ex)
{
var id = KubernetesObjectId.For(e.Item.Metadata());
if (ex is Microsoft.Rest.HttpOperationException httpEx)
{
_logger.LogError($"Microsoft.Rest.HttpOperationException response: {httpEx.Response.Content}");
}
_logger.LogError(ex, "Failed to process {eventType} {kind} {@id} due to exception",
e.Type, e.Item.Kind, id);
await _secretsWatcher.Stop();
foreach (var certificatesWatcher in _certificatesWatchers.Values)
{
await certificatesWatcher.Stop();
}
_eventQueue.Clear();
_logger.LogTrace("Watchers restarting");
await _secretsWatcher.Start();
foreach (var certificatesWatcher in _certificatesWatchers.Values)
{
await certificatesWatcher.Start();
}
_logger.LogTrace("Watchers restarted");
}
public async Task Handle(InternalSecretWatcherEvent notification, CancellationToken cancellationToken)
{
if (notification.Type != WatchEventType.Added && notification.Type != WatchEventType.Modified)
{
return;
}
var secret = notification.Item;
var secretId = KubernetesObjectId.For(secret.Metadata);
var metadata = secret.Metadata;
if (metadata.Labels == null)
{
return;
}
if (metadata.Labels.TryGetValue(CertManagerConstants.CertificateNameLabel, out var certificateName))
{
var certificateId = new KubernetesObjectId(secretId.Namespace, certificateName);
_logger.LogDebug("{kind} {@certificateId} has secret {@secretId}",
CertManagerConstants.CertificateKind, certificateId, secretId);
Certificate certificate = null;
foreach (var certificateResourceDefinitionVersion in notification.CertificateResourceDefinitionVersions)
{
try
{
var certificateJObject = await _client.GetNamespacedCustomObjectAsync(
CertManagerConstants.CrdGroup,
certificateResourceDefinitionVersion, metadata.NamespaceProperty,
CertManagerConstants.CertificatePlural,
certificateName, cancellationToken);
certificate = ((JObject)certificateJObject).ToObject <Certificate>();
}
catch (HttpOperationException exception) when(exception.Response.StatusCode ==
HttpStatusCode.NotFound)
{
}
}
if (certificate != null)
{
await Annotate(secret, certificate);
}
else
{
_logger.LogDebug("Could not find {kind} {@id}", CertManagerConstants.CertificateKind,
certificateId);
}
}
}
protected async Task OnResourceWatcherEvent(WatcherEvent <TResource> request)
{
var item = request.Item;
var eventType = request.Type;
var metadata = item.Metadata();
var id = KubernetesObjectId.For(metadata);
if (await OnResourceIgnoreCheck(item))
{
return;
}
switch (eventType)
{
case WatchEventType.Added:
case WatchEventType.Modified:
{
//Check if the source for auto reflection is still valid.
if (await CheckAutoReflectionSource(item))
{
return;
}
//Ensure auto reflections
await CheckAutoReflections(item);
//Update current item. If updated, return (Update will be picked up by watcher)
if (await ReflectSourceToSelf(item))
{
return;
}
//Update all child reflections
await ReflectSelfToReflections(item);
}
break;
case WatchEventType.Deleted:
{
_reflections.TryRemove(id, out _);
_autoReflections.TryRemove(id, out _);
//Remove all auto-reflections
await UpdateAutoReflections(item, new List <string>());
}
break;
case WatchEventType.Error: break;
default: throw new ArgumentOutOfRangeException();
}
}
private async Task OnEventHandlingError(WatcherEvent e, Exception ex)
{
var id = KubernetesObjectId.For(e.Item.Metadata());
_logger.LogError(ex, "Failed to process {eventType} {kind} {@id} due to exception",
e.Type, e.Item.Kind, id);
await WatchersStop();
_eventQueue.Clear();
_logger.LogInformation("Restarting watchers ");
await WatchersStart();
}
private async Task UpdateAutoReflections(TResource item, List <string> namespaces)
{
var metadata = item.Metadata();
var id = KubernetesObjectId.For(metadata);
var autoReflections = await FindAutoReflections(id);
var reflectionsToCreate = namespaces
.Where(s => !autoReflections.Any(m => m.NamespaceProperty.Equals(s))).ToList();
var reflectionsToRemove = autoReflections
.Where(s => !namespaces.Contains(s.NamespaceProperty)).ToList();
foreach (var reflectionNamespace in reflectionsToCreate)
{
var reflectionId = new KubernetesObjectId(reflectionNamespace, id.Name);
try
{
await OnResourceAutoReflect(_client, item, reflectionNamespace);
_logger.LogInformation(
"Created reflection {kind} for {@id} in namespace {namespace}.",
item.Kind, id, reflectionId.Namespace);
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.Conflict)
{
_logger.LogWarning(
"Cannot create reflection {kind} {@reflectionId} for {@id}. " +
$"Found conflicting {{kind}} with the same name which was not created by {nameof(Reflector)}",
item.Kind, reflectionId, id, item.Kind);
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.Forbidden)
{
_logger.LogWarning(
"Cannot create reflection {kind} {@reflectionId} for {@id}. " +
"Access to namespace forbidden.",
item.Kind, reflectionId, id, item.Kind);
}
}
foreach (var reflection in reflectionsToRemove)
{
await OnResourceDelete(_client, reflection.Name, reflection.NamespaceProperty);
_reflections.TryRemove(KubernetesObjectId.For(reflection), out _);
_logger.LogInformation(
"Deleted reflection {kind} for {@id} in namespace {targetNamespace}",
item.Kind, id, reflection.NamespaceProperty);
}
}
public static KubernetesObjectId AutoReflects(this V1ObjectMeta metadata)
{
if (metadata.SafeAnnotations().TryGetValue(Annotations.Reflection.AutoReflects, out var raw) &&
KubernetesObjectId.TryParse(raw, out var value))
{
if (value.Namespace == null)
{
return(new KubernetesObjectId(metadata.NamespaceProperty, value.Name));
}
return(value);
}
return(KubernetesObjectId.Empty);
}
private async Task OnEventHandlingError(WatcherEvent <V1Secret> e, Exception ex)
{
var id = KubernetesObjectId.For(e.Item.Metadata());
_logger.LogError(ex, "Failed to process {eventType} {kind} {@id} due to exception",
e.Type, e.Item.Kind, id);
await _secretsWatcher.Stop();
_eventQueue.Clear();
_logger.LogTrace("Watchers restarting");
await _secretsWatcher.Start();
_logger.LogTrace("Watchers restarted");
}
private async Task OnEvent(WatcherEvent e)
{
var id = KubernetesObjectId.For(e.Item.Metadata());
_logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, id);
switch (e)
{
case WatcherEvent <TResource> resourceEvent:
await OnResourceWatcherEvent(resourceEvent);
break;
case WatcherEvent <V1Namespace> namespaceEvent:
await OnNamespaceWatcherEvent(namespaceEvent);
break;
}
}
private async Task OnEventHandlingError(WatcherEvent e, Exception ex)
{
var id = KubernetesObjectId.For(e.Item.Metadata());
if (ex is Microsoft.Rest.HttpOperationException httpEx)
{
_logger.LogError($"Microsoft.Rest.HttpOperationException response: {httpEx.Response.Content}");
}
_logger.LogError(ex, "Failed to process {eventType} {kind} {@id} due to exception",
e.Type, e.Item.Kind, id);
await WatchersStop();
_eventQueue.Clear();
_logger.LogInformation("Restarting watchers ");
await WatchersStart();
}
private async Task <bool> CheckAutoReflectionSource(TResource item)
{
var metadata = item.Metadata();
var id = KubernetesObjectId.For(metadata);
var autoReflectionSource = metadata.AutoReflects();
if (autoReflectionSource.Equals(KubernetesObjectId.Empty))
{
return(false);
}
var delete = false;
try
{
var source = await OnResourceGet(_client,
autoReflectionSource.Name, autoReflectionSource.Namespace);
var sourceMeta = source.Metadata();
if (!sourceMeta.ReflectionAllowed() ||
!sourceMeta.ReflectionAutoEnabled() ||
!sourceMeta.ReflectionAllowedNamespacesMatch(id.Namespace) ||
!sourceMeta.ReflectionAutoNamespacesMatch(id.Namespace))
{
delete = true;
}
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
delete = true;
}
if (!delete)
{
return(false);
}
await OnResourceDelete(_client, id.Name, id.Namespace);
_logger.LogInformation(
"Deleted reflection {kind} for {@id} in namespace {targetNamespace}",
item.Kind, autoReflectionSource, id.Namespace);
return(true);
}
private async Task ReflectSelfToReflections(TResource item)
{
var metadata = item.Metadata();
var id = KubernetesObjectId.For(metadata);
var reflections = _reflections.Where(s => s.Value.Equals(id)).Select(s => s.Key).ToList();
foreach (var reflectionId in reflections)
{
try
{
var target = await OnResourceGet(_client, reflectionId.Name, reflectionId.Namespace);
await Reflect(item, target);
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning(
"Could not reflect {@sourceId} to {@targetId}. Reflection {@targetId} not found.",
id, reflectionId, reflectionId);
_reflections.TryRemove(reflectionId, out _);
}
}
protected override async Task <V1Secret> OnResourceAutoReflect(IKubernetes client, V1Secret item, string ns)
{
return(await client.CreateNamespacedSecretAsync(new V1Secret
{
ApiVersion = item.ApiVersion,
Kind = item.Kind,
Type = item.Type,
Data = item.Data,
Metadata = new V1ObjectMeta
{
Name = item.Metadata.Name,
NamespaceProperty = ns,
Annotations = new Dictionary <string, string>
{
[Annotations.Reflection.AutoReflects] = KubernetesObjectId.For(item.Metadata).ToString(),
[Annotations.Reflection.Reflects] = KubernetesObjectId.For(item.Metadata).ToString(),
[Annotations.Reflection.ReflectedVersion] = item.Metadata.ResourceVersion,
[Annotations.Reflection.ReflectedAt] = JsonConvert.SerializeObject(DateTimeOffset.UtcNow)
}
}
}, ns));
}
public async Task Handle(InternalCertificateWatcherEvent notification, CancellationToken cancellationToken)
{
if (notification.Type != WatchEventType.Added && notification.Type != WatchEventType.Modified)
{
return;
}
if (notification.Item.Metadata == null)
{
return;
}
if (notification.Item.Spec == null)
{
return;
}
var certificate = notification.Item;
var certificateId = KubernetesObjectId.For(certificate.Metadata);
var secretId = new KubernetesObjectId(certificateId.Namespace, certificate.Spec.SecretName);
_logger.LogDebug("{kind} {@certificateId} has secret {@secretId}",
certificate.Kind, certificateId, secretId);
V1Secret secret = null;
try
{
secret = await _client.ReadNamespacedSecretAsync(certificate.Spec.SecretName,
certificate.Metadata.NamespaceProperty, cancellationToken : cancellationToken);
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogDebug("Could not find matching secret {@secretId}", secretId);
}
if (secret != null)
{
await Annotate(secret, certificate);
}
}
private async Task <bool> ReflectSourceToSelf(TResource item)
{
var metadata = item.Metadata();
var id = KubernetesObjectId.For(metadata);
var sourceId = metadata.Reflects();
if (sourceId.Equals(KubernetesObjectId.Empty))
{
_reflections.TryRemove(id, out _);
}
else
{
_reflections[id] = sourceId;
TResource source = null;
try
{
source = await OnResourceGet(_client, sourceId.Name, sourceId.Namespace);
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning("{kind} {@id} cannot be reflected." +
" Source {@sourceId} could not be found",
item.Kind, id, sourceId);
}
if (source != null)
{
await Reflect(source, item);
}
return(true);
}
return(false);
}
private async Task OnEvent(WatcherEvent <V1Secret> e)
{
var id = KubernetesObjectId.For(e.Item.Metadata());
var item = e.Item;
_logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, id);
if (e.Type != WatchEventType.Added && e.Type != WatchEventType.Modified)
{
return;
}
if (!e.Item.Metadata.ReflectionAllowed() || !e.Item.Metadata.FortiReflectionEnabled())
{
return;
}
if (!e.Item.Type.Equals("kubernetes.io/tls", StringComparison.InvariantCultureIgnoreCase))
{
return;
}
var caCrt = Encoding.Default.GetString(item.Data["ca.crt"]);
var tlsCrt = Encoding.Default.GetString(item.Data["tls.crt"]);
var tlsKey = Encoding.Default.GetString(item.Data["tls.key"]);
var tlsCerts = tlsCrt.Split(new[] { "-----END CERTIFICATE-----" }, StringSplitOptions.RemoveEmptyEntries)
.Select(s => s.TrimStart())
.Where(s => !string.IsNullOrWhiteSpace(s))
.Select(s => $"{s}-----END CERTIFICATE-----")
.ToList();
var hostSecretIds = item.Metadata.FortiReflectionHosts().Select(s => new KubernetesObjectId(s)).ToList();
var fortiCertName = item.Metadata.FortiCertificate();
var fortiCertId = !string.IsNullOrWhiteSpace(fortiCertName)
? fortiCertName
: item.Metadata.Name.Substring(0, Math.Min(item.Metadata.Name.Length, 30));
foreach (var hostSecretId in hostSecretIds)
{
_logger.LogDebug(
"Reflecting {secretId} to FortiOS device using host secret {hostSecretId}.",
id, hostSecretId, hostSecretId);
string fortiHost;
string fortiUsername;
string fortiPassword;
try
{
var hostSecret = await _apiClient.ReadNamespacedSecretAsync(hostSecretId.Name, hostSecretId.Namespace);
if (hostSecret.Data is null || !hostSecret.Data.Keys.Any())
{
_logger.LogWarning("Cannot reflect {secretId} to {hostSecretId}. " +
"Host secret {hostSecretId} has no data.",
id, hostSecretId, hostSecretId);
continue;
}
fortiHost = hostSecret.Data.ContainsKey("host")
? Encoding.Default.GetString(hostSecret.Data["host"])
: null;
fortiUsername = hostSecret.Data.ContainsKey("username")
? Encoding.Default.GetString(hostSecret.Data["username"])
: null;
fortiPassword = hostSecret.Data.ContainsKey("password")
? Encoding.Default.GetString(hostSecret.Data["password"])
: null;
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning("Cannot reflect {secretId} to {hostSecretId}. Host secret {hostSecretId} not found.",
id, hostSecretId, hostSecretId);
continue;
}
if (string.IsNullOrWhiteSpace(fortiHost) || string.IsNullOrWhiteSpace(fortiUsername) ||
string.IsNullOrWhiteSpace(fortiPassword))
{
_logger.LogWarning(
"Cannot reflect {secretId} to FortiOS device using host secret {hostSecretId}. " +
"Host secret {hostSecretId} must contain 'host', 'username' and 'password' values.",
id, hostSecretId, hostSecretId);
continue;
}
var hostParts = fortiHost.Split(new[] { ":" }, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim())
.Where(s => !string.IsNullOrWhiteSpace(s)).ToList();
if (!hostParts.Any() || hostParts.Count > 2)
{
_logger.LogWarning(
"Cannot reflect {secretId} to FortiOS device using host secret {hostSecretId}. " +
"Host secret {hostSecretId} contains invalid 'host' data. " +
"'host' must be in the format 'host:port' where port is optional.",
id, hostSecretId, hostSecretId);
}
var hostPort = hostParts.Count < 2 ? 22 : int.TryParse(hostParts.Last(), out var port) ? port : 22;
using var client = new SshClient(hostParts.First(), hostPort, fortiUsername, fortiPassword)
{
ConnectionInfo =
{
Timeout = TimeSpan.FromSeconds(10)
}
};
try
{
_logger.LogDebug("Connecting for FortiOS device at {host}", fortiHost);
client.Connect();
}
catch (Exception exception)
{
_logger.LogError(exception,
"Cannot reflect {secretId} to FortiOS device using host secret {hostSecretId} due to exception.",
id, hostSecretId);
continue;
}
_logger.LogDebug("Checking for certificate {certId} on FortiOS device {host}", fortiCertId, fortiHost);
string checkOutput;
await using (var shell = client.CreateShellStream(nameof(FortiMirror), 128, 1024, 800, 600, 1024))
{
var lastLine = string.Empty;
var outputBuilder = new StringBuilder();
shell.WriteLine("config vpn certificate local");
shell.WriteLine($"edit {fortiCertId}");
shell.WriteLine("show full");
shell.WriteLine("end");
while (!(lastLine.Contains("#") && lastLine.Contains("end")))
{
lastLine = shell.ReadLine();
outputBuilder.AppendLine(lastLine);
}
shell.Close();
checkOutput = outputBuilder.ToString();
}
var localCertificateUpToDate = true;
if (!string.IsNullOrWhiteSpace(tlsKey))
{
if (checkOutput.Contains("unset private-key"))
{
localCertificateUpToDate = false;
}
}
if (!checkOutput.Contains("set certificate", StringComparison.InvariantCultureIgnoreCase))
{
localCertificateUpToDate = false;
}
else
{
var remoteCertificate = checkOutput.Substring(checkOutput.IndexOf("set certificate \"",
StringComparison.InvariantCultureIgnoreCase));
remoteCertificate = remoteCertificate.Replace("set certificate \"", string.Empty,
StringComparison.InvariantCultureIgnoreCase);
remoteCertificate = remoteCertificate.Substring(0,
remoteCertificate.IndexOf("\"", StringComparison.InvariantCultureIgnoreCase));
remoteCertificate = remoteCertificate.Replace("\r", string.Empty);
if (!tlsCerts.Select(s => s.Replace("\r", string.Empty)).Contains(remoteCertificate))
{
localCertificateUpToDate = false;
}
}
var success = true;
if (!localCertificateUpToDate)
{
var commandBuilder = new StringBuilder();
commandBuilder.AppendLine("config vpn certificate local");
commandBuilder.AppendLine($"edit {fortiCertId}");
commandBuilder.AppendLine($"set private-key \"{tlsKey}\"");
commandBuilder.AppendLine($"set certificate \"{tlsCrt}\"");
commandBuilder.AppendLine("end");
var command = client.RunCommand(commandBuilder.ToString());
if (command.ExitStatus != 0 || !string.IsNullOrWhiteSpace(command.Error))
{
_logger.LogWarning(
"Checking for certificate {certId} could not be installed on FortiOS device {host} due to error: {error}",
id, fortiHost, command.Error);
success = false;
}
}
if (!localCertificateUpToDate && success)
{
var caCerts = tlsCerts.ToList();
if (!string.IsNullOrWhiteSpace(caCrt))
{
caCerts.Add(caCrt);
}
var caId = 0;
for (var i = 0; i < caCerts.Count; i++)
{
_logger.LogDebug("Installing CA certificate {index} for {certId} on FortiOS device {host}",
i + 1, id, fortiHost);
var commandBuilder = new StringBuilder();
commandBuilder.AppendLine("config vpn certificate ca");
commandBuilder.AppendLine($"edit {fortiCertId}_CA{(caId == 0 ? string.Empty : caId.ToString())}");
commandBuilder.AppendLine($"set ca \"{tlsCerts[i]}\"");
commandBuilder.AppendLine("end");
var command = client.RunCommand(commandBuilder.ToString());
if (command.ExitStatus == 0 && string.IsNullOrWhiteSpace(command.Error))
{
caId++;
continue;
}
if (command.Error.Contains("This CA certificate is duplicated."))
{
_logger.LogWarning("Skipping CA certificate {index} since it is duplicated by another certificate.",
i + i);
}
else if (command.Error.Contains("Input is not a valid CA certificate."))
{
_logger.LogDebug("Skipping CA certificate {index} since it is not a valid CA certificate.",
i + i);
}
else
{
_logger.LogWarning("Could not install CA {index} certificate due to error: {response}",
i + 1, command.Result);
success = false;
}
}
}
if (!success)
{
_logger.LogError("Reflecting {secretId} to FortiOS device using host secret {hostSecretId} completed with errors.",
id, hostSecretId);
}
else if (!localCertificateUpToDate)
{
_logger.LogInformation("Reflected {secretId} to FortiOS device using host secret {hostSecretId}.",
id, hostSecretId);
}
}
}
private async Task OnEvent(WatcherEvent <V1Secret> e)
{
if (e.Item.Type.StartsWith("helm.sh"))
{
return;
}
var secretId = KubernetesObjectId.For(e.Item.Metadata());
var item = e.Item;
_logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, secretId);
if (e.Type != WatchEventType.Added && e.Type != WatchEventType.Modified)
{
return;
}
if (!e.Item.Metadata.ReflectionAllowed() || !e.Item.Metadata.UbiquitiReflectionEnabled())
{
return;
}
if (!e.Item.Type.Equals("kubernetes.io/tls", StringComparison.InvariantCultureIgnoreCase))
{
return;
}
_logger.LogDebug("Ubiquiti enabled using host secret {secretId}.", secretId);
var tlsCrt = Encoding.Default.GetString(item.Data["tls.crt"]);
var tlsKey = Encoding.Default.GetString(item.Data["tls.key"]);
var hostSecretIds =
item.Metadata.UbiquitiReflectionHosts()
.Select(s => new KubernetesObjectId(s))
.ToList();
foreach (var hostSecretId in hostSecretIds)
{
_logger.LogDebug(
"Reflecting {secretId} to Ubiquiti device using host secret {hostSecretId}.",
secretId, hostSecretId, hostSecretId);
string hostAddress;
string username;
string password;
try
{
var hostSecret = await _apiClient.ReadNamespacedSecretAsync(hostSecretId.Name,
string.IsNullOrWhiteSpace(hostSecretId.Namespace)
?e.Item.Metadata.NamespaceProperty
: hostSecretId.Namespace);
if (hostSecret.Data is null || !hostSecret.Data.Keys.Any())
{
_logger.LogWarning("Cannot reflect {secretId} to {hostSecretId}. " +
"Host secret {hostSecretId} has no data.",
secretId, hostSecretId, hostSecretId);
continue;
}
hostAddress = hostSecret.Data.ContainsKey("host")
? Encoding.Default.GetString(hostSecret.Data["host"])
: null;
username = hostSecret.Data.ContainsKey("username")
? Encoding.Default.GetString(hostSecret.Data["username"])
: null;
password = hostSecret.Data.ContainsKey("password")
? Encoding.Default.GetString(hostSecret.Data["password"])
: null;
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning(
"Cannot reflect {secretId} to {hostSecretId}. Host secret {hostSecretId} not found.",
secretId, hostSecretId, hostSecretId);
continue;
}
if (string.IsNullOrWhiteSpace(hostAddress) || string.IsNullOrWhiteSpace(username) ||
string.IsNullOrWhiteSpace(password))
{
_logger.LogWarning(
"Cannot reflect {secretId} to Ubiquiti device using host secret {hostSecretId}. " +
"Host secret {hostSecretId} must contain 'host', 'username' and 'password' values.",
secretId, hostSecretId, hostSecretId);
continue;
}
var hostParts = hostAddress.Split(new[] { ":" }, StringSplitOptions.RemoveEmptyEntries)
.Select(s => s.Trim())
.Where(s => !string.IsNullOrWhiteSpace(s)).ToList();
if (!hostParts.Any() || hostParts.Count > 2)
{
_logger.LogWarning(
"Cannot reflect {secretId} to Ubiquiti device using host secret {hostSecretId}. " +
"Host secret {hostSecretId} contains invalid 'host' data. " +
"'host' must be in the format 'host:port' where port is optional.",
secretId, hostSecretId, hostSecretId);
}
var host = hostParts.First();
var port =
hostParts.Count < 2
? 22
: int.TryParse(hostParts.Last(), out var p)
? p
: 22;
// SSH
using var client = new SshClient(host, port, username, password);
client.ErrorOccurred += delegate(object sender, ExceptionEventArgs exceptionEventArgs)
{
_logger.LogError(exceptionEventArgs.Exception,
"Cannot reflect {secretId} to Ubiquiti device using host secret {hostSecretId} due to exception.",
secretId, hostSecretId);
};
_logger.LogDebug("Connecting to Ubiquiti device at {host}", hostAddress);
client.Connect();
_logger.LogDebug("Check certificate on Ubiquiti device at {host}", hostAddress);
var catCommand = client.RunCommand("cat /etc/ssl/private/cloudkey.crt");
if (catCommand.Result.Contains(tlsCrt))
{
_logger.LogDebug(
"Skip reflecting {secretId} to Ubiquiti device using host secret {hostSecretId}. Already exists.",
secretId, hostSecretId);
return;
}
_logger.LogDebug("Configuring new Let's Encrypt certs on Ubiquiti device at {host}", hostAddress);
client.RunCommand($"echo \"{tlsCrt}\" > /etc/ssl/private/cloudkey.crt");
client.RunCommand($"echo \"{tlsKey}\" > /etc/ssl/private/cloudkey.key");
client.RunCommand(
"rm -f /etc/ssl/private/cert.tar /etc/ssl/private/unifi.keystore.jks /etc/ssl/private/ssl-cert-snakeoil.key /etc/ssl/private/fullchain.pem");
client.RunCommand(
"openssl pkcs12 -export -in /etc/ssl/private/cloudkey.crt -inkey /etc/ssl/private/cloudkey.key -out /etc/ssl/private/cloudkey.p12 -name unifi -password pass:aircontrolenterprise");
client.RunCommand(
"keytool -importkeystore -deststorepass aircontrolenterprise -destkeypass aircontrolenterprise -destkeystore /usr/lib/unifi/data/keystore -srckeystore /etc/ssl/private/cloudkey.p12 -srcstoretype PKCS12 -srcstorepass aircontrolenterprise -alias unifi");
client.RunCommand("rm -f /etc/ssl/private/cloudkey.p12");
client.RunCommand("tar -cvf /etc/ssl/private/cert.tar /etc/ssl/private/*");
client.RunCommand("chown root:ssl-cert /etc/ssl/private/*");
client.RunCommand("chmod 640 /etc/ssl/private/*");
_logger.LogDebug("Restarting on Ubiquiti device at {host}", hostAddress);
client.RunCommand("systemctl restart nginx; systemctl restart unifi");
client.Disconnect();
_logger.LogInformation("Reflected {secretId} to Ubiquiti device using host secret {hostSecretId}.",
secretId, hostSecretId);
}
}
private async Task Annotate(V1Secret secret, Certificate certificate)
{
var secretId = KubernetesObjectId.For(secret.Metadata);
var certificateId = KubernetesObjectId.For(certificate.Metadata);
var secretAnnotations =
new Dictionary <string, string>(secret.Metadata.Annotations ?? new Dictionary <string, string>());
var original = secretAnnotations.ToDictionary(s => s.Key, s => s.Value);
var certAnnotations =
new Dictionary <string, string>(certificate.Metadata.Annotations ?? new Dictionary <string, string>());
void MatchAnnotations(Dictionary <string, string> pairs)
{
foreach (var pair in pairs)
{
if (certAnnotations.TryGetValue(pair.Key, out var value))
{
secretAnnotations[pair.Value] = value;
}
else
{
secretAnnotations.Remove(pair.Value);
}
}
}
MatchAnnotations(new Dictionary <string, string>
{
{
Annotations.CertManagerCertificate.SecretReflectionAllowed,
Annotations.Reflection.Allowed
},
{
Annotations.CertManagerCertificate.SecretReflectionAllowedNamespaces,
Annotations.Reflection.AllowedNamespaces
},
{
Annotations.CertManagerCertificate.SecretReflectionAutoEnabled,
Annotations.Reflection.AutoEnabled
},
{
Annotations.CertManagerCertificate.SecretReflectionAutoNamespaces,
Annotations.Reflection.AutoNamespaces
},
{
Annotations.CertManagerCertificate.SecretFortiEnabled,
Annotations.Reflection.FortiEnabled
},
{
Annotations.CertManagerCertificate.SecretFortiHosts,
Annotations.Reflection.FortiHosts
},
{
Annotations.CertManagerCertificate.SecretFortiCertificate,
Annotations.Reflection.FortiCertificate
}
});
if (secretAnnotations.Count == original.Count &&
secretAnnotations.Keys.All(s => original.ContainsKey(s)) &&
secretAnnotations.All(s => original[s.Key] == s.Value))
{
_logger.LogDebug(
"{secretKind} {@secretId} matches {certificateKind} {@certificateId} annotations",
secret.Kind, secretId, certificate.Kind, certificateId, Annotations.Prefix);
return;
}
_logger.LogTrace(
"Patching {secretKind} {@secretId} to match {certificateKind} {@certificateId} {reflectorPrefix} annotations",
secret.Kind, secretId, certificate.Kind, certificateId, Annotations.Prefix);
var patch = new JsonPatchDocument <V1Secret>();
patch.Replace(e => e.Metadata.Annotations, secretAnnotations);
await _client.PatchNamespacedSecretWithHttpMessagesAsync(new V1Patch(patch),
secret.Metadata.Name, secret.Metadata.NamespaceProperty);
_logger.LogInformation(
"Patched {secretKind} {@secretId} to match {certificateKind} {@certificateId} {reflectorPrefix} annotations",
secret.Kind, secretId, certificate.Kind, certificateId, Annotations.Prefix);
}
private async Task OnEvent(WatcherEvent <V1Secret> e)
{
if (e.Item.Type.StartsWith("helm.sh"))
{
return;
}
var secretId = KubernetesObjectId.For(e.Item.Metadata());
var item = e.Item;
_logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, secretId);
if (e.Type != WatchEventType.Added && e.Type != WatchEventType.Modified)
{
return;
}
if (!e.Item.Metadata.ReflectionAllowed() || !e.Item.Metadata.VMwareReflectionEnabled())
{
return;
}
if (!e.Item.Type.Equals("kubernetes.io/tls", StringComparison.InvariantCultureIgnoreCase))
{
return;
}
_logger.LogDebug("VMware enabled using host secret {secretId}.", secretId);
var tlsCrt = Encoding.Default.GetString(item.Data["tls.crt"]);
var tlsKey = Encoding.Default.GetString(item.Data["tls.key"]);
var hostSecretIds =
item.Metadata.VMwareReflectionHosts()
.Select(s => new KubernetesObjectId(s))
.ToList();
foreach (var hostSecretId in hostSecretIds)
{
_logger.LogDebug(
"Reflecting {secretId} to VMware ESXi device using host secret {hostSecretId}.",
secretId, hostSecretId, hostSecretId);
string hostAddress;
string username;
string password;
try
{
var hostSecret = await _apiClient.ReadNamespacedSecretAsync(hostSecretId.Name,
string.IsNullOrWhiteSpace(hostSecretId.Namespace)
?e.Item.Metadata.NamespaceProperty
: hostSecretId.Namespace);
if (hostSecret.Data is null || !hostSecret.Data.Keys.Any())
{
_logger.LogWarning("Cannot reflect {secretId} to {hostSecretId}. " +
"Host secret {hostSecretId} has no data.",
secretId, hostSecretId, hostSecretId);
continue;
}
hostAddress = hostSecret.Data.ContainsKey("host")
? Encoding.Default.GetString(hostSecret.Data["host"])
: null;
username = hostSecret.Data.ContainsKey("username")
? Encoding.Default.GetString(hostSecret.Data["username"])
: null;
password = hostSecret.Data.ContainsKey("password")
? Encoding.Default.GetString(hostSecret.Data["password"])
: null;
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning(
"Cannot reflect {secretId} to {hostSecretId}. Host secret {hostSecretId} not found.",
secretId, hostSecretId, hostSecretId);
continue;
}
if (string.IsNullOrWhiteSpace(hostAddress) || string.IsNullOrWhiteSpace(username) ||
string.IsNullOrWhiteSpace(password))
{
_logger.LogWarning(
"Cannot reflect {secretId} to VMware ESXi device using host secret {hostSecretId}. " +
"Host secret {hostSecretId} must contain 'host', 'username' and 'password' values.",
secretId, hostSecretId, hostSecretId);
continue;
}
var hostParts = hostAddress.Split(new[] { ":" }, StringSplitOptions.RemoveEmptyEntries)
.Select(s => s.Trim())
.Where(s => !string.IsNullOrWhiteSpace(s)).ToList();
if (!hostParts.Any() || hostParts.Count > 2)
{
_logger.LogWarning(
"Cannot reflect {secretId} to VMware ESXi device using host secret {hostSecretId}. " +
"Host secret {hostSecretId} contains invalid 'host' data. " +
"'host' must be in the format 'host:port' where port is optional.",
secretId, hostSecretId, hostSecretId);
}
var host = hostParts.First();
var port =
hostParts.Count < 2
? 22
: int.TryParse(hostParts.Last(), out var p)
? p
: 22;
// SSH
void HandleKeyEvent(object sender, AuthenticationPromptEventArgs eventArgs)
{
foreach (var prompt in eventArgs.Prompts)
{
if (prompt.Request.IndexOf("Password:"******"Cannot reflect {secretId} to VMware ESXi device using host secret {hostSecretId} due to exception.",
secretId, hostSecretId);
};
_logger.LogDebug("Connecting to VMware ESXi device at {host}", hostAddress);
client.Connect();
_logger.LogDebug("Check certificate on VMware ESXi device at {host}", hostAddress);
var catCommand = client.RunCommand("cat /etc/vmware/ssl/rui.crt");
if (catCommand.Result.Contains(tlsCrt))
{
_logger.LogDebug(
"Skip reflecting {secretId} to VMware ESXi device using host secret {hostSecretId}. Already exists.",
secretId, hostSecretId);
return;
}
_logger.LogDebug("Configuring new Let's Encrypt certs on VMware ESXi device at {host}", hostAddress);
client.RunCommand($"echo \"{tlsCrt}\" > /etc/vmware/ssl/rui.crt");
client.RunCommand($"echo \"{tlsKey}\" > /etc/vmware/ssl/rui.key");
_logger.LogDebug("Restarting on VMware ESXi device at {host}", hostAddress);
client.RunCommand("services.sh restart");
client.Disconnect();
_logger.LogInformation("Reflected {secretId} to VMware ESXi device using host secret {hostSecretId}.",
secretId, hostSecretId);
}
}
private async Task OnEvent(WatcherEvent <V1Secret> e)
{
if (e.Item.Type.StartsWith("helm.sh"))
{
return;
}
var secretId = KubernetesObjectId.For(e.Item.Metadata());
var item = e.Item;
_logger.LogTrace("[{eventType}] {kind} {@id}", e.Type, e.Item.Kind, secretId);
if (e.Type != WatchEventType.Added && e.Type != WatchEventType.Modified)
{
return;
}
if (!e.Item.Metadata.ReflectionAllowed() || !e.Item.Metadata.FreeNasReflectionEnabled())
{
return;
}
if (!e.Item.Type.Equals("kubernetes.io/tls", StringComparison.InvariantCultureIgnoreCase))
{
return;
}
_logger.LogDebug("FreeNas enabled using host secret {secretId}.", secretId);
var tlsCrt = Encoding.Default.GetString(item.Data["tls.crt"]);
var tlsKey = Encoding.Default.GetString(item.Data["tls.key"]);
var hostSecretIds =
item.Metadata.FreeNasReflectionHosts()
.Select(s => new KubernetesObjectId(s))
.ToList();
var certName = item.Metadata.FreeNasCertificate();
foreach (var hostSecretId in hostSecretIds)
{
_logger.LogDebug(
"Reflecting {secretId} to FreeNas device using host secret {hostSecretId}.",
secretId, hostSecretId, hostSecretId);
string hostAddress;
string username;
string password;
try
{
var hostSecret = await _apiClient.ReadNamespacedSecretAsync(hostSecretId.Name,
string.IsNullOrWhiteSpace(hostSecretId.Namespace)
?e.Item.Metadata.NamespaceProperty
: hostSecretId.Namespace);
if (hostSecret.Data is null || !hostSecret.Data.Keys.Any())
{
_logger.LogWarning("Cannot reflect {secretId} to {hostSecretId}. " +
"Host secret {hostSecretId} has no data.",
secretId, hostSecretId, hostSecretId);
continue;
}
hostAddress = hostSecret.Data.ContainsKey("host")
? Encoding.Default.GetString(hostSecret.Data["host"])
: null;
username = hostSecret.Data.ContainsKey("username")
? Encoding.Default.GetString(hostSecret.Data["username"])
: null;
password = hostSecret.Data.ContainsKey("password")
? Encoding.Default.GetString(hostSecret.Data["password"])
: null;
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.LogWarning(
"Cannot reflect {secretId} to {hostSecretId}. Host secret {hostSecretId} not found.",
secretId, hostSecretId, hostSecretId);
continue;
}
if (string.IsNullOrWhiteSpace(hostAddress) || string.IsNullOrWhiteSpace(username) ||
string.IsNullOrWhiteSpace(password))
{
_logger.LogWarning(
"Cannot reflect {secretId} to FreeNas device using host secret {hostSecretId}. " +
"Host secret {hostSecretId} must contain 'host', 'username' and 'password' values.",
secretId, hostSecretId, hostSecretId);
continue;
}
var hostParts = hostAddress.Split(new[] { ":" }, StringSplitOptions.RemoveEmptyEntries)
.Select(s => s.Trim())
.Where(s => !string.IsNullOrWhiteSpace(s)).ToList();
if (!hostParts.Any() || hostParts.Count > 2)
{
_logger.LogWarning(
"Cannot reflect {secretId} to FreeNas device using host secret {hostSecretId}. " +
"Host secret {hostSecretId} contains invalid 'host' data. " +
"'host' must be in the format 'host:port' where port is optional.",
secretId, hostSecretId, hostSecretId);
}
var host = hostParts.First();
// Check if certificate is the same
var client = _clientFactory.CreateClient();
client.BaseAddress = new Uri($"https://{host}/api/v2.0/");
client.DefaultRequestHeaders.Clear();
client.DefaultRequestHeaders.ConnectionClose = true;
client.DefaultRequestHeaders
.Accept
.Add(new MediaTypeWithQualityHeaderValue("application/json")); //ACCEPT header
client.DefaultRequestHeaders.Add("User-Agent", "Emberstack/Reflector");
var authenticationString = $"{username}:{password}";
var base64EncodedAuthenticationString =
Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationString));
client.DefaultRequestHeaders.Authorization =
new AuthenticationHeaderValue("Basic", base64EncodedAuthenticationString);
var response = await client.GetAsync(CertificateUri);
var options = new JsonSerializerOptions {
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
await using var responseStream = await response.Content.ReadAsStreamAsync();
using var responseStr = response.Content.ReadAsStringAsync();
var certificates =
await JsonSerializer.DeserializeAsync <List <FreeNasCertificate> >(responseStream, options);
// var bytes = Encoding.ASCII.GetBytes(tlsCrt);
// var x509Certificate2 = new X509Certificate2(bytes);
//
// var name = $"{certName}-{x509Certificate2.NotAfter:s}";
var name = certName;
var cert = certificates
.SingleOrDefault(x => x.Name == name);
var certExists = !(cert is null);
if (certExists)
{
if (tlsCrt.Contains(cert.Certificate))
{
_logger.LogDebug(
"Skip reflecting {secretId} to FreeNas device using host secret {hostSecretId}. Already exists.",
secretId, hostSecretId);
return;
}
}
// Create the certificate
var bodyCreate = JsonSerializer.Serialize(new FreeNasCertificateCreateImported
{
Name = name,
Certificate = tlsCrt,
Privatekey = tlsKey
}, options);
await client.PostAsync(CertificateUri, new StringContent(bodyCreate));
// Set the certificate as default
var certId = certExists
? cert.Id
: certificates.Single(x => x.Name == name).Id;
var bodyGeneral = JsonSerializer.Serialize(new FreeNasSystemGeneral
{
UiCertificate = certId
}, options);
await client.PutAsync("system/general/", new StringContent(bodyGeneral));
_logger.LogInformation("Reflected {secretId} to FreeNas device using host secret {hostSecretId}.",
secretId, hostSecretId);
}
}
