/// <summary>
/// Ensure that an externally-facing Service resource does not exist for the specified database server.
/// </summary>
public async Task EnsureExternalServiceAbsent()
{
RequireCurrentState();
ServiceV1 existingExternalService = await FindExternalService();
if (existingExternalService != null)
{
Log.LogInformation("Deleting external service {ServiceName} for server {ServerId}...",
existingExternalService.Metadata.Name,
State.Id
);
StatusV1 result = await KubeClient.ServicesV1().Delete(
name: existingExternalService.Metadata.Name,
kubeNamespace: KubeOptions.KubeNamespace
);
if (result.Status != "Success" && result.Reason != "NotFound")
{
Log.LogError("Failed to delete external service {ServiceName} for server {ServerId} (Message:{FailureMessage}, Reason:{FailureReason}).",
existingExternalService.Metadata.Name,
State.Id,
result.Message,
result.Reason
);
}
Log.LogInformation("Deleted external service {ServiceName} for server {ServerId}.",
existingExternalService.Metadata.Name,
State.Id
);
}
}
public void should_return_service_from_k8s()
{
var token = "Bearer txpc696iUhbVoudg164r93CxDTrKRVWG";
var serviceEntryOne = new ServiceV1()
{
Kind = "service",
ApiVersion = "1.0",
Metadata = new ObjectMetaV1()
{
Namespace = "dev"
},
Spec = new ServiceSpecV1()
{
ClusterIP = "localhost"
},
Status = new ServiceStatusV1()
{
LoadBalancer = new LoadBalancerStatusV1()
}
};
serviceEntryOne.Spec.Ports.Add(
new ServicePortV1()
{
Port = 80
}
);
this.Given(x => GivenThereIsAFakeKubeServiceDiscoveryProvider(_fakekubeServiceDiscoveryUrl, _serviceName, _namespaces))
.And(x => GivenTheServicesAreRegisteredWithKube(serviceEntryOne))
.When(x => WhenIGetTheServices())
.Then(x => ThenTheCountIs(1))
.And(_ => _receivedToken.ShouldBe(token))
.BDDfy();
}
/// <summary>
/// Ensure that an externally-facing Service resource exists for the specified database server.
/// </summary>
/// <returns>
/// A <see cref="Task"/> representing the operation.
/// </returns>
public async Task EnsureExternalServicePresent()
{
RequireCurrentState();
ServiceV1 existingExternalService = await FindExternalService();
if (existingExternalService == null)
{
Log.LogInformation("Creating external service for server {ServerId}...",
State.Id
);
ServiceV1 createdService = await KubeClient.ServicesV1().Create(
KubeResources.ExternalService(State,
kubeNamespace: KubeOptions.KubeNamespace
)
);
Log.LogInformation("Successfully created external service {ServiceName} for server {ServerId}.",
createdService.Metadata.Name,
State.Id
);
}
else
{
Log.LogInformation("Found existing external service {ServiceName} for server {ServerId}.",
existingExternalService.Metadata.Name,
State.Id
);
}
}
public KubeServiceDiscoveryProviderTests()
{
_serviceName = "test";
_namespaces = "dev";
_port = 8001;
_kubeHost = "localhost";
_fakekubeServiceDiscoveryUrl = $"http://{_kubeHost}:{_port}";
_serviceEntries = new ServiceV1();
_factory = new Mock <IOcelotLoggerFactory>();
var option = new KubeClientOptions
{
ApiEndPoint = new Uri(_fakekubeServiceDiscoveryUrl),
AccessToken = "txpc696iUhbVoudg164r93CxDTrKRVWG",
AuthStrategy = KubeClient.KubeAuthStrategy.BearerToken,
AllowInsecure = true
};
_clientFactory = KubeApiClient.Create(option);
_logger = new Mock <IOcelotLogger>();
_factory.Setup(x => x.CreateLogger <Kube>()).Returns(_logger.Object);
var config = new KubeRegistryConfiguration()
{
KeyOfServiceInK8s = _serviceName,
KubeNamespace = _namespaces
};
_provider = new Kube(config, _factory.Object, _clientFactory);
}
public ServiceResources(DeploymentV1 deployment, StatefulSetV1 statefulSet, ServiceV1 service, IngressV1Beta1 ingress, SecretV1 secret)
{
Deployment = deployment;
StatefulSet = statefulSet;
Service = service;
Ingress = ingress;
Secret = secret;
}
private bool IsValid(ServiceV1 service)
{
if (string.IsNullOrEmpty(service.Spec.ClusterIP) || service.Spec.Ports.Count <= 0)
{
return(false);
}
return(true);
}
private Service BuildService(ServiceV1 serviceEntry)
{
var servicePort = serviceEntry.Spec.Ports.FirstOrDefault();
return(new Service(
serviceEntry.Metadata.Name,
new ServiceHostAndPort(serviceEntry.Spec.ClusterIP, servicePort.Port),
serviceEntry.Metadata.Uid,
string.Empty,
Enumerable.Empty <string>()));
}
/// <summary>
/// Get the public TCP port number on which the database server is accessible.
/// </summary>
/// <param name="server">
/// A <see cref="DatabaseServer"/> describing the server.
/// </param>
/// <param name="kubeNamespace">
/// An optional target Kubernetes namespace.
/// </param>
/// <returns>
/// The port, or <c>null</c> if the externally-facing service for the server cannot be found.
/// </returns>
public async Task <int?> GetPublicPort()
{
RequireCurrentState();
List <ServiceV1> matchingServices = await KubeClient.ServicesV1().List(
labelSelector: $"cloud.dimensiondata.daas.server-id = {State.Id}, cloud.dimensiondata.daas.service-type = external",
kubeNamespace: KubeOptions.KubeNamespace
);
if (matchingServices.Count == 0)
{
return(null);
}
ServiceV1 externalService = matchingServices[matchingServices.Count - 1];
return(externalService.Spec.Ports[0].NodePort);
}
/// <summary>
/// Request creation of a <see cref="ServiceV1"/>.
/// </summary>
/// <param name="newService">
/// A <see cref="ServiceV1"/> representing the Service to create.
/// </param>
/// <param name="cancellationToken">
/// An optional <see cref="CancellationToken"/> that can be used to cancel the request.
/// </param>
/// <returns>
/// A <see cref="ServiceV1"/> representing the current state for the newly-created Service.
/// </returns>
public async Task <ServiceV1> Create(ServiceV1 newService, CancellationToken cancellationToken = default)
{
if (newService == null)
{
throw new ArgumentNullException(nameof(newService));
}
return(await Http
.PostAsJsonAsync(
Requests.Collection.WithTemplateParameters(new
{
Namespace = newService?.Metadata?.Namespace ?? KubeClient.DefaultNamespace
}),
postBody : newService,
cancellationToken : cancellationToken
)
.ReadContentAsAsync <ServiceV1, StatusV1>());
}
/// <summary>
/// Request creation of a <see cref="ServiceV1"/>.
/// </summary>
/// <param name="newService">
/// A <see cref="ServiceV1"/> representing the Service to create.
/// </param>
/// <param name="cancellationToken">
/// An optional <see cref="CancellationToken"/> that can be used to cancel the request.
/// </param>
/// <returns>
/// A <see cref="ServiceV1"/> representing the current state for the newly-created Service.
/// </returns>
public async Task <ServiceV1> Create(ServiceV1 newService, CancellationToken cancellationToken = default)
{
if (newService == null)
{
throw new ArgumentNullException(nameof(newService));
}
return(await Http
.PostAsJsonAsync(
Requests.Collection.WithTemplateParameters(new
{
Namespace = newService?.Metadata?.Namespace ?? KubeClient.DefaultNamespace
}),
postBody : newService,
cancellationToken : cancellationToken
)
.ReadContentAsObjectV1Async <ServiceV1>(
operationDescription: $"create v1/Service resource in namespace '{newService?.Metadata?.Namespace ?? KubeClient.DefaultNamespace}'"
));
}
/// <summary>
/// Determine the connection string for the specified server.
/// </summary>
/// <param name="serverId">
/// The Id of the target server.
/// </param>
/// <returns>
/// The base UR.
/// </returns>
async Task <Uri> GetServerBaseAddress(string serverId)
{
if (String.IsNullOrWhiteSpace(serverId))
{
throw new ArgumentException("Argument cannot be null, empty, or entirely composed of whitespace: 'serverId'.", nameof(serverId));
}
Log.LogInformation("Determining connection string for server {ServerId}...",
serverId
);
DatabaseServer targetServer = await GetServer(serverId);
List <ServiceV1> matchingServices = await KubeClient.ServicesV1().List(
labelSelector: $"cloud.dimensiondata.daas.server-id = {serverId},cloud.dimensiondata.daas.service-type = internal",
kubeNamespace: KubeOptions.KubeNamespace
);
if (matchingServices.Count == 0)
{
Log.LogWarning("Cannot determine connection string for server {ServerId} (server's associated Kubernetes Service not found).",
serverId
);
throw RespondWith(NotFound(new
{
Reason = "EndPointNotFound",
Id = serverId,
EntityType = "DatabaseServer",
Message = $"Cannot determine base address for server '{targetServer.Id}'."
}));
}
ServiceV1 serverService = matchingServices[matchingServices.Count - 1];
string serverFQDN = $"{serverService.Metadata.Name}.{serverService.Metadata.Namespace}.svc.cluster.local";
int serverPort = serverService.Spec.Ports[0].Port;
Log.LogInformation("Database proxy will connect to RavenDB server '{ServerFQDN}' on {ServerPort}.", serverFQDN, serverPort);
return(new Uri($"http://{serverFQDN}:{serverPort}"));
}
/// <summary>
/// Determine the connection string for the specified <see cref="SqlRequest"/>.
/// </summary>
/// <param name="request">
/// The <see cref="SqlRequest"/> being executed.
/// </param>
/// <returns>
/// The connection string.
/// </returns>
async Task <string> GetConnectionString(SqlRequest request)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
Log.LogInformation("Determining connection string for database {DatabaseId} in server {ServerId}...",
request.DatabaseId,
request.ServerId
);
DatabaseServer targetServer = await DocumentSession.LoadAsync <DatabaseServer>(request.ServerId);
if (targetServer == null)
{
Log.LogWarning("Cannot determine connection string for database {DatabaseId} in server {ServerId} (server not found).",
request.DatabaseId,
request.ServerId
);
throw RespondWith(Ok(new SqlResult
{
ResultCode = -1,
Errors =
{
new SqlError
{
Kind = SqlErrorKind.Infrastructure,
Message = $"Unable to determine connection settings for database {request.DatabaseId} in server {request.ServerId} (server not found)."
}
}
}));
}
List <ServiceV1> matchingServices = await KubeClient.ServicesV1().List(
labelSelector: $"cloud.dimensiondata.daas.server-id = {targetServer.Id},cloud.dimensiondata.daas.service-type = internal",
kubeNamespace: KubeOptions.KubeNamespace
);
if (matchingServices.Count == 0)
{
Log.LogWarning("Cannot determine connection string for database {DatabaseId} in server {ServerId} (server's associated Kubernetes Service not found).",
request.DatabaseId,
request.ServerId
);
throw RespondWith(Ok(new SqlResult
{
ResultCode = -1,
Errors =
{
new SqlError
{
Kind = SqlErrorKind.Infrastructure,
Message = $"Unable to determine connection settings for database {request.DatabaseId} in server {request.ServerId} (server's associated Kubernetes Service not found)."
}
}
}));
}
ServiceV1 serverService = matchingServices[matchingServices.Count - 1];
(string serverFQDN, int?serverPort) = serverService.GetHostAndPort(portName: "sql-server");
if (serverPort == null)
{
Log.LogWarning("Cannot determine connection string for database {DatabaseId} in server {ServerId} (cannot find the port named 'sql-server' on server's associated Kubernetes Service).",
request.DatabaseId,
request.ServerId
);
throw RespondWith(Ok(new SqlResult
{
ResultCode = -1,
Errors =
{
new SqlError
{
Kind = SqlErrorKind.Infrastructure,
Message = $"Unable to determine connection settings for database {request.DatabaseId} in server {request.ServerId} (cannot find the port named 'sql-server' on server's associated Kubernetes Service)."
}
}
}));
}
Log.LogInformation("Database proxy will connect to SQL Server '{ServerFQDN}' on {ServerPort}.", serverFQDN, serverPort);
var connectionStringBuilder = new SqlClient.SqlConnectionStringBuilder
{
DataSource = $"tcp:{serverFQDN},{serverPort}",
};
var serverSettings = targetServer.GetSettings <SqlServerSettings>();
if (request.DatabaseId != MasterDatabaseId)
{
DatabaseInstance targetDatabase = await DocumentSession.LoadAsync <DatabaseInstance>(request.DatabaseId);
if (targetDatabase == null)
{
Log.LogWarning("Cannot determine connection string for database {DatabaseId} in server {ServerId} (database not found).",
request.DatabaseId,
request.ServerId
);
throw RespondWith(Ok(new SqlResult
{
ResultCode = -1,
Errors =
{
new SqlError
{
Kind = SqlErrorKind.Infrastructure,
Message = $"Unable to determine connection settings for database {request.DatabaseId} in server {request.ServerId} (database not found)."
}
}
}));
}
connectionStringBuilder.InitialCatalog = targetDatabase.Name;
if (request.ExecuteAsAdminUser)
{
connectionStringBuilder.UserID = "sa";
connectionStringBuilder.Password = serverSettings.AdminPassword;
}
else
{
connectionStringBuilder.UserID = targetDatabase.DatabaseUser;
connectionStringBuilder.Password = targetDatabase.DatabasePassword;
}
}
else
{
connectionStringBuilder.InitialCatalog = "master";
connectionStringBuilder.UserID = "sa";
connectionStringBuilder.Password = serverSettings.AdminPassword;
}
Log.LogInformation("Successfully determined connection string for database {DatabaseId} ({DatabaseName}) in server {ServerId} ({ServerSqlName}).",
request.DatabaseId,
connectionStringBuilder.InitialCatalog,
request.ServerId,
connectionStringBuilder.DataSource
);
return(connectionStringBuilder.ConnectionString);
}
private void GivenTheServicesAreRegisteredWithKube(ServiceV1 serviceEntries)
{
_serviceEntries = serviceEntries;
}
private async Task <ServiceResources> DeployService(DeployCommand.Types.Service command, string kubeNamespace)
{
DeploymentV1 deployment = null;
StatefulSetV1 statefulSet = null;
var secret = await CreateSecret();
if (string.IsNullOrWhiteSpace(command.PersistentStoragePath))
{
deployment = await CreateDeployment();
}
else
{
statefulSet = await CreateStatefulSet();
}
var service = await CreateService();
var ingress = await CreateServiceIngress();
return(new ServiceResources(deployment, statefulSet, service, ingress, secret));
async Task <SecretV1> CreateSecret()
{
if (!command.Secrets.Any())
{
return(null);
}
var existingSecret = await kubeApiClient.SecretsV1().Get(command.Name, kubeNamespace);
var secretResource = new SecretV1
{
Metadata = new ObjectMetaV1
{
Name = command.Name,
Namespace = kubeNamespace,
},
Type = "Opaque",
};
if (existingSecret != null)
{
foreach (var existingData in existingSecret.Data)
{
secretResource.Data[existingData.Key] = existingData.Value;
}
}
foreach (var secretCommand in command.Secrets.Where(s => s.Value != null))
{
secretResource.Data[secretCommand.Name] = Convert.ToBase64String(Encoding.UTF8.GetBytes(secretCommand.Value));
}
foreach (var existingSecretNames in secretResource.Data.Keys)
{
if (command.Secrets.All(secretCommand => secretCommand.Name != existingSecretNames))
{
secretResource.Data.Remove(existingSecretNames);
}
}
return(await kubeApiClient.Dynamic().Apply(secretResource, fieldManager: "clud", force: true));
}
async Task <DeploymentV1> CreateDeployment()
{
var deployment = new DeploymentV1
{
Metadata = new ObjectMetaV1
{
Name = command.Name,
Namespace = kubeNamespace,
},
Spec = new DeploymentSpecV1
{
Selector = new LabelSelectorV1
{
MatchLabels = { { KubeNaming.AppLabelKey, command.Name } },
},
Replicas = command.Replicas,
Template = new PodTemplateSpecV1
{
Metadata = new ObjectMetaV1
{
Name = command.Name,
Namespace = kubeNamespace,
Labels = { { KubeNaming.AppLabelKey, command.Name } }
},
Spec = new PodSpecV1
{
Containers =
{
new ContainerV1
{
Name = command.Name,
Image = DockerImageName(),
}
},
},
}
}
};
AddEnvironmentVariables(deployment.Spec.Template.Spec.Containers.Single().Env);
return(await kubeApiClient.Dynamic().Apply(deployment, fieldManager: "clud", force: true));
}
async Task <StatefulSetV1> CreateStatefulSet()
{
var statefulSet = new StatefulSetV1
{
Metadata = new ObjectMetaV1
{
Name = command.Name,
Namespace = kubeNamespace,
},
Spec = new StatefulSetSpecV1
{
Selector = new LabelSelectorV1
{
MatchLabels = { { KubeNaming.AppLabelKey, command.Name } },
},
Template = new PodTemplateSpecV1
{
Metadata = new ObjectMetaV1
{
Name = command.Name,
Namespace = kubeNamespace,
Labels = { { KubeNaming.AppLabelKey, command.Name } }
},
Spec = new PodSpecV1
{
Containers =
{
new ContainerV1
{
Name = command.Name,
Image = DockerImageName(),
VolumeMounts = { new VolumeMountV1
{
Name = command.Name,
MountPath = command.PersistentStoragePath,
} }
},
},
},
},
VolumeClaimTemplates =
{
new PersistentVolumeClaimV1
{
Metadata = new ObjectMetaV1
{
Name = command.Name,
Namespace = kubeNamespace,
},
Spec = new PersistentVolumeClaimSpecV1
{
AccessModes ={ "ReadWriteOnce" },
Resources = new ResourceRequirementsV1
{
Requests ={ { "storage", "100Mi" } },
}
}
}
}
}
};
AddEnvironmentVariables(statefulSet.Spec.Template.Spec.Containers.Single().Env);
return(await kubeApiClient.Dynamic().Apply(statefulSet, fieldManager: "clud", force: true));
}
string DockerImageName()
{
return(command.IsPublicDockerImage
? command.DockerImage
: $"{KubeNaming.DockerRegistryLocation}/{command.DockerImage}");
}
void AddEnvironmentVariables(List <EnvVarV1> envVarV1s)
{
envVarV1s.AddRange(command.EnvironmentVariables.Select(env => new EnvVarV1
{
Name = env.Name,
Value = env.Value
}));
envVarV1s.AddRange(command.Secrets.Select(secret => new EnvVarV1
{
Name = secret.Name,
ValueFrom = new EnvVarSourceV1
{
SecretKeyRef = new SecretKeySelectorV1
{
Name = command.Name,
Key = secret.Name,
Optional = false,
}
}
}));
}
async Task <ServiceV1> CreateService()
{
var service = new ServiceV1
{
Metadata = new ObjectMetaV1
{
Name = command.Name,
Namespace = kubeNamespace,
},
Spec = new ServiceSpecV1
{
Selector = { { KubeNaming.AppLabelKey, command.Name } },
},
};
if (command.HttpPort != null)
{
service.Spec.Ports.Add(new ServicePortV1
{
Name = KubeNaming.HttpPortName,
Protocol = "TCP",
Port = command.HttpPort.Value
});
}
service.Spec.Ports.AddRange(command.TcpPorts.Select(port => new ServicePortV1
{
Name = $"tcp-{port}",
Protocol = "TCP",
Port = port,
}));
service.Spec.Ports.AddRange(command.UdpPorts.Select(port => new ServicePortV1
{
Name = $"udp-{port}",
Protocol = "UDP",
Port = port,
}));
return(await kubeApiClient.Dynamic().Apply(service, fieldManager: "clud", force: true));
}
async Task <IngressV1Beta1> CreateServiceIngress()
{
if (command.HttpPort == null)
{
return(null);
}
var ingress = new IngressV1Beta1
{
Metadata = new ObjectMetaV1
{
Name = command.Name,
Namespace = kubeNamespace,
},
Spec = new IngressSpecV1Beta1
{
Rules =
{
new IngressRuleV1Beta1
{
Host = $"{command.Name}-{kubeNamespace}.{cludOptions.BaseHostname}",
Http = new HTTPIngressRuleValueV1Beta1
{
Paths =
{
new HTTPIngressPathV1Beta1
{
Path = "/",
Backend = new IngressBackendV1Beta1
{
ServiceName = service.Metadata.Name,
ServicePort = KubeNaming.HttpPortName,
}
}
}
}
}
},
},
};
return(await kubeApiClient.Dynamic().Apply(ingress, fieldManager: "clud", force: true));
}
}
static async Task <int> Start(StartOptions options)
{
var client = KubeApiClient.Create(new KubeClientOptions {
ApiEndPoint = new Uri("http://localhost:8001"),
KubeNamespace = "tempest"
});
if (!(await client.NamespacesV1().List()).Items.Exists(x => x.Metadata.Name == "tempest"))
{
await client.NamespacesV1().Create(new NamespaceV1 {
Metadata = new ObjectMetaV1 {
Name = "tempest"
}
});
}
if ((await client.ServicesV1().List()).Items.Count != 0)
{
Console.Error.WriteLine("Tempest services already exist! Shut down existing services to start anew");
return(1);
}
var svc = new ServiceV1 {
Metadata = new ObjectMetaV1 {
Name = "tempest", Namespace = "tempest"
},
Kind = "Service",
Spec = new ServiceSpecV1 {
Selector = { ["app"] = "tempest" }
}
};
svc.Spec.Ports.Add(new ServicePortV1 {
Protocol = "TCP", Port = 12345, TargetPort = 12345
});
svc.Spec.Type = "NodePort";
await client.ServicesV1().Create(svc);
Console.WriteLine("Generating server certificate");
var serverCert = CertGenerator.Generate();
Console.WriteLine("Generating client certificate");
var clientCert = CertGenerator.Generate();
Console.WriteLine("Done, creating deployment");
await client.DeploymentsV1().Create(new DeploymentV1 {
Metadata = new ObjectMetaV1 {
Name = "tempest", Namespace = "tempest"
},
Kind = "Deployment",
Spec = new DeploymentSpecV1 {
Replicas = (await client.NodesV1().List()).Items.Count,
Selector = new LabelSelectorV1 {
MatchLabels = { ["app"] = "tempest" }
},
Template = new PodTemplateSpecV1 {
Metadata = new ObjectMetaV1 {
Labels = { ["app"] = "tempest" }
},
Spec = new PodSpecV1 {
Containers =
{
new ContainerV1 {
Name = "tempest",
Image = "daeken/tempest-server:v1",
ImagePullPolicy = "Always",
Ports = { new ContainerPortV1 {
ContainerPort = 12345
} },
Env = { new EnvVarV1 {
Name = "SERVER_PFX",
Value = Convert.ToBase64String(serverCert.Export(X509ContentType.Pfx))
}, new EnvVarV1 {
Name = "CLIENT_CERT",
Value = Convert.ToBase64String(clientCert.Export(X509ContentType.Cert))
} }
}
}
}
}
}
});
Console.WriteLine("Services created, waiting for ready state");
while (true)
{
var deployment = await client.DeploymentsV1().Get("tempest");
if (deployment.Status.ReadyReplicas != null &&
deployment.Status.ReadyReplicas == deployment.Status.Replicas)
{
break;
}
}
var tempestPath = Path.Join(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
".tempest");
try { File.Delete(Path.Join(tempestPath, "server.cert")); } catch (Exception) {}
try { File.Delete(Path.Join(tempestPath, "client.pfx")); } catch (Exception) {}
try { File.Delete(Path.Join(tempestPath, "nodes")); } catch (Exception) {}
Directory.CreateDirectory(tempestPath);
await using (var fp = File.OpenWrite(Path.Join(tempestPath, "server.cert")))
fp.Write(serverCert.Export(X509ContentType.Cert));
await using (var fp = File.OpenWrite(Path.Join(tempestPath, "client.pfx")))
fp.Write(serverCert.Export(X509ContentType.Pfx));
var ips = new List <string>();
await using (var fp = File.OpenWrite(Path.Join(tempestPath, "nodes"))) {
await using var sw = new StreamWriter(fp);
foreach (var elem in await client.NodesV1().List())
{
var ip = elem.Status.Addresses[2].Address;
sw.WriteLine(ip);
ips.Add(ip);
}
}
Console.WriteLine($"Tempest started successfully on {(await client.DeploymentsV1().Get("tempest")).Status.Replicas} nodes");
Console.WriteLine("Checking authentication");
var pclient = new Protocol.Client(new TcpClient(ips[0], 12345));
return(0);
}
