/// <summary> /// Ensure that an externally-facing Service resource does not exist for the specified database server. /// </summary> public async Task EnsureExternalServiceAbsent() { RequireCurrentState(); ServiceV1 existingExternalService = await FindExternalService(); if (existingExternalService != null) { Log.LogInformation("Deleting external service {ServiceName} for server {ServerId}...", existingExternalService.Metadata.Name, State.Id ); StatusV1 result = await KubeClient.ServicesV1().Delete( name: existingExternalService.Metadata.Name, kubeNamespace: KubeOptions.KubeNamespace ); if (result.Status != "Success" && result.Reason != "NotFound") { Log.LogError("Failed to delete external service {ServiceName} for server {ServerId} (Message:{FailureMessage}, Reason:{FailureReason}).", existingExternalService.Metadata.Name, State.Id, result.Message, result.Reason ); } Log.LogInformation("Deleted external service {ServiceName} for server {ServerId}.", existingExternalService.Metadata.Name, State.Id ); } }
public void should_return_service_from_k8s() { var token = "Bearer txpc696iUhbVoudg164r93CxDTrKRVWG"; var serviceEntryOne = new ServiceV1() { Kind = "service", ApiVersion = "1.0", Metadata = new ObjectMetaV1() { Namespace = "dev" }, Spec = new ServiceSpecV1() { ClusterIP = "localhost" }, Status = new ServiceStatusV1() { LoadBalancer = new LoadBalancerStatusV1() } }; serviceEntryOne.Spec.Ports.Add( new ServicePortV1() { Port = 80 } ); this.Given(x => GivenThereIsAFakeKubeServiceDiscoveryProvider(_fakekubeServiceDiscoveryUrl, _serviceName, _namespaces)) .And(x => GivenTheServicesAreRegisteredWithKube(serviceEntryOne)) .When(x => WhenIGetTheServices()) .Then(x => ThenTheCountIs(1)) .And(_ => _receivedToken.ShouldBe(token)) .BDDfy(); }
/// <summary> /// Ensure that an externally-facing Service resource exists for the specified database server. /// </summary> /// <returns> /// A <see cref="Task"/> representing the operation. /// </returns> public async Task EnsureExternalServicePresent() { RequireCurrentState(); ServiceV1 existingExternalService = await FindExternalService(); if (existingExternalService == null) { Log.LogInformation("Creating external service for server {ServerId}...", State.Id ); ServiceV1 createdService = await KubeClient.ServicesV1().Create( KubeResources.ExternalService(State, kubeNamespace: KubeOptions.KubeNamespace ) ); Log.LogInformation("Successfully created external service {ServiceName} for server {ServerId}.", createdService.Metadata.Name, State.Id ); } else { Log.LogInformation("Found existing external service {ServiceName} for server {ServerId}.", existingExternalService.Metadata.Name, State.Id ); } }
public KubeServiceDiscoveryProviderTests() { _serviceName = "test"; _namespaces = "dev"; _port = 8001; _kubeHost = "localhost"; _fakekubeServiceDiscoveryUrl = $"http://{_kubeHost}:{_port}"; _serviceEntries = new ServiceV1(); _factory = new Mock <IOcelotLoggerFactory>(); var option = new KubeClientOptions { ApiEndPoint = new Uri(_fakekubeServiceDiscoveryUrl), AccessToken = "txpc696iUhbVoudg164r93CxDTrKRVWG", AuthStrategy = KubeClient.KubeAuthStrategy.BearerToken, AllowInsecure = true }; _clientFactory = KubeApiClient.Create(option); _logger = new Mock <IOcelotLogger>(); _factory.Setup(x => x.CreateLogger <Kube>()).Returns(_logger.Object); var config = new KubeRegistryConfiguration() { KeyOfServiceInK8s = _serviceName, KubeNamespace = _namespaces }; _provider = new Kube(config, _factory.Object, _clientFactory); }
public ServiceResources(DeploymentV1 deployment, StatefulSetV1 statefulSet, ServiceV1 service, IngressV1Beta1 ingress, SecretV1 secret) { Deployment = deployment; StatefulSet = statefulSet; Service = service; Ingress = ingress; Secret = secret; }
private bool IsValid(ServiceV1 service) { if (string.IsNullOrEmpty(service.Spec.ClusterIP) || service.Spec.Ports.Count <= 0) { return(false); } return(true); }
private Service BuildService(ServiceV1 serviceEntry) { var servicePort = serviceEntry.Spec.Ports.FirstOrDefault(); return(new Service( serviceEntry.Metadata.Name, new ServiceHostAndPort(serviceEntry.Spec.ClusterIP, servicePort.Port), serviceEntry.Metadata.Uid, string.Empty, Enumerable.Empty <string>())); }
/// <summary> /// Get the public TCP port number on which the database server is accessible. /// </summary> /// <param name="server"> /// A <see cref="DatabaseServer"/> describing the server. /// </param> /// <param name="kubeNamespace"> /// An optional target Kubernetes namespace. /// </param> /// <returns> /// The port, or <c>null</c> if the externally-facing service for the server cannot be found. /// </returns> public async Task <int?> GetPublicPort() { RequireCurrentState(); List <ServiceV1> matchingServices = await KubeClient.ServicesV1().List( labelSelector: $"cloud.dimensiondata.daas.server-id = {State.Id}, cloud.dimensiondata.daas.service-type = external", kubeNamespace: KubeOptions.KubeNamespace ); if (matchingServices.Count == 0) { return(null); } ServiceV1 externalService = matchingServices[matchingServices.Count - 1]; return(externalService.Spec.Ports[0].NodePort); }
/// <summary> /// Request creation of a <see cref="ServiceV1"/>. /// </summary> /// <param name="newService"> /// A <see cref="ServiceV1"/> representing the Service to create. /// </param> /// <param name="cancellationToken"> /// An optional <see cref="CancellationToken"/> that can be used to cancel the request. /// </param> /// <returns> /// A <see cref="ServiceV1"/> representing the current state for the newly-created Service. /// </returns> public async Task <ServiceV1> Create(ServiceV1 newService, CancellationToken cancellationToken = default) { if (newService == null) { throw new ArgumentNullException(nameof(newService)); } return(await Http .PostAsJsonAsync( Requests.Collection.WithTemplateParameters(new { Namespace = newService?.Metadata?.Namespace ?? KubeClient.DefaultNamespace }), postBody : newService, cancellationToken : cancellationToken ) .ReadContentAsAsync <ServiceV1, StatusV1>()); }
/// <summary> /// Request creation of a <see cref="ServiceV1"/>. /// </summary> /// <param name="newService"> /// A <see cref="ServiceV1"/> representing the Service to create. /// </param> /// <param name="cancellationToken"> /// An optional <see cref="CancellationToken"/> that can be used to cancel the request. /// </param> /// <returns> /// A <see cref="ServiceV1"/> representing the current state for the newly-created Service. /// </returns> public async Task <ServiceV1> Create(ServiceV1 newService, CancellationToken cancellationToken = default) { if (newService == null) { throw new ArgumentNullException(nameof(newService)); } return(await Http .PostAsJsonAsync( Requests.Collection.WithTemplateParameters(new { Namespace = newService?.Metadata?.Namespace ?? KubeClient.DefaultNamespace }), postBody : newService, cancellationToken : cancellationToken ) .ReadContentAsObjectV1Async <ServiceV1>( operationDescription: $"create v1/Service resource in namespace '{newService?.Metadata?.Namespace ?? KubeClient.DefaultNamespace}'" )); }
/// <summary> /// Determine the connection string for the specified server. /// </summary> /// <param name="serverId"> /// The Id of the target server. /// </param> /// <returns> /// The base UR. /// </returns> async Task <Uri> GetServerBaseAddress(string serverId) { if (String.IsNullOrWhiteSpace(serverId)) { throw new ArgumentException("Argument cannot be null, empty, or entirely composed of whitespace: 'serverId'.", nameof(serverId)); } Log.LogInformation("Determining connection string for server {ServerId}...", serverId ); DatabaseServer targetServer = await GetServer(serverId); List <ServiceV1> matchingServices = await KubeClient.ServicesV1().List( labelSelector: $"cloud.dimensiondata.daas.server-id = {serverId},cloud.dimensiondata.daas.service-type = internal", kubeNamespace: KubeOptions.KubeNamespace ); if (matchingServices.Count == 0) { Log.LogWarning("Cannot determine connection string for server {ServerId} (server's associated Kubernetes Service not found).", serverId ); throw RespondWith(NotFound(new { Reason = "EndPointNotFound", Id = serverId, EntityType = "DatabaseServer", Message = $"Cannot determine base address for server '{targetServer.Id}'." })); } ServiceV1 serverService = matchingServices[matchingServices.Count - 1]; string serverFQDN = $"{serverService.Metadata.Name}.{serverService.Metadata.Namespace}.svc.cluster.local"; int serverPort = serverService.Spec.Ports[0].Port; Log.LogInformation("Database proxy will connect to RavenDB server '{ServerFQDN}' on {ServerPort}.", serverFQDN, serverPort); return(new Uri($"http://{serverFQDN}:{serverPort}")); }
/// <summary> /// Determine the connection string for the specified <see cref="SqlRequest"/>. /// </summary> /// <param name="request"> /// The <see cref="SqlRequest"/> being executed. /// </param> /// <returns> /// The connection string. /// </returns> async Task <string> GetConnectionString(SqlRequest request) { if (request == null) { throw new ArgumentNullException(nameof(request)); } Log.LogInformation("Determining connection string for database {DatabaseId} in server {ServerId}...", request.DatabaseId, request.ServerId ); DatabaseServer targetServer = await DocumentSession.LoadAsync <DatabaseServer>(request.ServerId); if (targetServer == null) { Log.LogWarning("Cannot determine connection string for database {DatabaseId} in server {ServerId} (server not found).", request.DatabaseId, request.ServerId ); throw RespondWith(Ok(new SqlResult { ResultCode = -1, Errors = { new SqlError { Kind = SqlErrorKind.Infrastructure, Message = $"Unable to determine connection settings for database {request.DatabaseId} in server {request.ServerId} (server not found)." } } })); } List <ServiceV1> matchingServices = await KubeClient.ServicesV1().List( labelSelector: $"cloud.dimensiondata.daas.server-id = {targetServer.Id},cloud.dimensiondata.daas.service-type = internal", kubeNamespace: KubeOptions.KubeNamespace ); if (matchingServices.Count == 0) { Log.LogWarning("Cannot determine connection string for database {DatabaseId} in server {ServerId} (server's associated Kubernetes Service not found).", request.DatabaseId, request.ServerId ); throw RespondWith(Ok(new SqlResult { ResultCode = -1, Errors = { new SqlError { Kind = SqlErrorKind.Infrastructure, Message = $"Unable to determine connection settings for database {request.DatabaseId} in server {request.ServerId} (server's associated Kubernetes Service not found)." } } })); } ServiceV1 serverService = matchingServices[matchingServices.Count - 1]; (string serverFQDN, int?serverPort) = serverService.GetHostAndPort(portName: "sql-server"); if (serverPort == null) { Log.LogWarning("Cannot determine connection string for database {DatabaseId} in server {ServerId} (cannot find the port named 'sql-server' on server's associated Kubernetes Service).", request.DatabaseId, request.ServerId ); throw RespondWith(Ok(new SqlResult { ResultCode = -1, Errors = { new SqlError { Kind = SqlErrorKind.Infrastructure, Message = $"Unable to determine connection settings for database {request.DatabaseId} in server {request.ServerId} (cannot find the port named 'sql-server' on server's associated Kubernetes Service)." } } })); } Log.LogInformation("Database proxy will connect to SQL Server '{ServerFQDN}' on {ServerPort}.", serverFQDN, serverPort); var connectionStringBuilder = new SqlClient.SqlConnectionStringBuilder { DataSource = $"tcp:{serverFQDN},{serverPort}", }; var serverSettings = targetServer.GetSettings <SqlServerSettings>(); if (request.DatabaseId != MasterDatabaseId) { DatabaseInstance targetDatabase = await DocumentSession.LoadAsync <DatabaseInstance>(request.DatabaseId); if (targetDatabase == null) { Log.LogWarning("Cannot determine connection string for database {DatabaseId} in server {ServerId} (database not found).", request.DatabaseId, request.ServerId ); throw RespondWith(Ok(new SqlResult { ResultCode = -1, Errors = { new SqlError { Kind = SqlErrorKind.Infrastructure, Message = $"Unable to determine connection settings for database {request.DatabaseId} in server {request.ServerId} (database not found)." } } })); } connectionStringBuilder.InitialCatalog = targetDatabase.Name; if (request.ExecuteAsAdminUser) { connectionStringBuilder.UserID = "sa"; connectionStringBuilder.Password = serverSettings.AdminPassword; } else { connectionStringBuilder.UserID = targetDatabase.DatabaseUser; connectionStringBuilder.Password = targetDatabase.DatabasePassword; } } else { connectionStringBuilder.InitialCatalog = "master"; connectionStringBuilder.UserID = "sa"; connectionStringBuilder.Password = serverSettings.AdminPassword; } Log.LogInformation("Successfully determined connection string for database {DatabaseId} ({DatabaseName}) in server {ServerId} ({ServerSqlName}).", request.DatabaseId, connectionStringBuilder.InitialCatalog, request.ServerId, connectionStringBuilder.DataSource ); return(connectionStringBuilder.ConnectionString); }
private void GivenTheServicesAreRegisteredWithKube(ServiceV1 serviceEntries) { _serviceEntries = serviceEntries; }
private async Task <ServiceResources> DeployService(DeployCommand.Types.Service command, string kubeNamespace) { DeploymentV1 deployment = null; StatefulSetV1 statefulSet = null; var secret = await CreateSecret(); if (string.IsNullOrWhiteSpace(command.PersistentStoragePath)) { deployment = await CreateDeployment(); } else { statefulSet = await CreateStatefulSet(); } var service = await CreateService(); var ingress = await CreateServiceIngress(); return(new ServiceResources(deployment, statefulSet, service, ingress, secret)); async Task <SecretV1> CreateSecret() { if (!command.Secrets.Any()) { return(null); } var existingSecret = await kubeApiClient.SecretsV1().Get(command.Name, kubeNamespace); var secretResource = new SecretV1 { Metadata = new ObjectMetaV1 { Name = command.Name, Namespace = kubeNamespace, }, Type = "Opaque", }; if (existingSecret != null) { foreach (var existingData in existingSecret.Data) { secretResource.Data[existingData.Key] = existingData.Value; } } foreach (var secretCommand in command.Secrets.Where(s => s.Value != null)) { secretResource.Data[secretCommand.Name] = Convert.ToBase64String(Encoding.UTF8.GetBytes(secretCommand.Value)); } foreach (var existingSecretNames in secretResource.Data.Keys) { if (command.Secrets.All(secretCommand => secretCommand.Name != existingSecretNames)) { secretResource.Data.Remove(existingSecretNames); } } return(await kubeApiClient.Dynamic().Apply(secretResource, fieldManager: "clud", force: true)); } async Task <DeploymentV1> CreateDeployment() { var deployment = new DeploymentV1 { Metadata = new ObjectMetaV1 { Name = command.Name, Namespace = kubeNamespace, }, Spec = new DeploymentSpecV1 { Selector = new LabelSelectorV1 { MatchLabels = { { KubeNaming.AppLabelKey, command.Name } }, }, Replicas = command.Replicas, Template = new PodTemplateSpecV1 { Metadata = new ObjectMetaV1 { Name = command.Name, Namespace = kubeNamespace, Labels = { { KubeNaming.AppLabelKey, command.Name } } }, Spec = new PodSpecV1 { Containers = { new ContainerV1 { Name = command.Name, Image = DockerImageName(), } }, }, } } }; AddEnvironmentVariables(deployment.Spec.Template.Spec.Containers.Single().Env); return(await kubeApiClient.Dynamic().Apply(deployment, fieldManager: "clud", force: true)); } async Task <StatefulSetV1> CreateStatefulSet() { var statefulSet = new StatefulSetV1 { Metadata = new ObjectMetaV1 { Name = command.Name, Namespace = kubeNamespace, }, Spec = new StatefulSetSpecV1 { Selector = new LabelSelectorV1 { MatchLabels = { { KubeNaming.AppLabelKey, command.Name } }, }, Template = new PodTemplateSpecV1 { Metadata = new ObjectMetaV1 { Name = command.Name, Namespace = kubeNamespace, Labels = { { KubeNaming.AppLabelKey, command.Name } } }, Spec = new PodSpecV1 { Containers = { new ContainerV1 { Name = command.Name, Image = DockerImageName(), VolumeMounts = { new VolumeMountV1 { Name = command.Name, MountPath = command.PersistentStoragePath, } } }, }, }, }, VolumeClaimTemplates = { new PersistentVolumeClaimV1 { Metadata = new ObjectMetaV1 { Name = command.Name, Namespace = kubeNamespace, }, Spec = new PersistentVolumeClaimSpecV1 { AccessModes ={ "ReadWriteOnce" }, Resources = new ResourceRequirementsV1 { Requests ={ { "storage", "100Mi" } }, } } } } } }; AddEnvironmentVariables(statefulSet.Spec.Template.Spec.Containers.Single().Env); return(await kubeApiClient.Dynamic().Apply(statefulSet, fieldManager: "clud", force: true)); } string DockerImageName() { return(command.IsPublicDockerImage ? command.DockerImage : $"{KubeNaming.DockerRegistryLocation}/{command.DockerImage}"); } void AddEnvironmentVariables(List <EnvVarV1> envVarV1s) { envVarV1s.AddRange(command.EnvironmentVariables.Select(env => new EnvVarV1 { Name = env.Name, Value = env.Value })); envVarV1s.AddRange(command.Secrets.Select(secret => new EnvVarV1 { Name = secret.Name, ValueFrom = new EnvVarSourceV1 { SecretKeyRef = new SecretKeySelectorV1 { Name = command.Name, Key = secret.Name, Optional = false, } } })); } async Task <ServiceV1> CreateService() { var service = new ServiceV1 { Metadata = new ObjectMetaV1 { Name = command.Name, Namespace = kubeNamespace, }, Spec = new ServiceSpecV1 { Selector = { { KubeNaming.AppLabelKey, command.Name } }, }, }; if (command.HttpPort != null) { service.Spec.Ports.Add(new ServicePortV1 { Name = KubeNaming.HttpPortName, Protocol = "TCP", Port = command.HttpPort.Value }); } service.Spec.Ports.AddRange(command.TcpPorts.Select(port => new ServicePortV1 { Name = $"tcp-{port}", Protocol = "TCP", Port = port, })); service.Spec.Ports.AddRange(command.UdpPorts.Select(port => new ServicePortV1 { Name = $"udp-{port}", Protocol = "UDP", Port = port, })); return(await kubeApiClient.Dynamic().Apply(service, fieldManager: "clud", force: true)); } async Task <IngressV1Beta1> CreateServiceIngress() { if (command.HttpPort == null) { return(null); } var ingress = new IngressV1Beta1 { Metadata = new ObjectMetaV1 { Name = command.Name, Namespace = kubeNamespace, }, Spec = new IngressSpecV1Beta1 { Rules = { new IngressRuleV1Beta1 { Host = $"{command.Name}-{kubeNamespace}.{cludOptions.BaseHostname}", Http = new HTTPIngressRuleValueV1Beta1 { Paths = { new HTTPIngressPathV1Beta1 { Path = "/", Backend = new IngressBackendV1Beta1 { ServiceName = service.Metadata.Name, ServicePort = KubeNaming.HttpPortName, } } } } } }, }, }; return(await kubeApiClient.Dynamic().Apply(ingress, fieldManager: "clud", force: true)); } }
static async Task <int> Start(StartOptions options) { var client = KubeApiClient.Create(new KubeClientOptions { ApiEndPoint = new Uri("http://localhost:8001"), KubeNamespace = "tempest" }); if (!(await client.NamespacesV1().List()).Items.Exists(x => x.Metadata.Name == "tempest")) { await client.NamespacesV1().Create(new NamespaceV1 { Metadata = new ObjectMetaV1 { Name = "tempest" } }); } if ((await client.ServicesV1().List()).Items.Count != 0) { Console.Error.WriteLine("Tempest services already exist! Shut down existing services to start anew"); return(1); } var svc = new ServiceV1 { Metadata = new ObjectMetaV1 { Name = "tempest", Namespace = "tempest" }, Kind = "Service", Spec = new ServiceSpecV1 { Selector = { ["app"] = "tempest" } } }; svc.Spec.Ports.Add(new ServicePortV1 { Protocol = "TCP", Port = 12345, TargetPort = 12345 }); svc.Spec.Type = "NodePort"; await client.ServicesV1().Create(svc); Console.WriteLine("Generating server certificate"); var serverCert = CertGenerator.Generate(); Console.WriteLine("Generating client certificate"); var clientCert = CertGenerator.Generate(); Console.WriteLine("Done, creating deployment"); await client.DeploymentsV1().Create(new DeploymentV1 { Metadata = new ObjectMetaV1 { Name = "tempest", Namespace = "tempest" }, Kind = "Deployment", Spec = new DeploymentSpecV1 { Replicas = (await client.NodesV1().List()).Items.Count, Selector = new LabelSelectorV1 { MatchLabels = { ["app"] = "tempest" } }, Template = new PodTemplateSpecV1 { Metadata = new ObjectMetaV1 { Labels = { ["app"] = "tempest" } }, Spec = new PodSpecV1 { Containers = { new ContainerV1 { Name = "tempest", Image = "daeken/tempest-server:v1", ImagePullPolicy = "Always", Ports = { new ContainerPortV1 { ContainerPort = 12345 } }, Env = { new EnvVarV1 { Name = "SERVER_PFX", Value = Convert.ToBase64String(serverCert.Export(X509ContentType.Pfx)) }, new EnvVarV1 { Name = "CLIENT_CERT", Value = Convert.ToBase64String(clientCert.Export(X509ContentType.Cert)) } } } } } } } }); Console.WriteLine("Services created, waiting for ready state"); while (true) { var deployment = await client.DeploymentsV1().Get("tempest"); if (deployment.Status.ReadyReplicas != null && deployment.Status.ReadyReplicas == deployment.Status.Replicas) { break; } } var tempestPath = Path.Join(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".tempest"); try { File.Delete(Path.Join(tempestPath, "server.cert")); } catch (Exception) {} try { File.Delete(Path.Join(tempestPath, "client.pfx")); } catch (Exception) {} try { File.Delete(Path.Join(tempestPath, "nodes")); } catch (Exception) {} Directory.CreateDirectory(tempestPath); await using (var fp = File.OpenWrite(Path.Join(tempestPath, "server.cert"))) fp.Write(serverCert.Export(X509ContentType.Cert)); await using (var fp = File.OpenWrite(Path.Join(tempestPath, "client.pfx"))) fp.Write(serverCert.Export(X509ContentType.Pfx)); var ips = new List <string>(); await using (var fp = File.OpenWrite(Path.Join(tempestPath, "nodes"))) { await using var sw = new StreamWriter(fp); foreach (var elem in await client.NodesV1().List()) { var ip = elem.Status.Addresses[2].Address; sw.WriteLine(ip); ips.Add(ip); } } Console.WriteLine($"Tempest started successfully on {(await client.DeploymentsV1().Get("tempest")).Status.Replicas} nodes"); Console.WriteLine("Checking authentication"); var pclient = new Protocol.Client(new TcpClient(ips[0], 12345)); return(0); }