public V1Service GetServiceByNameAndNamespace(string ServiceName, string _Namespace)
{
try
{
V1ServiceList ServiceList = KClient.ListNamespacedService(_Namespace);
if (ServiceList != null)
{
for (int i = 0; i < ServiceList.Items.Count; ++i)
{
if (ServiceList.Items[i].Name() == ServiceName)
{
return(ServiceList.Items[i]);
}
}
}
return(null);
}
catch (Exception ex)
{
//Return null if pod could not be found and throw exception when connection error occurs
throw ex;
}
}
private static void AddServiceResult(Mock <IKubernetes> mock, V1ServiceList results, string serviceName = null)
{
// If we have a specific service name, this will restrict the method to that
var expectedFilter = $"service-name={serviceName}";
var result = new HttpOperationResponse <V1ServiceList>()
{
Body = results
};
// Kubernetes client uses extension methods to find services, this is the acutal method that will finally be called
mock.Setup(m =>
m.ListNamespacedServiceWithHttpMessagesAsync(It.IsAny <string>(),
It.IsAny <bool?>(),
It.IsAny <string>(),
It.IsAny <string>(),
It.Is <string>(s => string.IsNullOrEmpty(serviceName) || s == expectedFilter),
It.IsAny <int?>(),
It.IsAny <string>(),
It.IsAny <int?>(),
It.IsAny <bool?>(),
It.IsAny <string>(),
It.IsAny <Dictionary <string, List <string> > >(),
It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(result));
}
private Dictionary <string, string> GetCurrentServiceConfig(V1ServiceList currentServices)
{
return(currentServices.Items.ToDictionary(
service =>
{
if (service?.Metadata?.Name != null)
{
return service.Metadata.Name;
}
Events.InvalidCreationString("service", "null service");
throw new NullReferenceException("null service in list");
},
service =>
{
if (service == null)
{
Events.InvalidCreationString("service", "null service");
throw new NullReferenceException("null service in list");
}
if (service.Metadata?.Annotations != null &&
service.Metadata.Annotations.TryGetValue(Constants.CreationString, out string creationString))
{
return creationString;
}
Events.InvalidCreationString(service.Kind, service.Metadata?.Name);
var serviceWithoutStatus = new V1Service(service.ApiVersion, service.Kind, service.Metadata, service.Spec);
return JsonConvert.SerializeObject(serviceWithoutStatus);
}));
}
public async Task PurgeModulesAsync()
{
// Delete all services for current edge deployment
V1ServiceList services = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
var serviceTasks = services.Items
.Select(service => this.client.DeleteNamespacedServiceAsync(service.Metadata.Name, this.deviceNamespace, new V1DeleteOptions()));
await Task.WhenAll(serviceTasks);
// Delete all deployments for current edge deployment
V1DeploymentList deployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
var deploymentTasks = deployments.Items
.Select(
deployment => this.client.DeleteNamespacedDeploymentAsync(
deployment.Metadata.Name,
this.deviceNamespace,
new V1DeleteOptions(propagationPolicy: KubernetesConstants.DefaultDeletePropagationPolicy),
propagationPolicy: KubernetesConstants.DefaultDeletePropagationPolicy));
await Task.WhenAll(deploymentTasks);
V1PersistentVolumeClaimList pvcs = await this.client.ListNamespacedPersistentVolumeClaimAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
var pvcTasks = pvcs.Items
.Select(pvc => this.client.DeleteNamespacedPersistentVolumeClaimAsync(pvc.Metadata.Name, this.deviceNamespace, new V1DeleteOptions()));
await Task.WhenAll(pvcTasks);
// Delete the service account for all deployments
V1ServiceAccountList serviceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
var serviceAccountTasks = serviceAccounts.Items
.Select(service => this.client.DeleteNamespacedServiceAsync(service.Metadata.Name, this.deviceNamespace, new V1DeleteOptions()));
await Task.WhenAll(serviceAccountTasks);
}
public async Task PurgeModulesAsync()
{
// Delete all services for current edge deployment
V1ServiceList services = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
var serviceTasks = services.Items
.Select(service => this.client.DeleteNamespacedServiceAsync(service.Metadata.Name, this.deviceNamespace, new V1DeleteOptions()));
await Task.WhenAll(serviceTasks);
// Delete all deployments for current edge deployment
V1DeploymentList deployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
var deploymentTasks = deployments.Items
.Select(
deployment => this.client.DeleteNamespacedDeployment1Async(
deployment.Metadata.Name,
this.deviceNamespace,
new V1DeleteOptions(propagationPolicy: "Foreground"),
propagationPolicy: "Foreground"));
await Task.WhenAll(deploymentTasks);
// Delete the service account for all deployments
V1ServiceAccountList serviceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
var serviceAccountTasks = serviceAccounts.Items
.Select(service => this.client.DeleteNamespacedServiceAsync(service.Metadata.Name, this.deviceNamespace, new V1DeleteOptions()));
await Task.WhenAll(serviceAccountTasks);
}
public async Task <ModuleSet> DeployModulesAsync(IReadOnlyList <KubernetesModule> modules, ModuleSet currentModules)
{
var desiredModules = ModuleSet.Create(modules.ToArray());
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);
var labels = modules
.ToDictionary(
module => module.Name,
module => new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeModuleLabel] = moduleIdentities[module.Name].DeploymentName(),
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
});
var deviceOnlyLabels = new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
};
var desiredServices = modules
.Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Name], module, labels[module.Name]))
.FilterMap()
.ToList();
V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServices(currentServices, desiredServices);
var desiredDeployments = modules
.Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Name], module, labels[module.Name]))
.ToList();
V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageDeployments(currentDeployments, desiredDeployments);
var desiredPvcs = modules
.Select(module => this.pvcMapper.CreatePersistentVolumeClaims(module, deviceOnlyLabels))
.FilterMap()
.SelectMany(x => x)
.Distinct(KubernetesPvcByValueEqualityComparer);
// Modules may use PVCs created by the user, we get all PVCs and then work on ours.
V1PersistentVolumeClaimList currentPvcList = await this.client.ListNamespacedPersistentVolumeClaimAsync(this.deviceNamespace);
await this.ManagePvcs(currentPvcList, desiredPvcs);
var desiredServiceAccounts = modules
.Select(module => this.serviceAccountMapper.CreateServiceAccount(moduleIdentities[module.Name], labels[module.Name]))
.ToList();
V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);
return(desiredModules);
}
public async Task <IActionResult> GetNamespace([FromRoute] string namespaceName)
{
V1Namespace @namespace = await this.kubernetesClient.ReadNamespaceAsync(namespaceName);
V1ServiceList services = await this.kubernetesClient.ListNamespacedServiceAsync(namespaceParameter : namespaceName, labelSelector : "application=openbrisk");
return(this.Ok(new NamespaceInfo
{
Name = @namespace.Metadata.Name,
FunctionCount = services.Items.Count,
}));
}
public async Task <IActionResult> GetFunctions([FromRoute] string namespaceName)
{
V1ServiceList services = await this.kubernetesClient.ListNamespacedServiceAsync(namespaceName, labelSelector : "application=openbrisk");
return(this.Ok(services.Items.Select(x => new FunctionInfo
{
Name = x.Metadata.Name,
Namespace = x.Metadata.NamespaceProperty,
Runtime = "",
Version = 0,
ReplicaCount = x.Spec.Ports.Count,
InvocationCount = 0,
})));
}
public void TestKubernetesReturnsService()
{
const string expectedUrl = "http://127.0.0.1:80";
const string serviceName = "my-service-working";
const string missingServiceName = "my-service-not-found";
var clusterResult = new V1ServiceList()
{
Items = new List <V1Service>()
{
new V1Service()
{
Spec = new V1ServiceSpec()
{
ClusterIP = "127.0.0.1",
Ports = new List <V1ServicePort>()
{
new V1ServicePort(80, "http")
}
}
}
}
};
var mockClient = new Mock <IKubernetes>();
// and a service result for other service names
AddServiceResult(mockClient, new V1ServiceList());
// Add explicit service result
AddServiceResult(mockClient, clusterResult, serviceName);
CreateMockFactory(mockClient);
var resolver = serviceCollection.BuildServiceProvider().GetService <IServiceResolver>() as KubernetesServiceResolver;
Assert.NotNull(resolver);
var result = resolver.ResolveService(serviceName).Result;
Assert.Equal(expectedUrl, result);
// Ensure that a service doesn't exist has no results
var noResult = resolver.ResolveService(missingServiceName).Result;
Assert.Null(noResult);
VerifyKubernetesServiceSearch(mockClient, serviceName);
VerifyKubernetesServiceSearch(mockClient, missingServiceName);
}
private async Task WatchDeploymentEventsAsync(WatchEventType type, object custom)
{
EdgeDeploymentDefinition <TConfig> edgeDeploymentDefinition;
try
{
string customString = JsonConvert.SerializeObject(custom);
edgeDeploymentDefinition = this.deploymentSerde.Deserialize(customString);
}
catch (Exception e)
{
Events.EdgeDeploymentDeserializeFail(e);
return;
}
// only operate on the device that matches this operator.
if (string.Equals(edgeDeploymentDefinition.Metadata.Name, this.resourceName, StringComparison.OrdinalIgnoreCase))
{
using (await this.watchLock.LockAsync())
{
V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.k8sNamespace, labelSelector : this.deploymentSelector);
V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.k8sNamespace, labelSelector : this.deploymentSelector);
Events.DeploymentStatus(type, this.resourceName);
switch (type)
{
case WatchEventType.Added:
case WatchEventType.Modified:
await this.ManageDeployments(currentServices, currentDeployments, edgeDeploymentDefinition);
break;
case WatchEventType.Deleted:
await this.DeleteDeployments(currentServices, currentDeployments);
break;
case WatchEventType.Error:
Events.DeploymentError();
break;
}
}
}
else
{
Events.DeploymentNameMismatch(edgeDeploymentDefinition.Metadata.Name, this.resourceName);
}
}
private async Task DeleteDeployments(V1ServiceList currentServices, V1DeploymentList currentDeployments)
{
// Delete the deployment.
// Delete any services.
IEnumerable <Task <V1Status> > removeServiceTasks = currentServices.Items.Select(i => this.client.DeleteNamespacedServiceAsync(i.Metadata.Name, this.k8sNamespace, new V1DeleteOptions()));
await Task.WhenAll(removeServiceTasks);
IEnumerable <Task <V1Status> > removeDeploymentTasks = currentDeployments.Items.Select(
d => this.client.DeleteNamespacedDeployment1Async(
d.Metadata.Name,
this.k8sNamespace,
new V1DeleteOptions(propagationPolicy: "Foreground"),
propagationPolicy: "Foreground"));
await Task.WhenAll(removeDeploymentTasks);
this.currentModules = ModuleSet.Empty;
}
public async Task <ModuleSet> DeployModulesAsync(IList <KubernetesModule> modules, ModuleSet currentModules)
{
var desiredModules = ModuleSet.Create(modules.ToArray());
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);
var labels = modules
.ToDictionary(
module => module.Name,
module => new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeModuleLabel] = moduleIdentities[module.Name].DeploymentName(),
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
});
var desiredServices = modules
.Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Name], module, labels[module.Name]))
.Where(service => service.HasValue)
.Select(service => service.OrDefault())
.ToList();
V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServices(currentServices, desiredServices);
var desiredDeployments = modules
.Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Name], module, labels[module.Name]))
.ToList();
V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageDeployments(currentDeployments, desiredDeployments);
var desiredServiceAccounts = modules
.Select(module => this.serviceAccountMapper.CreateServiceAccount(moduleIdentities[module.Name], labels[module.Name]))
.ToList();
V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);
return(desiredModules);
}
public void TestKubernetesNoServicesMatch()
{
const string serviceName = "my-service";
var clusterResult = new V1ServiceList();
var mockClient = new Mock <IKubernetes>();
AddServiceResult(mockClient, clusterResult);
CreateMockFactory(mockClient);
var resolver = serviceCollection.BuildServiceProvider().GetService <IServiceResolver>() as KubernetesServiceResolver;
Assert.NotNull(resolver);
var result = resolver.ResolveService(serviceName).Result;
Assert.Null(result);
// Verify that we searched for a service with the name expected
VerifyKubernetesServiceSearch(mockClient, serviceName);
}
public void TestKubernetesReturnsNoServiceWithoutPort80()
{
const string serviceName = "my-service-but-no-http";
var clusterResult = new V1ServiceList()
{
Items = new List <V1Service>()
{
new V1Service()
{
Metadata = new V1ObjectMeta()
{
Name = "Service without http"
},
Spec = new V1ServiceSpec()
{
ClusterIP = "127.0.0.1",
Ports = new List <V1ServicePort>()
{
new V1ServicePort(81, "tcp") // this isn't an http port, so it wont be returned
}
}
}
}
};
var mockClient = new Mock <IKubernetes>();
AddServiceResult(mockClient, clusterResult, serviceName);
CreateMockFactory(mockClient);
var resolver = serviceCollection.BuildServiceProvider().GetService <IServiceResolver>() as KubernetesServiceResolver;
Assert.NotNull(resolver);
var result = resolver.ResolveService(serviceName).Result;
Assert.Null(result);
}
public void CleanLab(IKubernetes client, string userName)
{
var dList = client.ListNamespacedDeployment(userName);
V1ServiceList sList = null;
V1NetworkPolicyList nList = null;
Networkingv1beta1IngressList iList = null;
if (dList != null && dList.Items.Count > 0)
{
sList = client.ListNamespacedService(userName);
nList = client.ListNamespacedNetworkPolicy(userName);
iList = client.ListNamespacedIngress1(userName);
foreach (var item in dList.Items)
{
client.DeleteNamespacedDeployment(item.Metadata.Name, userName);
}
}
if (sList != null && sList.Items.Count > 0)
{
foreach (var item in sList.Items)
{
client.DeleteNamespacedService(item.Metadata.Name, userName);
}
}
if (iList != null && iList.Items.Count > 0)
{
foreach (var item in iList.Items)
{
client.DeleteNamespacedIngress1(item.Metadata.Name, userName);
}
}
if (nList != null && nList.Items.Count > 0)
{
foreach (var item in nList.Items)
{
client.DeleteNamespacedNetworkPolicy(item.Metadata.Name, userName);
}
}
}
async Task ManageServices(V1ServiceList existing, IEnumerable <V1Service> desired)
{
// find difference between desired and existing services
var diff = FindServiceDiff(desired, existing.Items);
// Update only those services if configurations have not matched
var updatingTask = diff.Updated
.Select(
update =>
{
Events.UpdateService(update.To);
this.serviceMapper.UpdateService(update.To, update.From);
return(this.client.ReplaceNamespacedServiceAsync(update.To, update.To.Metadata.Name, this.deviceNamespace));
});
await Task.WhenAll(updatingTask);
// Delete all existing services that are not in desired list
var removingTasks = diff.Removed
.Select(
name =>
{
Events.DeleteService(name);
return(this.client.DeleteNamespacedServiceAsync(name, this.deviceNamespace));
});
await Task.WhenAll(removingTasks);
// Create new desired services
var addingTasks = diff.Added
.Select(
service =>
{
Events.CreateService(service);
return(this.client.CreateNamespacedServiceAsync(service, this.deviceNamespace));
});
await Task.WhenAll(addingTasks);
}
private async void AssertNoMatchingService(string deviceSelector, string moduleName)
{
V1ServiceList currentServiceList = await this.client.ListServicesAsync(deviceSelector);
Assert.Single(currentServiceList.Items, s => s.Metadata.Name == moduleName);
}
private async void AssertNoServicesExist(string deviceSelector)
{
V1ServiceList currentServices = await this.client.ListServicesAsync(deviceSelector);
Assert.Empty(currentServices.Items);
}
public async Task <EdgeDeploymentStatus> DeployModulesAsync(ModuleSet desiredModules, ModuleSet currentModules)
{
try
{
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);
// having desired modules an no module identities means that we are unable to obtain a list of module identities
if (desiredModules.Modules.Any() && !moduleIdentities.Any())
{
Events.NoModuleIdentities();
return(EdgeDeploymentStatus.Failure("Unable to obtain identities for desired modules"));
}
var labels = desiredModules.Modules
.ToDictionary(
module => module.Key,
module => new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeModuleLabel] = moduleIdentities[module.Key].DeploymentName(),
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
});
var deviceOnlyLabels = new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
};
var desiredServices = desiredModules.Modules
.Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Key], (KubernetesModule)module.Value, labels[module.Key]))
.FilterMap()
.ToList();
V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServices(currentServices, desiredServices);
var desiredDeployments = desiredModules.Modules
.Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Key], (KubernetesModule)module.Value, labels[module.Key]))
.ToList();
V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageDeployments(currentDeployments, desiredDeployments);
var desiredPvcs = desiredModules.Modules
.Select(module => this.pvcMapper.CreatePersistentVolumeClaims((KubernetesModule)module.Value, deviceOnlyLabels))
.FilterMap()
.SelectMany(x => x)
.Distinct(KubernetesPvcByValueEqualityComparer);
// Modules may use PVCs created by the user, we get all PVCs and then work on ours.
V1PersistentVolumeClaimList currentPvcList = await this.client.ListNamespacedPersistentVolumeClaimAsync(this.deviceNamespace);
await this.ManagePvcs(currentPvcList, desiredPvcs);
var desiredServiceAccounts = desiredModules.Modules
.Select(module => this.serviceAccountMapper.CreateServiceAccount((KubernetesModule)module.Value, moduleIdentities[module.Key], labels[module.Key]))
.ToList();
V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);
return(EdgeDeploymentStatus.Success("Successfully deployed"));
}
catch (HttpOperationException e)
{
Events.DeployModulesException(e);
return(EdgeDeploymentStatus.Failure(e));
}
}
public void TestKubernetesReturnsServiceWithPort80()
{
const string expectedUrl = "http://my-host:80";
const string serviceName = "my-service-non-standard-port";
// Define 3 services
// one has the wrong port defined, needs 80
// one has the right port, but no ClusterIP (we only support them at the moment)
// the last one is correct
var clusterResult = new V1ServiceList()
{
Items = new List <V1Service>()
{
// Bad Service
new V1Service()
{
Metadata = new V1ObjectMeta()
{
Name = "Wrong Service",
},
Spec = new V1ServiceSpec()
{
ClusterIP = "127.0.0.1",
Ports = new List <V1ServicePort>()
{
new V1ServicePort(242, "ssh")
}
}
},
new V1Service()
{
Metadata = new V1ObjectMeta()
{
Name = "Right Service but no cluster IP",
},
Spec = new V1ServiceSpec()
{
LoadBalancerIP = "my-host",
Ports = new List <V1ServicePort>()
{
new V1ServicePort(80, "tcp"),
}
}
},
new V1Service()
{
Metadata = new V1ObjectMeta()
{
Name = "Right Service",
},
Spec = new V1ServiceSpec()
{
ClusterIP = "my-host",
Ports = new List <V1ServicePort>()
{
new V1ServicePort(22, "ssh"), // we shouldn't use this port
new V1ServicePort(80, "tcp"),
}
}
}
}
};
var mockClient = new Mock <IKubernetes>();
AddServiceResult(mockClient, clusterResult, serviceName);
CreateMockFactory(mockClient);
var resolver = serviceCollection.BuildServiceProvider().GetService <IServiceResolver>() as KubernetesServiceResolver;
Assert.NotNull(resolver);
var result = resolver.ResolveService(serviceName).Result;
Assert.Equal(expectedUrl, result);
VerifyKubernetesServiceSearch(mockClient, serviceName);
}
public Task <string> ResolveService(string serviceName)
{
var labelFilter = $"{LABEL_FILTER_NAME}={serviceName}";
// Are we configured? if not we won't have setup the client
if (string.IsNullOrEmpty(_kubernetesNamespace) || _kubernetesClient == null)
{
return(Task.FromResult <string>(null));
}
// Attempt to get the list of services in our namespace that match our label
// We must use our Namespace as there could more than one of the service in our cluster, across namespaces
// E.g Alpha and Dev pods are hosted in the same cluster
V1ServiceList list = null;
try
{
list = _kubernetesClient.ListNamespacedService(_kubernetesNamespace, labelSelector: labelFilter);
}
catch (HttpOperationException e)
{
// If we don't have access to query the namespace (e.g default), we will get a forbidden exception
_log.LogWarning($"Failed to query cluster for service {serviceName} due to error. Returning empty result. Error: {e.Message}");
return(Task.FromResult <string>(null));
}
if (list?.Items == null || list.Items.Count == 0)
{
return(Task.FromResult <string>(null));
}
foreach (var item in list.Items)
{
// We require the kubernetes service to define an HTTP Port, and have a CLUSTER IP specified.
// Any service without both of these will be ignored.
var spec = item.Spec;
var httpPort = item
.Spec
.Ports
.FirstOrDefault(p => p.Port == PORT_NUMBER);
if (httpPort == null)
{
_log.LogWarning($"Could not find Port {PORT_NUMBER} for the service '{item.Metadata?.Name}' - ignoring");
}
else if (string.IsNullOrEmpty(item.Spec.ClusterIP))
{
_log.LogWarning($"No clusterIP provided for service {item.Metadata.Name} - ignoring");
}
else
{
// First one found that matches
var url = $"http://{spec.ClusterIP}:{httpPort.Port}";
_log.LogInformation($"Found '{url}' for the service name '{serviceName}' from kubernetes");
return(Task.FromResult(url));
}
}
// No results
return(Task.FromResult <string>(null));
}
private async Task ManageDeployments(V1ServiceList currentServices, V1DeploymentList currentDeployments, EdgeDeploymentDefinition <TConfig> customObject)
{
var desiredModules = ModuleSet.Create(customObject.Spec.ToArray());
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, this.currentModules);
// Pull current configuration from annotations.
Dictionary <string, string> currentV1ServicesFromAnnotations = this.GetCurrentServiceConfig(currentServices);
// strip out edgeAgent so edgeAgent doesn't update itself.
// TODO: remove this filter.
var agentDeploymentName = this.DeploymentName(CoreConstants.EdgeAgentModuleName);
Dictionary <string, string> currentDeploymentsFromAnnotations = this.GetCurrentDeploymentConfig(currentDeployments)
.ToDictionary(
pair => pair.Key,
pair => pair.Value);
var desiredServices = new List <V1Service>();
var desiredDeployments = new List <V1Deployment>();
foreach (KubernetesModule <TConfig> module in customObject.Spec)
{
var moduleId = moduleIdentities[module.Name];
if (string.Equals(module.Type, "docker"))
{
// Default labels
var labels = new Dictionary <string, string>
{
[Constants.K8sEdgeModuleLabel] = KubeUtils.SanitizeLabelValue(moduleId.ModuleId),
[Constants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.deviceId),
[Constants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.iotHubHostname)
};
// Create a Service for every network interface of each module. (label them with hub, device and module id)
Option <V1Service> moduleService = this.GetServiceFromModule(labels, module, moduleId);
moduleService.ForEach(service => desiredServices.Add(service));
// Create a Pod for each module, and a proxy container.
V1PodTemplateSpec v1PodSpec = this.GetPodFromModule(labels, module, moduleId);
// if this is the edge agent's deployment then it needs to run under a specific service account
if (moduleIdentities[module.Name].ModuleId == CoreConstants.EdgeAgentModuleIdentityName)
{
v1PodSpec.Spec.ServiceAccountName = this.serviceAccountName;
}
// Bundle into a deployment
string deploymentName = this.DeploymentName(moduleIdentities[module.Name].ModuleId);
// Deployment data
var deploymentMeta = new V1ObjectMeta(name: deploymentName, labels: labels);
var selector = new V1LabelSelector(matchLabels: labels);
var deploymentSpec = new V1DeploymentSpec(replicas: 1, selector: selector, template: v1PodSpec);
desiredDeployments.Add(new V1Deployment(metadata: deploymentMeta, spec: deploymentSpec));
}
else
{
Events.InvalidModuleType(module);
}
}
// Find current Services/Deployments which need to be removed and updated
var servicesRemoved = new List <V1Service>(currentServices.Items);
servicesRemoved.RemoveAll(s => desiredServices.Exists(i => string.Equals(i.Metadata.Name, s.Metadata.Name)));
var deploymentsRemoved = new List <V1Deployment>(currentDeployments.Items);
deploymentsRemoved.RemoveAll(
d =>
{
return(desiredDeployments.Exists(i => string.Equals(i.Metadata.Name, d.Metadata.Name)));
});
var newServices = new List <V1Service>();
desiredServices.ForEach(
s =>
{
string creationString = JsonConvert.SerializeObject(s);
if (currentV1ServicesFromAnnotations.ContainsKey(s.Metadata.Name))
{
string serviceAnnotation = currentV1ServicesFromAnnotations[s.Metadata.Name];
// If configuration matches, no need to update service
if (string.Equals(serviceAnnotation, creationString))
{
return;
}
if (s.Metadata.Annotations == null)
{
s.Metadata.Annotations = new Dictionary <string, string>();
}
s.Metadata.Annotations[Constants.CreationString] = creationString;
servicesRemoved.Add(s);
newServices.Add(s);
Events.UpdateService(s.Metadata.Name);
}
else
{
if (s.Metadata.Annotations == null)
{
s.Metadata.Annotations = new Dictionary <string, string>();
}
s.Metadata.Annotations[Constants.CreationString] = creationString;
newServices.Add(s);
Events.CreateService(s.Metadata.Name);
}
});
var deploymentsUpdated = new List <V1Deployment>();
var newDeployments = new List <V1Deployment>();
List <V1Deployment> currentDeploymentsList = currentDeployments.Items.ToList();
desiredDeployments.ForEach(
d =>
{
if (currentDeploymentsFromAnnotations.ContainsKey(d.Metadata.Name))
{
V1Deployment current = currentDeploymentsList.Find(i => string.Equals(i.Metadata.Name, d.Metadata.Name));
string currentFromAnnotation = currentDeploymentsFromAnnotations[d.Metadata.Name];
string creationString = JsonConvert.SerializeObject(d);
// If configuration matches, or this is edgeAgent deployment and the images match,
// no need to do update deployment
if (string.Equals(currentFromAnnotation, creationString) ||
(string.Equals(d.Metadata.Name, this.DeploymentName(CoreConstants.EdgeAgentModuleName)) && V1DeploymentEx.ImageEquals(current, d)))
{
return;
}
d.Metadata.ResourceVersion = current.Metadata.ResourceVersion;
if (d.Metadata.Annotations == null)
{
var annotations = new Dictionary <string, string>
{
[Constants.CreationString] = creationString
};
d.Metadata.Annotations = annotations;
}
else
{
d.Metadata.Annotations[Constants.CreationString] = creationString;
}
deploymentsUpdated.Add(d);
Events.UpdateDeployment(d.Metadata.Name);
}
else
{
string creationString = JsonConvert.SerializeObject(d);
var annotations = new Dictionary <string, string>
{
[Constants.CreationString] = creationString
};
d.Metadata.Annotations = annotations;
newDeployments.Add(d);
Events.CreateDeployment(d.Metadata.Name);
}
});
// Remove the old
IEnumerable <Task <V1Status> > removeServiceTasks = servicesRemoved.Select(
i =>
{
Events.DeletingService(i);
return(this.client.DeleteNamespacedServiceAsync(i.Metadata.Name, this.k8sNamespace, new V1DeleteOptions()));
});
await Task.WhenAll(removeServiceTasks);
IEnumerable <Task <V1Status> > removeDeploymentTasks = deploymentsRemoved.Select(
d =>
{
Events.DeletingDeployment(d);
return(this.client.DeleteNamespacedDeployment1Async(d.Metadata.Name, this.k8sNamespace, new V1DeleteOptions(propagationPolicy: "Foreground"), propagationPolicy: "Foreground"));
});
await Task.WhenAll(removeDeploymentTasks);
// Create the new.
IEnumerable <Task <V1Service> > createServiceTasks = newServices.Select(
s =>
{
Events.CreatingService(s);
return(this.client.CreateNamespacedServiceAsync(s, this.k8sNamespace));
});
await Task.WhenAll(createServiceTasks);
IEnumerable <Task <V1Deployment> > createDeploymentTasks = newDeployments.Select(
deployment =>
{
Events.CreatingDeployment(deployment);
return(this.client.CreateNamespacedDeploymentAsync(deployment, this.k8sNamespace));
});
await Task.WhenAll(createDeploymentTasks);
// Update the existing - should only do this when different.
IEnumerable <Task <V1Deployment> > updateDeploymentTasks = deploymentsUpdated.Select(deployment => this.client.ReplaceNamespacedDeploymentAsync(deployment, deployment.Metadata.Name, this.k8sNamespace));
await Task.WhenAll(updateDeploymentTasks);
this.currentModules = desiredModules;
}
