V1Deployment PrepareDeployment(IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
string name = identity.DeploymentName();
var podSpec = this.GetPod(name, identity, module, labels);
var selector = new V1LabelSelector(matchLabels: labels);
var deploymentSpec = new V1DeploymentSpec(replicas: 1, selector: selector, template: podSpec);
var deploymentMeta = new V1ObjectMeta(name: name, labels: labels, annotations: new Dictionary <string, string>());
return(new V1Deployment(metadata: deploymentMeta, spec: deploymentSpec));
}
public V1ServiceList GetService(string namespaceProperty, V1LabelSelector selector)
{
try
{
var serviceSelector = selector.BuildSelector();
logger.LogDebug("Loading referenced service {selector}", serviceSelector);
var service = client.ListNamespacedService(namespaceProperty, labelSelector: serviceSelector);
return(service);
}
catch (HttpOperationException opEx) when(opEx.Response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
return(null);
}
}
V1Deployment PrepareDeployment(IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
string name = identity.DeploymentName();
var podSpec = this.GetPod(name, identity, module, labels);
var selector = new V1LabelSelector(matchLabels: labels);
// Desired status in Deployment should only be Running or Stopped. Assume Running if not Stopped
int replicas = (module.DesiredStatus != ModuleStatus.Stopped) ? 1 : 0;
var deploymentSpec = new V1DeploymentSpec(replicas: replicas, selector: selector, template: podSpec);
var deploymentMeta = new V1ObjectMeta(
name: name,
labels: labels,
annotations: new Dictionary <string, string>(),
ownerReferences: module.Owner.ToOwnerReferences());
return(new V1Deployment(metadata: deploymentMeta, spec: deploymentSpec));
}
public static string BuildSelector(this V1LabelSelector selector)
{
return(string.Join(",", selector.MatchLabels.Select(m => $"{m.Key}={m.Value}")));
}
public IEnumerable <DatabaseServerResource> GetServerResources(string namespaceProperty, V1LabelSelector selector)
{
var servers = k8sService.GetDatabaseServer(namespaceProperty, selector);
foreach (var server in servers.Items)
{
rehydrator.Rehydrate(server);
}
return(servers.Items);
}
public static string ToRelatedSelector(this V1LabelSelector l) => l.MatchLabels.ToRelatedSelector();
private async Task ManageDeployments(V1ServiceList currentServices, V1DeploymentList currentDeployments, EdgeDeploymentDefinition <TConfig> customObject)
{
var desiredModules = ModuleSet.Create(customObject.Spec.ToArray());
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, this.currentModules);
// Pull current configuration from annotations.
Dictionary <string, string> currentV1ServicesFromAnnotations = this.GetCurrentServiceConfig(currentServices);
// strip out edgeAgent so edgeAgent doesn't update itself.
// TODO: remove this filter.
var agentDeploymentName = this.DeploymentName(CoreConstants.EdgeAgentModuleName);
Dictionary <string, string> currentDeploymentsFromAnnotations = this.GetCurrentDeploymentConfig(currentDeployments)
.ToDictionary(
pair => pair.Key,
pair => pair.Value);
var desiredServices = new List <V1Service>();
var desiredDeployments = new List <V1Deployment>();
foreach (KubernetesModule <TConfig> module in customObject.Spec)
{
var moduleId = moduleIdentities[module.Name];
if (string.Equals(module.Type, "docker"))
{
// Default labels
var labels = new Dictionary <string, string>
{
[Constants.K8sEdgeModuleLabel] = KubeUtils.SanitizeLabelValue(moduleId.ModuleId),
[Constants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.deviceId),
[Constants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.iotHubHostname)
};
// Create a Service for every network interface of each module. (label them with hub, device and module id)
Option <V1Service> moduleService = this.GetServiceFromModule(labels, module, moduleId);
moduleService.ForEach(service => desiredServices.Add(service));
// Create a Pod for each module, and a proxy container.
V1PodTemplateSpec v1PodSpec = this.GetPodFromModule(labels, module, moduleId);
// if this is the edge agent's deployment then it needs to run under a specific service account
if (moduleIdentities[module.Name].ModuleId == CoreConstants.EdgeAgentModuleIdentityName)
{
v1PodSpec.Spec.ServiceAccountName = this.serviceAccountName;
}
// Bundle into a deployment
string deploymentName = this.DeploymentName(moduleIdentities[module.Name].ModuleId);
// Deployment data
var deploymentMeta = new V1ObjectMeta(name: deploymentName, labels: labels);
var selector = new V1LabelSelector(matchLabels: labels);
var deploymentSpec = new V1DeploymentSpec(replicas: 1, selector: selector, template: v1PodSpec);
desiredDeployments.Add(new V1Deployment(metadata: deploymentMeta, spec: deploymentSpec));
}
else
{
Events.InvalidModuleType(module);
}
}
// Find current Services/Deployments which need to be removed and updated
var servicesRemoved = new List <V1Service>(currentServices.Items);
servicesRemoved.RemoveAll(s => desiredServices.Exists(i => string.Equals(i.Metadata.Name, s.Metadata.Name)));
var deploymentsRemoved = new List <V1Deployment>(currentDeployments.Items);
deploymentsRemoved.RemoveAll(
d =>
{
return(desiredDeployments.Exists(i => string.Equals(i.Metadata.Name, d.Metadata.Name)));
});
var newServices = new List <V1Service>();
desiredServices.ForEach(
s =>
{
string creationString = JsonConvert.SerializeObject(s);
if (currentV1ServicesFromAnnotations.ContainsKey(s.Metadata.Name))
{
string serviceAnnotation = currentV1ServicesFromAnnotations[s.Metadata.Name];
// If configuration matches, no need to update service
if (string.Equals(serviceAnnotation, creationString))
{
return;
}
if (s.Metadata.Annotations == null)
{
s.Metadata.Annotations = new Dictionary <string, string>();
}
s.Metadata.Annotations[Constants.CreationString] = creationString;
servicesRemoved.Add(s);
newServices.Add(s);
Events.UpdateService(s.Metadata.Name);
}
else
{
if (s.Metadata.Annotations == null)
{
s.Metadata.Annotations = new Dictionary <string, string>();
}
s.Metadata.Annotations[Constants.CreationString] = creationString;
newServices.Add(s);
Events.CreateService(s.Metadata.Name);
}
});
var deploymentsUpdated = new List <V1Deployment>();
var newDeployments = new List <V1Deployment>();
List <V1Deployment> currentDeploymentsList = currentDeployments.Items.ToList();
desiredDeployments.ForEach(
d =>
{
if (currentDeploymentsFromAnnotations.ContainsKey(d.Metadata.Name))
{
V1Deployment current = currentDeploymentsList.Find(i => string.Equals(i.Metadata.Name, d.Metadata.Name));
string currentFromAnnotation = currentDeploymentsFromAnnotations[d.Metadata.Name];
string creationString = JsonConvert.SerializeObject(d);
// If configuration matches, or this is edgeAgent deployment and the images match,
// no need to do update deployment
if (string.Equals(currentFromAnnotation, creationString) ||
(string.Equals(d.Metadata.Name, this.DeploymentName(CoreConstants.EdgeAgentModuleName)) && V1DeploymentEx.ImageEquals(current, d)))
{
return;
}
d.Metadata.ResourceVersion = current.Metadata.ResourceVersion;
if (d.Metadata.Annotations == null)
{
var annotations = new Dictionary <string, string>
{
[Constants.CreationString] = creationString
};
d.Metadata.Annotations = annotations;
}
else
{
d.Metadata.Annotations[Constants.CreationString] = creationString;
}
deploymentsUpdated.Add(d);
Events.UpdateDeployment(d.Metadata.Name);
}
else
{
string creationString = JsonConvert.SerializeObject(d);
var annotations = new Dictionary <string, string>
{
[Constants.CreationString] = creationString
};
d.Metadata.Annotations = annotations;
newDeployments.Add(d);
Events.CreateDeployment(d.Metadata.Name);
}
});
// Remove the old
IEnumerable <Task <V1Status> > removeServiceTasks = servicesRemoved.Select(
i =>
{
Events.DeletingService(i);
return(this.client.DeleteNamespacedServiceAsync(i.Metadata.Name, this.k8sNamespace, new V1DeleteOptions()));
});
await Task.WhenAll(removeServiceTasks);
IEnumerable <Task <V1Status> > removeDeploymentTasks = deploymentsRemoved.Select(
d =>
{
Events.DeletingDeployment(d);
return(this.client.DeleteNamespacedDeployment1Async(d.Metadata.Name, this.k8sNamespace, new V1DeleteOptions(propagationPolicy: "Foreground"), propagationPolicy: "Foreground"));
});
await Task.WhenAll(removeDeploymentTasks);
// Create the new.
IEnumerable <Task <V1Service> > createServiceTasks = newServices.Select(
s =>
{
Events.CreatingService(s);
return(this.client.CreateNamespacedServiceAsync(s, this.k8sNamespace));
});
await Task.WhenAll(createServiceTasks);
IEnumerable <Task <V1Deployment> > createDeploymentTasks = newDeployments.Select(
deployment =>
{
Events.CreatingDeployment(deployment);
return(this.client.CreateNamespacedDeploymentAsync(deployment, this.k8sNamespace));
});
await Task.WhenAll(createDeploymentTasks);
// Update the existing - should only do this when different.
IEnumerable <Task <V1Deployment> > updateDeploymentTasks = deploymentsUpdated.Select(deployment => this.client.ReplaceNamespacedDeploymentAsync(deployment, deployment.Metadata.Name, this.k8sNamespace));
await Task.WhenAll(updateDeploymentTasks);
this.currentModules = desiredModules;
}
public CustomResourceList <DatabaseResource> GetDatabases(string namespaceProperty, V1LabelSelector selector)
{
try
{
var databaseSelector = selector.BuildSelector();
logger.LogDebug("Loading referenced server {selector}", databaseSelector);
var plural = DatabaseOperator.PluralName.ToLower();
var query = client.ListNamespacedCustomObject(DatabaseOperator.ApiVersion.Group, DatabaseOperator.ApiVersion.Version, namespaceProperty, plural, labelSelector: databaseSelector);
var databases = ((JObject)query).ToObject <CustomResourceList <DatabaseResource> >();
return(databases);
}
catch (HttpOperationException opEx) when(opEx.Response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
return(null);
}
}
