public async Task <ModuleSet> DeployModulesAsync(IReadOnlyList <KubernetesModule> modules, ModuleSet currentModules)
{
var desiredModules = ModuleSet.Create(modules.ToArray());
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);
var labels = modules
.ToDictionary(
module => module.Name,
module => new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeModuleLabel] = moduleIdentities[module.Name].DeploymentName(),
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
});
var deviceOnlyLabels = new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
};
var desiredServices = modules
.Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Name], module, labels[module.Name]))
.FilterMap()
.ToList();
V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServices(currentServices, desiredServices);
var desiredDeployments = modules
.Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Name], module, labels[module.Name]))
.ToList();
V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageDeployments(currentDeployments, desiredDeployments);
var desiredPvcs = modules
.Select(module => this.pvcMapper.CreatePersistentVolumeClaims(module, deviceOnlyLabels))
.FilterMap()
.SelectMany(x => x)
.Distinct(KubernetesPvcByValueEqualityComparer);
// Modules may use PVCs created by the user, we get all PVCs and then work on ours.
V1PersistentVolumeClaimList currentPvcList = await this.client.ListNamespacedPersistentVolumeClaimAsync(this.deviceNamespace);
await this.ManagePvcs(currentPvcList, desiredPvcs);
var desiredServiceAccounts = modules
.Select(module => this.serviceAccountMapper.CreateServiceAccount(moduleIdentities[module.Name], labels[module.Name]))
.ToList();
V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);
return(desiredModules);
}
bool IsCreatedByController(V1PersistentVolumeClaim claim)
{
var labels = claim.Metadata?.Labels;
if (labels == null)
{
return(false);
}
if (!labels.ContainsKey(KubernetesConstants.K8sEdgeDeviceLabel))
{
return(false);
}
return(labels[KubernetesConstants.K8sEdgeDeviceLabel] == KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId));
}
public void IgnoreOtherMetadataTest()
{
var pvcWithOwnerRefMetadata = new V1PersistentVolumeClaim
{
Metadata = new V1ObjectMeta(
name: "pvc1",
labels: new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue("device1"),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue("hostname")
},
ownerReferences: new List <V1OwnerReference>
{
new V1OwnerReference("v1", name: "iotedged", kind: "Deployment", uid: "123")
}),
Spec = new V1PersistentVolumeClaimSpec
{
VolumeName = "steve",
Resources = new V1ResourceRequirements {
Requests = new Dictionary <string, ResourceQuantity> {
["storage"] = new ResourceQuantity("10M")
}
}
}
};
var pvcWithoutOwnerRefMetadata = new V1PersistentVolumeClaim
{
Metadata = new V1ObjectMeta(
name: "pvc1",
labels: new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue("device1"),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue("hostname")
}),
Spec = new V1PersistentVolumeClaimSpec
{
VolumeName = "steve",
Resources = new V1ResourceRequirements {
Requests = new Dictionary <string, ResourceQuantity> {
["storage"] = new ResourceQuantity("10M")
}
}
}
};
Assert.False(comparer.Equals(pvcWithOwnerRefMetadata, pvcWithoutOwnerRefMetadata));
}
public void NoAccessModeTest()
{
var x = new V1PersistentVolumeClaim
{
Metadata = new V1ObjectMeta(
name: "pvc1",
labels: new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue("device1"),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue("hostname")
}),
Spec = new V1PersistentVolumeClaimSpec
{
VolumeName = "steve",
Resources = new V1ResourceRequirements {
Requests = new Dictionary <string, ResourceQuantity> {
["storage"] = new ResourceQuantity("10M")
}
}
}
};
var y = new V1PersistentVolumeClaim
{
Metadata = new V1ObjectMeta(
name: "pvc1",
labels: new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue("device1"),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue("hostname")
}),
Spec = new V1PersistentVolumeClaimSpec
{
VolumeName = "steve",
Resources = new V1ResourceRequirements {
Requests = new Dictionary <string, ResourceQuantity> {
["storage"] = new ResourceQuantity("10M")
}
}
}
};
Assert.False(comparer.Equals(x, y));
}
public async Task <ModuleSet> DeployModulesAsync(IList <KubernetesModule> modules, ModuleSet currentModules)
{
var desiredModules = ModuleSet.Create(modules.ToArray());
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);
var labels = modules
.ToDictionary(
module => module.Name,
module => new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeModuleLabel] = moduleIdentities[module.Name].DeploymentName(),
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
});
var desiredServices = modules
.Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Name], module, labels[module.Name]))
.Where(service => service.HasValue)
.Select(service => service.OrDefault())
.ToList();
V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServices(currentServices, desiredServices);
var desiredDeployments = modules
.Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Name], module, labels[module.Name]))
.ToList();
V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageDeployments(currentDeployments, desiredDeployments);
var desiredServiceAccounts = modules
.Select(module => this.serviceAccountMapper.CreateServiceAccount(moduleIdentities[module.Name], labels[module.Name]))
.ToList();
V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);
return(desiredModules);
}
async Task ManageImagePullSecrets(CancellationToken token)
{
var deviceOnlyLabels = new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
};
// Modules may share an image pull secret, so only pick unique ones to add to the dictionary.
List <V1Secret> desiredImagePullSecrets = this.modules
.Select(module => this.configProvider.GetCombinedConfig(module, this.runtimeInfo))
.Select(config => config.ImagePullSecret)
.FilterMap()
.GroupBy(secret => secret.Name)
.Select(secretGroup => this.CreateSecret(secretGroup.First(), deviceOnlyLabels))
.ToList();
V1SecretList currentImagePullSecrets = await this.client.ListNamespacedSecretAsync(this.deviceNamespace, labelSelector : this.deviceSelector, cancellationToken : token);
await this.ManageImagePullSecrets(currentImagePullSecrets, desiredImagePullSecrets, token);
}
public void SanitizeLabelValueFailTest(string raw) => Assert.Throws <InvalidKubernetesNameException>(() => KubeUtils.SanitizeLabelValue(raw));
public void SanitizeLabelValueTest(string expected, string raw) => Assert.Equal(expected, KubeUtils.SanitizeLabelValue(raw));
public KubernetesModule(
string iotHubHostname,
string deviceId,
string edgeDeviceHostName,
string proxyImage,
Option <string> proxyImagePullSecretName,
string proxyConfigPath,
string proxyConfigVolumeName,
string proxyConfigMapName,
string proxyTrustBundlePath,
string proxyTrustBundleVolumeName,
string proxyTrustBundleConfigMapName,
string apiVersion,
string deviceNamespace,
Uri managementUri,
Uri workloadUri,
IEnumerable <global::Docker.DotNet.Models.AuthConfig> dockerAuthConfig,
Option <UpstreamProtocol> upstreamProtocol,
Option <string> productInfo,
PortMapServiceType defaultMapServiceType,
bool enableServiceCallTracing,
bool useMountSourceForVolumeName,
string storageClassName,
Option <uint> persistentVolumeClaimSizeMb,
Option <IWebProxy> proxy,
bool closeOnIdleTimeout,
TimeSpan idleTimeout,
bool useServerHeartbeat,
KubernetesExperimentalFeatures experimentalFeatures,
KubernetesModuleOwner moduleOwner,
bool runAsNonRoot)
{
this.resourceName = new ResourceName(iotHubHostname, deviceId);
this.edgeDeviceHostName = Preconditions.CheckNonWhiteSpace(edgeDeviceHostName, nameof(edgeDeviceHostName));
this.proxyImage = Preconditions.CheckNonWhiteSpace(proxyImage, nameof(proxyImage));
this.proxyImagePullSecretName = proxyImagePullSecretName;
this.proxyConfigPath = Preconditions.CheckNonWhiteSpace(proxyConfigPath, nameof(proxyConfigPath));
this.proxyConfigVolumeName = Preconditions.CheckNonWhiteSpace(proxyConfigVolumeName, nameof(proxyConfigVolumeName));
this.proxyConfigMapName = Preconditions.CheckNonWhiteSpace(proxyConfigMapName, nameof(proxyConfigMapName));
this.proxyTrustBundlePath = Preconditions.CheckNonWhiteSpace(proxyTrustBundlePath, nameof(proxyTrustBundlePath));
this.proxyTrustBundleVolumeName = Preconditions.CheckNonWhiteSpace(proxyTrustBundleVolumeName, nameof(proxyTrustBundleVolumeName));
this.proxyTrustBundleConfigMapName = Preconditions.CheckNonWhiteSpace(proxyTrustBundleConfigMapName, nameof(proxyTrustBundleConfigMapName));
this.apiVersion = Preconditions.CheckNonWhiteSpace(apiVersion, nameof(apiVersion));
this.deviceSelector = $"{Constants.K8sEdgeDeviceLabel}={KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId)},{Constants.K8sEdgeHubNameLabel}={KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)}";
this.deviceNamespace = Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace));
this.managementUri = Preconditions.CheckNotNull(managementUri, nameof(managementUri));
this.workloadUri = Preconditions.CheckNotNull(workloadUri, nameof(workloadUri));
this.dockerAuthConfig = Preconditions.CheckNotNull(dockerAuthConfig, nameof(dockerAuthConfig));
this.upstreamProtocol = Preconditions.CheckNotNull(upstreamProtocol, nameof(upstreamProtocol));
this.productInfo = productInfo;
this.defaultMapServiceType = defaultMapServiceType;
this.enableServiceCallTracing = enableServiceCallTracing;
this.useMountSourceForVolumeName = useMountSourceForVolumeName;
this.storageClassName = storageClassName;
this.persistentVolumeClaimSizeMb = persistentVolumeClaimSizeMb;
this.proxy = proxy;
this.closeOnIdleTimeout = closeOnIdleTimeout;
this.idleTimeout = idleTimeout;
this.useServerHeartbeat = useServerHeartbeat;
this.experimentalFeatures = experimentalFeatures;
this.moduleOwner = moduleOwner;
this.runAsNonRoot = runAsNonRoot;
}
public async Task <EdgeDeploymentStatus> DeployModulesAsync(ModuleSet desiredModules, ModuleSet currentModules)
{
try
{
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);
// having desired modules an no module identities means that we are unable to obtain a list of module identities
if (desiredModules.Modules.Any() && !moduleIdentities.Any())
{
Events.NoModuleIdentities();
return(EdgeDeploymentStatus.Failure("Unable to obtain identities for desired modules"));
}
var labels = desiredModules.Modules
.ToDictionary(
module => module.Key,
module => new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeModuleLabel] = moduleIdentities[module.Key].DeploymentName(),
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
});
var deviceOnlyLabels = new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
};
var desiredServices = desiredModules.Modules
.Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Key], (KubernetesModule)module.Value, labels[module.Key]))
.FilterMap()
.ToList();
V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServices(currentServices, desiredServices);
var desiredDeployments = desiredModules.Modules
.Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Key], (KubernetesModule)module.Value, labels[module.Key]))
.ToList();
V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageDeployments(currentDeployments, desiredDeployments);
var desiredPvcs = desiredModules.Modules
.Select(module => this.pvcMapper.CreatePersistentVolumeClaims((KubernetesModule)module.Value, deviceOnlyLabels))
.FilterMap()
.SelectMany(x => x)
.Distinct(KubernetesPvcByValueEqualityComparer);
// Modules may use PVCs created by the user, we get all PVCs and then work on ours.
V1PersistentVolumeClaimList currentPvcList = await this.client.ListNamespacedPersistentVolumeClaimAsync(this.deviceNamespace);
await this.ManagePvcs(currentPvcList, desiredPvcs);
var desiredServiceAccounts = desiredModules.Modules
.Select(module => this.serviceAccountMapper.CreateServiceAccount((KubernetesModule)module.Value, moduleIdentities[module.Key], labels[module.Key]))
.ToList();
V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);
return(EdgeDeploymentStatus.Success("Successfully deployed"));
}
catch (HttpOperationException e)
{
Events.DeployModulesException(e);
return(EdgeDeploymentStatus.Failure(e));
}
}
private async Task ManageDeployments(V1ServiceList currentServices, V1DeploymentList currentDeployments, EdgeDeploymentDefinition <TConfig> customObject)
{
var desiredModules = ModuleSet.Create(customObject.Spec.ToArray());
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, this.currentModules);
// Pull current configuration from annotations.
Dictionary <string, string> currentV1ServicesFromAnnotations = this.GetCurrentServiceConfig(currentServices);
// strip out edgeAgent so edgeAgent doesn't update itself.
// TODO: remove this filter.
var agentDeploymentName = this.DeploymentName(CoreConstants.EdgeAgentModuleName);
Dictionary <string, string> currentDeploymentsFromAnnotations = this.GetCurrentDeploymentConfig(currentDeployments)
.ToDictionary(
pair => pair.Key,
pair => pair.Value);
var desiredServices = new List <V1Service>();
var desiredDeployments = new List <V1Deployment>();
foreach (KubernetesModule <TConfig> module in customObject.Spec)
{
var moduleId = moduleIdentities[module.Name];
if (string.Equals(module.Type, "docker"))
{
// Default labels
var labels = new Dictionary <string, string>
{
[Constants.K8sEdgeModuleLabel] = KubeUtils.SanitizeLabelValue(moduleId.ModuleId),
[Constants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.deviceId),
[Constants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.iotHubHostname)
};
// Create a Service for every network interface of each module. (label them with hub, device and module id)
Option <V1Service> moduleService = this.GetServiceFromModule(labels, module, moduleId);
moduleService.ForEach(service => desiredServices.Add(service));
// Create a Pod for each module, and a proxy container.
V1PodTemplateSpec v1PodSpec = this.GetPodFromModule(labels, module, moduleId);
// if this is the edge agent's deployment then it needs to run under a specific service account
if (moduleIdentities[module.Name].ModuleId == CoreConstants.EdgeAgentModuleIdentityName)
{
v1PodSpec.Spec.ServiceAccountName = this.serviceAccountName;
}
// Bundle into a deployment
string deploymentName = this.DeploymentName(moduleIdentities[module.Name].ModuleId);
// Deployment data
var deploymentMeta = new V1ObjectMeta(name: deploymentName, labels: labels);
var selector = new V1LabelSelector(matchLabels: labels);
var deploymentSpec = new V1DeploymentSpec(replicas: 1, selector: selector, template: v1PodSpec);
desiredDeployments.Add(new V1Deployment(metadata: deploymentMeta, spec: deploymentSpec));
}
else
{
Events.InvalidModuleType(module);
}
}
// Find current Services/Deployments which need to be removed and updated
var servicesRemoved = new List <V1Service>(currentServices.Items);
servicesRemoved.RemoveAll(s => desiredServices.Exists(i => string.Equals(i.Metadata.Name, s.Metadata.Name)));
var deploymentsRemoved = new List <V1Deployment>(currentDeployments.Items);
deploymentsRemoved.RemoveAll(
d =>
{
return(desiredDeployments.Exists(i => string.Equals(i.Metadata.Name, d.Metadata.Name)));
});
var newServices = new List <V1Service>();
desiredServices.ForEach(
s =>
{
string creationString = JsonConvert.SerializeObject(s);
if (currentV1ServicesFromAnnotations.ContainsKey(s.Metadata.Name))
{
string serviceAnnotation = currentV1ServicesFromAnnotations[s.Metadata.Name];
// If configuration matches, no need to update service
if (string.Equals(serviceAnnotation, creationString))
{
return;
}
if (s.Metadata.Annotations == null)
{
s.Metadata.Annotations = new Dictionary <string, string>();
}
s.Metadata.Annotations[Constants.CreationString] = creationString;
servicesRemoved.Add(s);
newServices.Add(s);
Events.UpdateService(s.Metadata.Name);
}
else
{
if (s.Metadata.Annotations == null)
{
s.Metadata.Annotations = new Dictionary <string, string>();
}
s.Metadata.Annotations[Constants.CreationString] = creationString;
newServices.Add(s);
Events.CreateService(s.Metadata.Name);
}
});
var deploymentsUpdated = new List <V1Deployment>();
var newDeployments = new List <V1Deployment>();
List <V1Deployment> currentDeploymentsList = currentDeployments.Items.ToList();
desiredDeployments.ForEach(
d =>
{
if (currentDeploymentsFromAnnotations.ContainsKey(d.Metadata.Name))
{
V1Deployment current = currentDeploymentsList.Find(i => string.Equals(i.Metadata.Name, d.Metadata.Name));
string currentFromAnnotation = currentDeploymentsFromAnnotations[d.Metadata.Name];
string creationString = JsonConvert.SerializeObject(d);
// If configuration matches, or this is edgeAgent deployment and the images match,
// no need to do update deployment
if (string.Equals(currentFromAnnotation, creationString) ||
(string.Equals(d.Metadata.Name, this.DeploymentName(CoreConstants.EdgeAgentModuleName)) && V1DeploymentEx.ImageEquals(current, d)))
{
return;
}
d.Metadata.ResourceVersion = current.Metadata.ResourceVersion;
if (d.Metadata.Annotations == null)
{
var annotations = new Dictionary <string, string>
{
[Constants.CreationString] = creationString
};
d.Metadata.Annotations = annotations;
}
else
{
d.Metadata.Annotations[Constants.CreationString] = creationString;
}
deploymentsUpdated.Add(d);
Events.UpdateDeployment(d.Metadata.Name);
}
else
{
string creationString = JsonConvert.SerializeObject(d);
var annotations = new Dictionary <string, string>
{
[Constants.CreationString] = creationString
};
d.Metadata.Annotations = annotations;
newDeployments.Add(d);
Events.CreateDeployment(d.Metadata.Name);
}
});
// Remove the old
IEnumerable <Task <V1Status> > removeServiceTasks = servicesRemoved.Select(
i =>
{
Events.DeletingService(i);
return(this.client.DeleteNamespacedServiceAsync(i.Metadata.Name, this.k8sNamespace, new V1DeleteOptions()));
});
await Task.WhenAll(removeServiceTasks);
IEnumerable <Task <V1Status> > removeDeploymentTasks = deploymentsRemoved.Select(
d =>
{
Events.DeletingDeployment(d);
return(this.client.DeleteNamespacedDeployment1Async(d.Metadata.Name, this.k8sNamespace, new V1DeleteOptions(propagationPolicy: "Foreground"), propagationPolicy: "Foreground"));
});
await Task.WhenAll(removeDeploymentTasks);
// Create the new.
IEnumerable <Task <V1Service> > createServiceTasks = newServices.Select(
s =>
{
Events.CreatingService(s);
return(this.client.CreateNamespacedServiceAsync(s, this.k8sNamespace));
});
await Task.WhenAll(createServiceTasks);
IEnumerable <Task <V1Deployment> > createDeploymentTasks = newDeployments.Select(
deployment =>
{
Events.CreatingDeployment(deployment);
return(this.client.CreateNamespacedDeploymentAsync(deployment, this.k8sNamespace));
});
await Task.WhenAll(createDeploymentTasks);
// Update the existing - should only do this when different.
IEnumerable <Task <V1Deployment> > updateDeploymentTasks = deploymentsUpdated.Select(deployment => this.client.ReplaceNamespacedDeploymentAsync(deployment, deployment.Metadata.Name, this.k8sNamespace));
await Task.WhenAll(updateDeploymentTasks);
this.currentModules = desiredModules;
}