public EdgeDeploymentCommand(
string deviceNamespace,
ResourceName resourceName,
IKubernetes client,
IEnumerable <IModule> modules,
IRuntimeInfo runtimeInfo,
ICombinedConfigProvider <CombinedKubernetesConfig> configProvider)
{
this.deviceNamespace = KubeUtils.SanitizeK8sValue(Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace)));
this.resourceName = Preconditions.CheckNotNull(resourceName, nameof(resourceName));
this.client = Preconditions.CheckNotNull(client, nameof(client));
this.modules = Preconditions.CheckNotNull(modules, nameof(modules)).ToList();
this.runtimeInfo = Preconditions.CheckNotNull(runtimeInfo, nameof(runtimeInfo));
this.configProvider = Preconditions.CheckNotNull(configProvider, nameof(configProvider));
this.id = new Lazy <string>(() => this.modules.Aggregate(string.Empty, (prev, module) => module.Name + prev));
this.serializerSettings = new JsonSerializerSettings
{
ContractResolver = new OverrideJsonIgnoreOfBaseClassContractResolver(
new Dictionary <Type, string[]>
{
[typeof(KubernetesModule)] = new[] { nameof(KubernetesModule.Name) }
})
{
// Environment variable (env) property JSON casing should be left alone
NamingStrategy = new CamelCaseNamingStrategy
{
ProcessDictionaryKeys = false
}
}
};
}
V1PersistentVolumeClaim ExtractPvc(Mount mount, IDictionary <string, string> labels)
{
string name = KubeUtils.SanitizeK8sValue(mount.Source);
bool readOnly = mount.ReadOnly;
var persistentVolumeClaimSpec = new V1PersistentVolumeClaimSpec()
{
// What happens if the PV access mode is not compatible with the access we're requesting?
// Deployment will be created and will be in a failed state. The user will see this as
// module running == false.
AccessModes = new List <string> {
readOnly ? "ReadOnlyMany" : "ReadWriteMany"
},
Resources = new V1ResourceRequirements()
{
Requests = new Dictionary <string, ResourceQuantity>()
{
{ "storage", new ResourceQuantity($"{this.persistentVolumeClaimSizeMb}Mi") }
}
},
};
// prefer persistent volume name to storage class name, if both are set.
if (this.persistentVolumeName.HasValue)
{
this.persistentVolumeName.ForEach(volumeName => persistentVolumeClaimSpec.VolumeName = volumeName);
}
else if (this.storageClassName.HasValue)
{
this.storageClassName.ForEach(storageClass => persistentVolumeClaimSpec.StorageClassName = storageClass);
}
return(new V1PersistentVolumeClaim(metadata: new V1ObjectMeta(name: name, labels: labels), spec: persistentVolumeClaimSpec));
}
public async Task <Plan> PlanAsync(
ModuleSet desired,
ModuleSet current,
IRuntimeInfo runtimeInfo,
IImmutableDictionary <string, IModuleIdentity> moduleIdentities)
{
Events.LogDesired(desired);
Events.LogCurrent(current);
Events.LogIdentities(moduleIdentities);
// Check that module names sanitize and remain unique.
var groupedModules = desired.Modules.ToLookup(pair => KubeUtils.SanitizeK8sValue(pair.Key));
if (groupedModules.Any(c => c.Count() > 1))
{
string nameList = groupedModules
.Where(c => c.Count() > 1)
.SelectMany(g => g, (pairs, pair) => pair.Key)
.Join(",");
throw new InvalidIdentityException($"Deployment will cause a name collision in Kubernetes namespace, modules: [{nameList}]");
}
// TODO: improve this so it is generic for all potential module types.
if (!desired.Modules.Values.All(p => p is IModule <DockerConfig>))
{
throw new InvalidModuleException($"Kubernetes deployment currently only handles type={typeof(DockerConfig).FullName}");
}
// This is a workaround for K8s Public Preview Refresh
// TODO: remove this workaround when merging to the main release
desired = new ModuleSet(desired.Modules.Remove(Constants.EdgeAgentModuleName));
current = new ModuleSet(current.Modules.Remove(Constants.EdgeAgentModuleName));
Diff moduleDifference = desired.Diff(current);
Plan plan;
if (!moduleDifference.IsEmpty)
{
// The "Plan" here is very simple - if we have any change, publish all desired modules to a EdgeDeployment CRD.
// The CRD allows us to give the customer a Kubernetes-centric way to see the deployment
// and the status of that deployment through the "edgedeployments" API.
var crdCommand = new EdgeDeploymentCommand(this.deviceNamespace, this.resourceName, this.client, desired.Modules.Values, runtimeInfo, this.configProvider);
var planCommand = await this.commandFactory.WrapAsync(crdCommand);
var planList = new List <ICommand>
{
planCommand
};
Events.PlanCreated(planList);
plan = new Plan(planList);
}
else
{
plan = Plan.Empty;
}
return(plan);
}
static Option <V1ContainerStatus> GetContainerByName(string name, V1Pod pod)
{
string containerName = KubeUtils.SanitizeDNSValue(name);
V1ContainerStatus status = pod.Status?.ContainerStatuses?
.FirstOrDefault(container => string.Equals(container.Name, containerName, StringComparison.OrdinalIgnoreCase));
return(Option.Maybe(status));
}
public async Task <ModuleSet> DeployModulesAsync(IReadOnlyList <KubernetesModule> modules, ModuleSet currentModules)
{
var desiredModules = ModuleSet.Create(modules.ToArray());
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);
var labels = modules
.ToDictionary(
module => module.Name,
module => new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeModuleLabel] = moduleIdentities[module.Name].DeploymentName(),
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
});
var deviceOnlyLabels = new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
};
var desiredServices = modules
.Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Name], module, labels[module.Name]))
.FilterMap()
.ToList();
V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServices(currentServices, desiredServices);
var desiredDeployments = modules
.Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Name], module, labels[module.Name]))
.ToList();
V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageDeployments(currentDeployments, desiredDeployments);
var desiredPvcs = modules
.Select(module => this.pvcMapper.CreatePersistentVolumeClaims(module, deviceOnlyLabels))
.FilterMap()
.SelectMany(x => x)
.Distinct(KubernetesPvcByValueEqualityComparer);
// Modules may use PVCs created by the user, we get all PVCs and then work on ours.
V1PersistentVolumeClaimList currentPvcList = await this.client.ListNamespacedPersistentVolumeClaimAsync(this.deviceNamespace);
await this.ManagePvcs(currentPvcList, desiredPvcs);
var desiredServiceAccounts = modules
.Select(module => this.serviceAccountMapper.CreateServiceAccount(moduleIdentities[module.Name], labels[module.Name]))
.ToList();
V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);
return(desiredModules);
}
public async Task <Plan> PlanAsync(
ModuleSet desired,
ModuleSet current,
IRuntimeInfo runtimeInfo,
IImmutableDictionary <string, IModuleIdentity> moduleIdentities)
{
Events.LogDesired(desired);
Events.LogCurrent(current);
Events.LogIdentities(moduleIdentities);
// Check that module names sanitize and remain unique.
var groupedModules = desired.Modules.GroupBy(pair => KubeUtils.SanitizeK8sValue(pair.Key)).ToArray();
if (groupedModules.Any(c => c.Count() > 1))
{
string nameList = groupedModules.Where(c => c.Count() > 1).SelectMany(g => g, (pairs, pair) => pair.Key).Join(",");
throw new InvalidIdentityException($"Deployment will cause a name collision in Kubernetes namespace, modules: [{nameList}]");
}
// TODO: improve this so it is generic for all potential module types.
if (!desired.Modules.Values.All(p => p is IModule <DockerConfig>))
{
throw new InvalidModuleException($"Kubernetes deployment currently only handles type={typeof(T).FullName}");
}
Diff moduleDifference = desired.Diff(current);
Plan plan;
if (!moduleDifference.IsEmpty)
{
// The "Plan" here is very simple - if we have any change, publish all desired modules to a CRD.
// The CRD allows us to give the customer a Kubernetes-centric way to see the deployment
// and the status of that deployment through the "edgedeployments" API.
var k8sModules = desired.Modules.Select(m => new KubernetesModule <DockerConfig>(m.Value as IModule <DockerConfig>));
var crdCommand = new KubernetesCrdCommand <CombinedDockerConfig>(this.deviceNamespace, this.iotHubHostname, this.deviceId, this.client, k8sModules.ToArray(), Option.Some(runtimeInfo), this.combinedConfigProvider as ICombinedConfigProvider <CombinedDockerConfig>);
var planCommand = await this.commandFactory.WrapAsync(crdCommand);
var planList = new List <ICommand>
{
planCommand
};
Events.PlanCreated(planList);
plan = new Plan(planList);
}
else
{
plan = Plan.Empty;
}
return(plan);
}
bool IsCreatedByController(V1PersistentVolumeClaim claim)
{
var labels = claim.Metadata?.Labels;
if (labels == null)
{
return(false);
}
if (!labels.ContainsKey(KubernetesConstants.K8sEdgeDeviceLabel))
{
return(false);
}
return(labels[KubernetesConstants.K8sEdgeDeviceLabel] == KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId));
}
public void IgnoreOtherMetadataTest()
{
var pvcWithOwnerRefMetadata = new V1PersistentVolumeClaim
{
Metadata = new V1ObjectMeta(
name: "pvc1",
labels: new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue("device1"),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue("hostname")
},
ownerReferences: new List <V1OwnerReference>
{
new V1OwnerReference("v1", name: "iotedged", kind: "Deployment", uid: "123")
}),
Spec = new V1PersistentVolumeClaimSpec
{
VolumeName = "steve",
Resources = new V1ResourceRequirements {
Requests = new Dictionary <string, ResourceQuantity> {
["storage"] = new ResourceQuantity("10M")
}
}
}
};
var pvcWithoutOwnerRefMetadata = new V1PersistentVolumeClaim
{
Metadata = new V1ObjectMeta(
name: "pvc1",
labels: new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue("device1"),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue("hostname")
}),
Spec = new V1PersistentVolumeClaimSpec
{
VolumeName = "steve",
Resources = new V1ResourceRequirements {
Requests = new Dictionary <string, ResourceQuantity> {
["storage"] = new ResourceQuantity("10M")
}
}
}
};
Assert.False(comparer.Equals(pvcWithOwnerRefMetadata, pvcWithoutOwnerRefMetadata));
}
V1PersistentVolumeClaim ExtractPvc(KubernetesModule module, Mount mount, IDictionary <string, string> labels)
{
string volumeName = KubeUtils.SanitizeK8sValue(mount.Source);
string pvcName = KubernetesModule.PvcName(module, mount);
bool readOnly = mount.ReadOnly;
var persistentVolumeClaimSpec = new V1PersistentVolumeClaimSpec()
{
// What happens if the PV access mode is not compatible with the access we're requesting?
// Deployment will be created and will be in a failed state. The user will see this as
// module running == false.
AccessModes = new List <string> {
readOnly ? "ReadOnlyMany" : "ReadWriteMany"
},
Resources = new V1ResourceRequirements()
{
Requests = new Dictionary <string, ResourceQuantity>()
{
{ "storage", new ResourceQuantity($"{this.persistentVolumeClaimSizeMb}Mi") }
}
},
};
if (this.persistentVolumeName.HasValue)
{
string pvName = this.persistentVolumeName.OrDefault();
if (pvName != volumeName)
{
throw new InvalidModuleException(string.Format("The mount name {0} has to be the same as the PV name {1}", volumeName, pvName));
}
persistentVolumeClaimSpec.VolumeName = volumeName;
}
if (this.storageClassName.HasValue)
{
persistentVolumeClaimSpec.StorageClassName = this.storageClassName.OrDefault();
}
var pvcMeta = new V1ObjectMeta(
name: pvcName,
labels: labels,
ownerReferences: module.Owner.ToOwnerReferences());
return(new V1PersistentVolumeClaim(metadata: pvcMeta, spec: persistentVolumeClaimSpec));
}
public void NoAccessModeTest()
{
var x = new V1PersistentVolumeClaim
{
Metadata = new V1ObjectMeta(
name: "pvc1",
labels: new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue("device1"),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue("hostname")
}),
Spec = new V1PersistentVolumeClaimSpec
{
VolumeName = "steve",
Resources = new V1ResourceRequirements {
Requests = new Dictionary <string, ResourceQuantity> {
["storage"] = new ResourceQuantity("10M")
}
}
}
};
var y = new V1PersistentVolumeClaim
{
Metadata = new V1ObjectMeta(
name: "pvc1",
labels: new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue("device1"),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue("hostname")
}),
Spec = new V1PersistentVolumeClaimSpec
{
VolumeName = "steve",
Resources = new V1ResourceRequirements {
Requests = new Dictionary <string, ResourceQuantity> {
["storage"] = new ResourceQuantity("10M")
}
}
}
};
Assert.False(comparer.Equals(x, y));
}
public EdgeDeploymentCommand(
string deviceNamespace,
ResourceName resourceName,
IKubernetes client,
IEnumerable <IModule> desiredmodules,
ModuleSet currentmodules,
IRuntimeInfo runtimeInfo,
ICombinedConfigProvider <CombinedKubernetesConfig> configProvider)
{
this.deviceNamespace = KubeUtils.SanitizeK8sValue(Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace)));
this.resourceName = Preconditions.CheckNotNull(resourceName, nameof(resourceName));
this.client = Preconditions.CheckNotNull(client, nameof(client));
this.modules = Preconditions.CheckNotNull(desiredmodules, nameof(desiredmodules)).ToList();
this.currentmodules = Preconditions.CheckNotNull(currentmodules, nameof(currentmodules));
this.runtimeInfo = Preconditions.CheckNotNull(runtimeInfo, nameof(runtimeInfo));
this.configProvider = Preconditions.CheckNotNull(configProvider, nameof(configProvider));
this.id = new Lazy <string>(() => this.modules.Aggregate(string.Empty, (prev, module) => module.Name + prev));
this.serializerSettings = EdgeDeploymentSerialization.SerializerSettings;
}
public async Task <ModuleSet> DeployModulesAsync(IList <KubernetesModule> modules, ModuleSet currentModules)
{
var desiredModules = ModuleSet.Create(modules.ToArray());
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);
var labels = modules
.ToDictionary(
module => module.Name,
module => new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeModuleLabel] = moduleIdentities[module.Name].DeploymentName(),
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
});
var desiredServices = modules
.Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Name], module, labels[module.Name]))
.Where(service => service.HasValue)
.Select(service => service.OrDefault())
.ToList();
V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServices(currentServices, desiredServices);
var desiredDeployments = modules
.Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Name], module, labels[module.Name]))
.ToList();
V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageDeployments(currentDeployments, desiredDeployments);
var desiredServiceAccounts = modules
.Select(module => this.serviceAccountMapper.CreateServiceAccount(moduleIdentities[module.Name], labels[module.Name]))
.ToList();
V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);
return(desiredModules);
}
async Task ManageImagePullSecrets(CancellationToken token)
{
var deviceOnlyLabels = new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
};
// Modules may share an image pull secret, so only pick unique ones to add to the dictionary.
List <V1Secret> desiredImagePullSecrets = this.modules
.Select(module => this.configProvider.GetCombinedConfig(module, this.runtimeInfo))
.Select(config => config.ImagePullSecret)
.FilterMap()
.GroupBy(secret => secret.Name)
.Select(secretGroup => this.CreateSecret(secretGroup.First(), deviceOnlyLabels))
.ToList();
V1SecretList currentImagePullSecrets = await this.client.ListNamespacedSecretAsync(this.deviceNamespace, labelSelector : this.deviceSelector, cancellationToken : token);
await this.ManageImagePullSecrets(currentImagePullSecrets, desiredImagePullSecrets, token);
}
public KubernetesCrdCommand(string deviceNamespace, string iotHubHostname, string deviceId, IKubernetes client, KubernetesModule <DockerConfig>[] modules, Option <IRuntimeInfo> runtimeInfo, ICombinedConfigProvider <T> combinedConfigProvider)
{
this.deviceNamespace = KubeUtils.SanitizeK8sValue(Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace)));
this.iotHubHostname = KubeUtils.SanitizeK8sValue(Preconditions.CheckNonWhiteSpace(iotHubHostname, nameof(iotHubHostname)));
this.deviceId = KubeUtils.SanitizeK8sValue(Preconditions.CheckNonWhiteSpace(deviceId, nameof(deviceId)));
this.client = Preconditions.CheckNotNull(client, nameof(client));
this.modules = Preconditions.CheckNotNull(modules, nameof(modules));
this.runtimeInfo = Preconditions.CheckNotNull(runtimeInfo, nameof(runtimeInfo));
this.combinedConfigProvider = Preconditions.CheckNotNull(combinedConfigProvider, nameof(combinedConfigProvider));
this.id = new Lazy <string>(() => this.modules.Aggregate(string.Empty, (prev, module) => module.Name + prev));
var deserializerTypesMap = new Dictionary <Type, IDictionary <string, Type> >
{
[typeof(IModule)] = new Dictionary <string, Type>
{
["docker"] = typeof(CombinedDockerConfig)
}
};
this.deploymentSerde = new TypeSpecificSerDe <EdgeDeploymentDefinition <DockerConfig> >(deserializerTypesMap);
}
public bool Equals(V1Deployment x, V1Deployment y)
{
if (ReferenceEquals(x, y))
{
return(true);
}
if (ReferenceEquals(x, null))
{
return(false);
}
if (ReferenceEquals(y, null))
{
return(false);
}
if (x.GetType() != y.GetType())
{
return(false);
}
if (x.Metadata.Name != y.Metadata.Name)
{
return(false);
}
// EdgeAgent deployments are equal when they have identical image sections
if (x.Metadata.Name == KubeUtils.SanitizeK8sValue(CoreConstants.EdgeAgentModuleName))
{
return(V1DeploymentEx.ImageEquals(x, y));
}
// compares by creation string
string xCreationString = GetCreationString(x);
string yCreationString = GetCreationString(y);
return(xCreationString == yCreationString);
}
public void SanitizeDNSDomainFailTest(string raw) => Assert.Throws <InvalidKubernetesNameException>(() => KubeUtils.SanitizeDNSDomain(raw));
public void SanitizeDNSDomainTest(string expected, string raw) => Assert.Equal(expected, KubeUtils.SanitizeDNSDomain(raw));
public void SanitizeDnsValueTest(string expected, string raw) => Assert.Equal(expected, KubeUtils.SanitizeDNSValue(raw));
public void SanitizeK8sValueFailTest(string raw) => Assert.Throws <InvalidKubernetesNameException>(() => KubeUtils.SanitizeK8sValue(raw));
public void SanitizeAnnotationKeyFailTest(string raw) => Assert.Throws <InvalidKubernetesNameException>(() => KubeUtils.SanitizeAnnotationKey(raw));
public void SanitizeAnnotationKeyTest(string expected, string raw) => Assert.Equal(expected, KubeUtils.SanitizeAnnotationKey(raw));
private Option <V1Service> GetServiceFromModule(Dictionary <string, string> labels, KubernetesModule <TConfig> module, IModuleIdentity moduleIdentity)
{
var portList = new List <V1ServicePort>();
Option <Dictionary <string, string> > serviceAnnotations = Option.None <Dictionary <string, string> >();
bool onlyExposedPorts = true;
if (module is IModule <AgentDocker.CombinedDockerConfig> moduleWithDockerConfig)
{
if (moduleWithDockerConfig.Config.CreateOptions?.Labels != null)
{
// Add annotations from Docker labels. This provides the customer a way to assign annotations to services if they want
// to tie backend services to load balancers via an Ingress Controller.
var annotations = new Dictionary <string, string>();
foreach (KeyValuePair <string, string> label in moduleWithDockerConfig.Config.CreateOptions?.Labels)
{
annotations.Add(KubeUtils.SanitizeAnnotationKey(label.Key), label.Value);
}
serviceAnnotations = Option.Some(annotations);
}
// Handle ExposedPorts entries
if (moduleWithDockerConfig.Config?.CreateOptions?.ExposedPorts != null)
{
// Entries in the Exposed Port list just tell Docker that this container wants to listen on that port.
// We interpret this as a "ClusterIP" service type listening on that exposed port, backed by this module.
// Users of this Module's exposed port should be able to find the service by connecting to "<module name>:<port>"
this.GetExposedPorts(moduleWithDockerConfig.Config.CreateOptions.ExposedPorts)
.ForEach(
exposedList =>
exposedList.ForEach((item) => portList.Add(new V1ServicePort(item.Port, name: $"ExposedPort-{item.Port}-{item.Protocol.ToLower()}", protocol: item.Protocol))));
}
// Handle HostConfig PortBindings entries
if (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.PortBindings != null)
{
foreach (KeyValuePair <string, IList <DockerModels.PortBinding> > portBinding in moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.PortBindings)
{
string[] portProtocol = portBinding.Key.Split('/');
if (portProtocol.Length == 2)
{
if (int.TryParse(portProtocol[0], out int port) && this.ValidateProtocol(portProtocol[1], out string protocol))
{
// Entries in Docker portMap wants to expose a port on the host (hostPort) and map it to the container's port (port)
// We interpret that as the pod wants the cluster to expose a port on a public IP (hostPort), and target it to the container's port (port)
foreach (DockerModels.PortBinding hostBinding in portBinding.Value)
{
if (int.TryParse(hostBinding.HostPort, out int hostPort))
{
// If a port entry contains the same "port", then remove it and replace with a new ServicePort that contains a target.
var duplicate = portList.SingleOrDefault(a => a.Port == hostPort);
if (duplicate != default(V1ServicePort))
{
portList.Remove(duplicate);
}
portList.Add(new V1ServicePort(hostPort, name: $"HostPort-{port}-{protocol.ToLower()}", protocol: protocol, targetPort: port));
onlyExposedPorts = false;
}
else
{
Events.PortBindingValue(module, portBinding.Key);
}
}
}
}
}
}
}
if (portList.Count > 0)
{
// Selector: by module name and device name, also how we will label this puppy.
var objectMeta = new V1ObjectMeta(annotations: serviceAnnotations.GetOrElse(() => null), labels: labels, name: KubeUtils.SanitizeDNSValue(moduleIdentity.ModuleId));
// How we manage this service is dependent on the port mappings user asks for.
// If the user tells us to only use ClusterIP ports, we will always set the type to ClusterIP.
// If all we had were exposed ports, we will assume ClusterIP. Otherwise, we use the given value as the default service type
//
// If the user wants to expose the ClusterIPs port externally, they should manually create a service to expose it.
// This gives the user more control as to how they want this to work.
string serviceType;
if (onlyExposedPorts)
{
serviceType = "ClusterIP";
}
else
{
serviceType = this.defaultMapServiceType;
}
return(Option.Some(new V1Service(metadata: objectMeta, spec: new V1ServiceSpec(type: serviceType, ports: portList, selector: labels))));
}
else
{
return(Option.None <V1Service>());
}
}
V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
// Convert docker labels to annotations because docker labels don't have the same restrictions as Kubernetes labels.
Dictionary <string, string> annotations = module.Config.CreateOptions.Labels
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
annotations[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId);
var(proxyContainer, proxyVolumes) = this.PrepareProxyContainer(module);
var(moduleContainer, moduleVolumes) = this.PrepareModuleContainer(name, identity, module);
var imagePullSecrets = new List <Option <string> > {
this.proxyImagePullSecretName, module.Config.AuthConfig.Map(auth => auth.Name)
}
.FilterMap()
.Distinct()
.Select(pullSecretName => new V1LocalObjectReference(pullSecretName))
.ToList();
V1PodSecurityContext securityContext = module.Config.CreateOptions.SecurityContext.GetOrElse(
() => this.runAsNonRoot
? new V1PodSecurityContext {
RunAsNonRoot = true, RunAsUser = 1000
}
: null);
return(new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Name = name,
Labels = labels,
Annotations = annotations
},
Spec = new V1PodSpec
{
Containers = new List <V1Container> {
proxyContainer, moduleContainer
},
Volumes = proxyVolumes.Concat(moduleVolumes).ToList(),
ImagePullSecrets = imagePullSecrets.Any() ? imagePullSecrets : null,
SecurityContext = securityContext,
ServiceAccountName = name,
NodeSelector = module.Config.CreateOptions.NodeSelector.OrDefault()
}
});
}
V1PodTemplateSpec GetPodFromModule(Dictionary <string, string> labels, KubernetesModule <TConfig> module, IModuleIdentity moduleIdentity)
{
if (module is IModule <AgentDocker.CombinedDockerConfig> moduleWithDockerConfig)
{
// pod labels
var podLabels = new Dictionary <string, string>(labels);
// pod annotations
var podAnnotations = new Dictionary <string, string>();
podAnnotations.Add(Constants.K8sEdgeOriginalModuleId, moduleIdentity.ModuleId);
// Convert docker labels to annotations because docker labels don't have the same restrictions as
// Kuberenetes labels.
if (moduleWithDockerConfig.Config.CreateOptions?.Labels != null)
{
foreach (KeyValuePair <string, string> label in moduleWithDockerConfig.Config.CreateOptions?.Labels)
{
podAnnotations.Add(KubeUtils.SanitizeAnnotationKey(label.Key), label.Value);
}
}
// Per container settings:
// exposed ports
Option <List <V1ContainerPort> > exposedPortsOption = (moduleWithDockerConfig.Config?.CreateOptions?.ExposedPorts != null)
? this.GetExposedPorts(moduleWithDockerConfig.Config.CreateOptions.ExposedPorts).Map(
servicePorts =>
servicePorts.Select(tuple => new V1ContainerPort(tuple.Port, protocol: tuple.Protocol)).ToList())
: Option.None <List <V1ContainerPort> >();
// privileged container
Option <V1SecurityContext> securityContext = (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Privileged == true) ? Option.Some(new V1SecurityContext(privileged: true)) : Option.None <V1SecurityContext>();
// Environment Variables.
List <V1EnvVar> env = this.CollectEnv(moduleWithDockerConfig, (KubernetesModuleIdentity)moduleIdentity);
// Bind mounts
(List <V1Volume> volumeList, List <V1VolumeMount> proxyMounts, List <V1VolumeMount> volumeMountList) = this.GetVolumesFromModule(moduleWithDockerConfig).GetOrElse((null, null, null));
// Image
string moduleImage = moduleWithDockerConfig.Config.Image;
var containerList = new List <V1Container>()
{
new V1Container(
KubeUtils.SanitizeDNSValue(moduleIdentity.ModuleId),
env: env,
image: moduleImage,
volumeMounts: volumeMountList,
securityContext: securityContext.GetOrElse(() => null),
ports: exposedPortsOption.GetOrElse(() => null)),
// TODO: Add Proxy container here - configmap for proxy configuration.
new V1Container(
"proxy",
env: env, // TODO: check these for validity for proxy.
image: this.proxyImage,
volumeMounts: proxyMounts)
};
Option <List <V1LocalObjectReference> > imageSecret = moduleWithDockerConfig.Config.AuthConfig.Map(
auth =>
{
var secret = new ImagePullSecret(auth);
var authList = new List <V1LocalObjectReference>
{
new V1LocalObjectReference(secret.Name)
};
return(authList);
});
var modulePodSpec = new V1PodSpec(containerList, volumes: volumeList, imagePullSecrets: imageSecret.GetOrElse(() => null));
var objectMeta = new V1ObjectMeta(labels: podLabels, annotations: podAnnotations);
return(new V1PodTemplateSpec(metadata: objectMeta, spec: modulePodSpec));
}
else
{
Events.InvalidModuleType(module);
}
return(new V1PodTemplateSpec());
}
V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
List <V1EnvVar> envVars = this.CollectEnv(module, identity);
// Convert docker labels to annotations because docker labels don't have the same restrictions as Kubernetes labels.
Dictionary <string, string> annotations = module.Config.CreateOptions.Labels
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
annotations[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId);
// Per container settings:
// exposed ports
Option <List <V1ContainerPort> > exposedPorts = module.Config.CreateOptions.ExposedPorts
.Map(PortExtensions.GetContainerPorts);
// privileged container
Option <V1SecurityContext> securityContext = module.Config.CreateOptions.HostConfig
.Filter(config => config.Privileged)
.Map(config => new V1SecurityContext(privileged: true));
// Bind mounts
(List <V1Volume> volumes, List <V1VolumeMount> proxyMounts, List <V1VolumeMount> volumeMounts) = this.GetVolumesFromModule(module);
var containers = new List <V1Container>
{
new V1Container(
name,
env: envVars,
image: module.Config.Image,
volumeMounts: volumeMounts,
securityContext: securityContext.OrDefault(),
ports: exposedPorts.OrDefault(),
resources: module.Config.CreateOptions.Resources.OrDefault()),
new V1Container(
"proxy",
env: envVars, // TODO: check these for validity for proxy.
image: this.proxyImage,
volumeMounts: proxyMounts)
};
Option <List <V1LocalObjectReference> > imageSecret = module.Config.AuthConfig
.Map(auth => new List <V1LocalObjectReference> {
new V1LocalObjectReference(auth.Name)
});
Option <IDictionary <string, string> > nodeSelector = Option.Maybe(module.Config.CreateOptions).FlatMap(options => options.NodeSelector);
var modulePodSpec = new V1PodSpec(containers, volumes: volumes, imagePullSecrets: imageSecret.OrDefault(), serviceAccountName: name, nodeSelector: nodeSelector.OrDefault());
var objectMeta = new V1ObjectMeta(name: name, labels: labels, annotations: annotations);
return(new V1PodTemplateSpec(objectMeta, modulePodSpec));
}
public KubernetesModule(
string iotHubHostname,
string deviceId,
string edgeDeviceHostName,
string proxyImage,
Option <string> proxyImagePullSecretName,
string proxyConfigPath,
string proxyConfigVolumeName,
string proxyConfigMapName,
string proxyTrustBundlePath,
string proxyTrustBundleVolumeName,
string proxyTrustBundleConfigMapName,
string apiVersion,
string deviceNamespace,
Uri managementUri,
Uri workloadUri,
IEnumerable <global::Docker.DotNet.Models.AuthConfig> dockerAuthConfig,
Option <UpstreamProtocol> upstreamProtocol,
Option <string> productInfo,
PortMapServiceType defaultMapServiceType,
bool enableServiceCallTracing,
string persistentVolumeName,
string storageClassName,
Option <uint> persistentVolumeClaimSizeMb,
Option <IWebProxy> proxy,
bool closeOnIdleTimeout,
TimeSpan idleTimeout,
KubernetesExperimentalFeatures experimentalFeatures,
KubernetesModuleOwner moduleOwner,
bool runAsNonRoot)
{
this.resourceName = new ResourceName(iotHubHostname, deviceId);
this.edgeDeviceHostName = Preconditions.CheckNonWhiteSpace(edgeDeviceHostName, nameof(edgeDeviceHostName));
this.proxyImage = Preconditions.CheckNonWhiteSpace(proxyImage, nameof(proxyImage));
this.proxyImagePullSecretName = proxyImagePullSecretName;
this.proxyConfigPath = Preconditions.CheckNonWhiteSpace(proxyConfigPath, nameof(proxyConfigPath));
this.proxyConfigVolumeName = Preconditions.CheckNonWhiteSpace(proxyConfigVolumeName, nameof(proxyConfigVolumeName));
this.proxyConfigMapName = Preconditions.CheckNonWhiteSpace(proxyConfigMapName, nameof(proxyConfigMapName));
this.proxyTrustBundlePath = Preconditions.CheckNonWhiteSpace(proxyTrustBundlePath, nameof(proxyTrustBundlePath));
this.proxyTrustBundleVolumeName = Preconditions.CheckNonWhiteSpace(proxyTrustBundleVolumeName, nameof(proxyTrustBundleVolumeName));
this.proxyTrustBundleConfigMapName = Preconditions.CheckNonWhiteSpace(proxyTrustBundleConfigMapName, nameof(proxyTrustBundleConfigMapName));
this.apiVersion = Preconditions.CheckNonWhiteSpace(apiVersion, nameof(apiVersion));
this.deviceSelector = $"{Constants.K8sEdgeDeviceLabel}={KubeUtils.SanitizeK8sValue(this.resourceName.DeviceId)},{Constants.K8sEdgeHubNameLabel}={KubeUtils.SanitizeK8sValue(this.resourceName.Hostname)}";
this.deviceNamespace = Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace));
this.managementUri = Preconditions.CheckNotNull(managementUri, nameof(managementUri));
this.workloadUri = Preconditions.CheckNotNull(workloadUri, nameof(workloadUri));
this.dockerAuthConfig = Preconditions.CheckNotNull(dockerAuthConfig, nameof(dockerAuthConfig));
this.upstreamProtocol = Preconditions.CheckNotNull(upstreamProtocol, nameof(upstreamProtocol));
this.productInfo = productInfo;
this.defaultMapServiceType = defaultMapServiceType;
this.enableServiceCallTracing = enableServiceCallTracing;
this.persistentVolumeName = persistentVolumeName;
this.storageClassName = storageClassName;
this.persistentVolumeClaimSizeMb = persistentVolumeClaimSizeMb;
this.proxy = proxy;
this.closeOnIdleTimeout = closeOnIdleTimeout;
this.idleTimeout = idleTimeout;
this.experimentalFeatures = experimentalFeatures;
this.moduleOwner = moduleOwner;
this.runAsNonRoot = runAsNonRoot;
}
protected override void Load(ContainerBuilder builder)
{
// IKubernetesClient
builder.Register(
c =>
{
if (this.enableServiceCallTracing)
{
// enable tracing of k8s requests made by the client
var loggerFactory = c.Resolve <ILoggerFactory>();
ILogger logger = loggerFactory.CreateLogger(typeof(Kubernetes));
ServiceClientTracing.IsEnabled = true;
ServiceClientTracing.AddTracingInterceptor(new DebugTracer(logger));
}
// load the k8s config from KUBECONFIG or $HOME/.kube/config or in-cluster if its available
KubernetesClientConfiguration kubeConfig = Option.Maybe(Environment.GetEnvironmentVariable("KUBECONFIG"))
.Else(() => Option.Maybe(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".kube", "config")))
.Filter(File.Exists)
.Map(path => KubernetesClientConfiguration.BuildConfigFromConfigFile(path))
.GetOrElse(KubernetesClientConfiguration.InClusterConfig);
return(new Kubernetes(kubeConfig));
})
.As <IKubernetes>()
.SingleInstance();
// IModuleClientProvider
builder.Register(
c => new ModuleClientProvider(
c.Resolve <ISdkModuleClientProvider>(),
this.upstreamProtocol,
this.proxy,
this.productInfo.OrDefault(),
this.closeOnIdleTimeout,
this.idleTimeout))
.As <IModuleClientProvider>()
.SingleInstance();
// IModuleManager
builder.Register(c => new ModuleManagementHttpClient(this.managementUri, this.apiVersion, Core.Constants.EdgeletClientApiVersion))
.As <IModuleManager>()
.As <IIdentityManager>()
.SingleInstance();
// IModuleIdentityLifecycleManager
var identityBuilder = new ModuleIdentityProviderServiceBuilder(this.resourceName.Hostname, this.resourceName.DeviceId, this.edgeDeviceHostName);
builder.Register(c => new KubernetesModuleIdentityLifecycleManager(c.Resolve <IIdentityManager>(), identityBuilder, this.workloadUri))
.As <IModuleIdentityLifecycleManager>()
.SingleInstance();
// CombinedKubernetesConfigProvider
builder.Register(
c =>
{
bool enableKubernetesExtensions = this.experimentalFeatures.Enabled && this.experimentalFeatures.EnableExtensions;
return(new CombinedKubernetesConfigProvider(this.dockerAuthConfig, this.workloadUri, this.managementUri, enableKubernetesExtensions));
})
.As <ICombinedConfigProvider <CombinedKubernetesConfig> >()
.SingleInstance();
// ICommandFactory
builder.Register(
c =>
{
var loggerFactory = c.Resolve <ILoggerFactory>();
var kubernetesCommandFactory = new KubernetesCommandFactory();
ICommandFactory factory = new LoggingCommandFactory(kubernetesCommandFactory, loggerFactory);
return(Task.FromResult(factory));
})
.As <Task <ICommandFactory> >()
.SingleInstance();
// IPlanner
builder.Register(
async c =>
{
var configProvider = c.Resolve <ICombinedConfigProvider <CombinedKubernetesConfig> >();
ICommandFactory commandFactory = await c.Resolve <Task <ICommandFactory> >();
IPlanner planner = new KubernetesPlanner(this.deviceNamespace, this.resourceName, c.Resolve <IKubernetes>(), commandFactory, configProvider);
return(planner);
})
.As <Task <IPlanner> >()
.SingleInstance();
// KubernetesRuntimeInfoProvider
builder.Register(c => new KubernetesRuntimeInfoProvider(this.deviceNamespace, c.Resolve <IKubernetes>(), c.Resolve <IModuleManager>()))
.As <IRuntimeInfoProvider>()
.As <IRuntimeInfoSource>()
.SingleInstance();
// KubernetesDeploymentProvider
builder.Register(
c => new KubernetesDeploymentMapper(
this.deviceNamespace,
this.edgeDeviceHostName,
this.proxyImage,
this.proxyConfigPath,
this.proxyConfigVolumeName,
this.proxyConfigMapName,
this.proxyTrustBundlePath,
this.proxyTrustBundleVolumeName,
this.proxyTrustBundleConfigMapName,
this.persistentVolumeName,
this.storageClassName,
this.apiVersion,
this.workloadUri,
this.managementUri))
.As <IKubernetesDeploymentMapper>();
// KubernetesServiceMapper
builder.Register(c => new KubernetesServiceMapper(this.defaultMapServiceType))
.As <IKubernetesServiceMapper>();
// KubernetesPvcMapper
builder.Register(c => new KubernetesPvcMapper(this.persistentVolumeName, this.storageClassName, this.persistentVolumeClaimSizeMb))
.As <IKubernetesPvcMapper>();
// KubernetesServiceAccountProvider
builder.Register(c => new KubernetesServiceAccountMapper())
.As <IKubernetesServiceAccountMapper>();
// EdgeDeploymentController
builder.Register(
c =>
{
var deploymentSelector = $"{Constants.K8sEdgeDeviceLabel}={KubeUtils.SanitizeK8sValue(this.resourceName.DeviceId)},{Constants.K8sEdgeHubNameLabel}={KubeUtils.SanitizeK8sValue(this.resourceName.Hostname)}";
IEdgeDeploymentController watchOperator = new EdgeDeploymentController(
this.resourceName,
deploymentSelector,
this.deviceNamespace,
c.Resolve <IKubernetes>(),
c.Resolve <IModuleIdentityLifecycleManager>(),
c.Resolve <IKubernetesServiceMapper>(),
c.Resolve <IKubernetesDeploymentMapper>(),
c.Resolve <IKubernetesPvcMapper>(),
c.Resolve <IKubernetesServiceAccountMapper>());
return(watchOperator);
})
.As <IEdgeDeploymentController>()
.SingleInstance();
// IEdgeDeploymentOperator
builder.Register(
c =>
{
IEdgeDeploymentOperator watchOperator = new EdgeDeploymentOperator(
this.resourceName,
this.deviceNamespace,
c.Resolve <IKubernetes>(),
c.Resolve <IEdgeDeploymentController>());
return(watchOperator);
})
.As <IEdgeDeploymentOperator>()
.SingleInstance();
// IKubernetesEnvironmentOperator
builder.Register(
c =>
{
IKubernetesEnvironmentOperator watchOperator = new KubernetesEnvironmentOperator(
this.deviceNamespace,
c.Resolve <IRuntimeInfoSource>(),
c.Resolve <IKubernetes>());
return(watchOperator);
})
.As <IKubernetesEnvironmentOperator>()
.SingleInstance();
// Task<IEnvironmentProvider>
builder.Register(
async c =>
{
var moduleStateStore = c.Resolve <IEntityStore <string, ModuleState> >();
var restartPolicyManager = c.Resolve <IRestartPolicyManager>();
IRuntimeInfoProvider runtimeInfoProvider = c.Resolve <IRuntimeInfoProvider>();
IEnvironmentProvider dockerEnvironmentProvider = await DockerEnvironmentProvider.CreateAsync(runtimeInfoProvider, moduleStateStore, restartPolicyManager);
return(dockerEnvironmentProvider);
})
.As <Task <IEnvironmentProvider> >()
.SingleInstance();
}
private VolumeOptions GetVolumesFromModule(IModule <AgentDocker.CombinedDockerConfig> moduleWithDockerConfig)
{
var v1ConfigMapVolumeSource = new V1ConfigMapVolumeSource(null, null, this.proxyConfigVolumeName, null);
var volumeList = new List <V1Volume>
{
new V1Volume(SocketVolumeName, emptyDir: new V1EmptyDirVolumeSource()),
new V1Volume(ConfigVolumeName, configMap: v1ConfigMapVolumeSource)
};
var proxyMountList = new List <V1VolumeMount>
{
new V1VolumeMount(SocketDir, SocketVolumeName)
};
var volumeMountList = new List <V1VolumeMount>(proxyMountList);
proxyMountList.Add(new V1VolumeMount(this.proxyConfigPath, ConfigVolumeName));
if ((moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Binds == null) && (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Mounts == null))
{
return(Option.Some((volumeList, proxyMountList, volumeMountList)));
}
if (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Binds != null)
{
foreach (string bind in moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Binds)
{
string[] bindSubstrings = bind.Split(':');
if (bindSubstrings.Count() >= 2)
{
string name = KubeUtils.SanitizeDNSValue(bindSubstrings[0]);
string type = "DirectoryOrCreate";
string hostPath = bindSubstrings[0];
string mountPath = bindSubstrings[1];
bool readOnly = (bindSubstrings.Count() > 2) && bindSubstrings[2].Contains("ro");
volumeList.Add(new V1Volume(name, hostPath: new V1HostPathVolumeSource(hostPath, type)));
volumeMountList.Add(new V1VolumeMount(mountPath, name, readOnlyProperty: readOnly));
}
}
}
if (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Mounts != null)
{
foreach (DockerModels.Mount mount in moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Mounts)
{
if (mount.Type.Equals("bind", StringComparison.InvariantCultureIgnoreCase))
{
string name = KubeUtils.SanitizeDNSValue(mount.Source);
string type = "DirectoryOrCreate";
string hostPath = mount.Source;
string mountPath = mount.Target;
bool readOnly = mount.ReadOnly;
volumeList.Add(new V1Volume(name, hostPath: new V1HostPathVolumeSource(hostPath, type)));
volumeMountList.Add(new V1VolumeMount(mountPath, name, readOnlyProperty: readOnly));
}
else if (mount.Type.Equals("volume", StringComparison.InvariantCultureIgnoreCase))
{
string name = KubeUtils.SanitizeDNSValue(mount.Source);
string mountPath = mount.Target;
bool readOnly = mount.ReadOnly;
volumeList.Add(new V1Volume(name, emptyDir: new V1EmptyDirVolumeSource()));
volumeMountList.Add(new V1VolumeMount(mountPath, name, readOnlyProperty: readOnly));
}
}
}
return(volumeList.Count > 0 || volumeMountList.Count > 0
? Option.Some((volumeList, proxyMountList, volumeMountList))
: Option.None <(List <V1Volume>, List <V1VolumeMount>, List <V1VolumeMount>)>());
}
public async Task <Plan> PlanAsync(
ModuleSet desired,
ModuleSet current,
IRuntimeInfo runtimeInfo,
IImmutableDictionary <string, IModuleIdentity> moduleIdentities)
{
Events.LogDesired(desired);
// We receive current ModuleSet from Agent based on what it reports (i.e. pods).
// We need to rebuild the current ModuleSet based on deployments (i.e. CRD).
Option <EdgeDeploymentDefinition> activeDeployment = await this.GetCurrentEdgeDeploymentDefinitionAsync();
ModuleSet currentModules =
activeDeployment.Match(
a => ModuleSet.Create(a.Spec.ToArray()),
() => ModuleSet.Empty);
Events.LogCurrent(currentModules);
// Check that module names sanitize and remain unique.
var groupedModules = desired.Modules.ToLookup(pair => KubeUtils.SanitizeK8sValue(pair.Key));
if (groupedModules.Any(c => c.Count() > 1))
{
string nameList = groupedModules
.Where(c => c.Count() > 1)
.SelectMany(g => g, (pairs, pair) => pair.Key)
.Join(",");
throw new InvalidIdentityException($"Deployment will cause a name collision in Kubernetes namespace, modules: [{nameList}]");
}
// TODO: improve this so it is generic for all potential module types.
if (!desired.Modules.Values.All(p => p is IModule <DockerConfig>))
{
throw new InvalidModuleException($"Kubernetes deployment currently only handles type={typeof(DockerConfig).FullName}");
}
Diff moduleDifference = desired.Diff(currentModules);
Plan plan;
if (!moduleDifference.IsEmpty)
{
// The "Plan" here is very simple - if we have any change, publish all desired modules to a EdgeDeployment CRD.
var crdCommand = new EdgeDeploymentCommand(this.resourceName, this.deviceSelector, this.deviceNamespace, this.client, desired.Modules.Values, activeDeployment, runtimeInfo, this.configProvider, this.moduleOwner);
var planCommand = await this.commandFactory.WrapAsync(crdCommand);
var planList = new List <ICommand>
{
planCommand
};
Events.PlanCreated(planList);
plan = new Plan(planList);
}
else
{
plan = Plan.Empty;
}
return(plan);
}
public async Task <EdgeDeploymentStatus> DeployModulesAsync(ModuleSet desiredModules, ModuleSet currentModules)
{
try
{
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);
// having desired modules an no module identities means that we are unable to obtain a list of module identities
if (desiredModules.Modules.Any() && !moduleIdentities.Any())
{
Events.NoModuleIdentities();
return(EdgeDeploymentStatus.Failure("Unable to obtain identities for desired modules"));
}
var labels = desiredModules.Modules
.ToDictionary(
module => module.Key,
module => new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeModuleLabel] = moduleIdentities[module.Key].DeploymentName(),
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
});
var deviceOnlyLabels = new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
[KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
};
var desiredServices = desiredModules.Modules
.Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Key], (KubernetesModule)module.Value, labels[module.Key]))
.FilterMap()
.ToList();
V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServices(currentServices, desiredServices);
var desiredDeployments = desiredModules.Modules
.Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Key], (KubernetesModule)module.Value, labels[module.Key]))
.ToList();
V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageDeployments(currentDeployments, desiredDeployments);
var desiredPvcs = desiredModules.Modules
.Select(module => this.pvcMapper.CreatePersistentVolumeClaims((KubernetesModule)module.Value, deviceOnlyLabels))
.FilterMap()
.SelectMany(x => x)
.Distinct(KubernetesPvcByValueEqualityComparer);
// Modules may use PVCs created by the user, we get all PVCs and then work on ours.
V1PersistentVolumeClaimList currentPvcList = await this.client.ListNamespacedPersistentVolumeClaimAsync(this.deviceNamespace);
await this.ManagePvcs(currentPvcList, desiredPvcs);
var desiredServiceAccounts = desiredModules.Modules
.Select(module => this.serviceAccountMapper.CreateServiceAccount((KubernetesModule)module.Value, moduleIdentities[module.Key], labels[module.Key]))
.ToList();
V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);
await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);
return(EdgeDeploymentStatus.Success("Successfully deployed"));
}
catch (HttpOperationException e)
{
Events.DeployModulesException(e);
return(EdgeDeploymentStatus.Failure(e));
}
}