V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
// Convert docker labels to annotations because docker labels don't have the same restrictions as Kubernetes labels.
Dictionary <string, string> annotations = module.Config.CreateOptions.Labels
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
annotations[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId);
var(proxyContainer, proxyVolumes) = this.PrepareProxyContainer(module);
var(moduleContainer, moduleVolumes) = this.PrepareModuleContainer(name, identity, module);
var imagePullSecrets = new List <Option <string> > {
this.proxyImagePullSecretName, module.Config.AuthConfig.Map(auth => auth.Name)
}
.FilterMap()
.Distinct()
.Select(pullSecretName => new V1LocalObjectReference(pullSecretName))
.ToList();
V1PodSecurityContext securityContext = module.Config.CreateOptions.SecurityContext.GetOrElse(
() => this.runAsNonRoot
? new V1PodSecurityContext {
RunAsNonRoot = true, RunAsUser = 1000
}
: null);
return(new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Name = name,
Labels = labels,
Annotations = annotations
},
Spec = new V1PodSpec
{
Containers = new List <V1Container> {
proxyContainer, moduleContainer
},
Volumes = proxyVolumes.Concat(moduleVolumes).ToList(),
ImagePullSecrets = imagePullSecrets.Any() ? imagePullSecrets : null,
SecurityContext = securityContext,
ServiceAccountName = name,
NodeSelector = module.Config.CreateOptions.NodeSelector.OrDefault()
}
});
}
public void SanitizeAnnotationKeyTest(string expected, string raw) => Assert.Equal(expected, KubeUtils.SanitizeAnnotationKey(raw));
public void SanitizeAnnotationKeyFailTest(string raw) => Assert.Throws <InvalidKubernetesNameException>(() => KubeUtils.SanitizeAnnotationKey(raw));
V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
List <V1EnvVar> envVars = this.CollectEnv(module, identity);
// Convert docker labels to annotations because docker labels don't have the same restrictions as Kubernetes labels.
Dictionary <string, string> annotations = module.Config.CreateOptions.Labels
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
annotations[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId);
// Per container settings:
// exposed ports
Option <List <V1ContainerPort> > exposedPorts = module.Config.CreateOptions.ExposedPorts
.Map(PortExtensions.GetContainerPorts);
// privileged container
Option <V1SecurityContext> securityContext = module.Config.CreateOptions.HostConfig
.Filter(config => config.Privileged)
.Map(config => new V1SecurityContext(privileged: true));
// Bind mounts
(List <V1Volume> volumes, List <V1VolumeMount> proxyMounts, List <V1VolumeMount> volumeMounts) = this.GetVolumesFromModule(module);
var containers = new List <V1Container>
{
new V1Container(
name,
env: envVars,
image: module.Config.Image,
volumeMounts: volumeMounts,
securityContext: securityContext.OrDefault(),
ports: exposedPorts.OrDefault(),
resources: module.Config.CreateOptions.Resources.OrDefault()),
new V1Container(
"proxy",
env: envVars, // TODO: check these for validity for proxy.
image: this.proxyImage,
volumeMounts: proxyMounts)
};
Option <List <V1LocalObjectReference> > imageSecret = module.Config.AuthConfig
.Map(auth => new List <V1LocalObjectReference> {
new V1LocalObjectReference(auth.Name)
});
Option <IDictionary <string, string> > nodeSelector = Option.Maybe(module.Config.CreateOptions).FlatMap(options => options.NodeSelector);
var modulePodSpec = new V1PodSpec(containers, volumes: volumes, imagePullSecrets: imageSecret.OrDefault(), serviceAccountName: name, nodeSelector: nodeSelector.OrDefault());
var objectMeta = new V1ObjectMeta(name: name, labels: labels, annotations: annotations);
return(new V1PodTemplateSpec(objectMeta, modulePodSpec));
}
private Option <V1Service> GetServiceFromModule(Dictionary <string, string> labels, KubernetesModule <TConfig> module, IModuleIdentity moduleIdentity)
{
var portList = new List <V1ServicePort>();
Option <Dictionary <string, string> > serviceAnnotations = Option.None <Dictionary <string, string> >();
bool onlyExposedPorts = true;
if (module is IModule <AgentDocker.CombinedDockerConfig> moduleWithDockerConfig)
{
if (moduleWithDockerConfig.Config.CreateOptions?.Labels != null)
{
// Add annotations from Docker labels. This provides the customer a way to assign annotations to services if they want
// to tie backend services to load balancers via an Ingress Controller.
var annotations = new Dictionary <string, string>();
foreach (KeyValuePair <string, string> label in moduleWithDockerConfig.Config.CreateOptions?.Labels)
{
annotations.Add(KubeUtils.SanitizeAnnotationKey(label.Key), label.Value);
}
serviceAnnotations = Option.Some(annotations);
}
// Handle ExposedPorts entries
if (moduleWithDockerConfig.Config?.CreateOptions?.ExposedPorts != null)
{
// Entries in the Exposed Port list just tell Docker that this container wants to listen on that port.
// We interpret this as a "ClusterIP" service type listening on that exposed port, backed by this module.
// Users of this Module's exposed port should be able to find the service by connecting to "<module name>:<port>"
this.GetExposedPorts(moduleWithDockerConfig.Config.CreateOptions.ExposedPorts)
.ForEach(
exposedList =>
exposedList.ForEach((item) => portList.Add(new V1ServicePort(item.Port, name: $"ExposedPort-{item.Port}-{item.Protocol.ToLower()}", protocol: item.Protocol))));
}
// Handle HostConfig PortBindings entries
if (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.PortBindings != null)
{
foreach (KeyValuePair <string, IList <DockerModels.PortBinding> > portBinding in moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.PortBindings)
{
string[] portProtocol = portBinding.Key.Split('/');
if (portProtocol.Length == 2)
{
if (int.TryParse(portProtocol[0], out int port) && this.ValidateProtocol(portProtocol[1], out string protocol))
{
// Entries in Docker portMap wants to expose a port on the host (hostPort) and map it to the container's port (port)
// We interpret that as the pod wants the cluster to expose a port on a public IP (hostPort), and target it to the container's port (port)
foreach (DockerModels.PortBinding hostBinding in portBinding.Value)
{
if (int.TryParse(hostBinding.HostPort, out int hostPort))
{
// If a port entry contains the same "port", then remove it and replace with a new ServicePort that contains a target.
var duplicate = portList.SingleOrDefault(a => a.Port == hostPort);
if (duplicate != default(V1ServicePort))
{
portList.Remove(duplicate);
}
portList.Add(new V1ServicePort(hostPort, name: $"HostPort-{port}-{protocol.ToLower()}", protocol: protocol, targetPort: port));
onlyExposedPorts = false;
}
else
{
Events.PortBindingValue(module, portBinding.Key);
}
}
}
}
}
}
}
if (portList.Count > 0)
{
// Selector: by module name and device name, also how we will label this puppy.
var objectMeta = new V1ObjectMeta(annotations: serviceAnnotations.GetOrElse(() => null), labels: labels, name: KubeUtils.SanitizeDNSValue(moduleIdentity.ModuleId));
// How we manage this service is dependent on the port mappings user asks for.
// If the user tells us to only use ClusterIP ports, we will always set the type to ClusterIP.
// If all we had were exposed ports, we will assume ClusterIP. Otherwise, we use the given value as the default service type
//
// If the user wants to expose the ClusterIPs port externally, they should manually create a service to expose it.
// This gives the user more control as to how they want this to work.
string serviceType;
if (onlyExposedPorts)
{
serviceType = "ClusterIP";
}
else
{
serviceType = this.defaultMapServiceType;
}
return(Option.Some(new V1Service(metadata: objectMeta, spec: new V1ServiceSpec(type: serviceType, ports: portList, selector: labels))));
}
else
{
return(Option.None <V1Service>());
}
}
V1PodTemplateSpec GetPodFromModule(Dictionary <string, string> labels, KubernetesModule <TConfig> module, IModuleIdentity moduleIdentity)
{
if (module is IModule <AgentDocker.CombinedDockerConfig> moduleWithDockerConfig)
{
// pod labels
var podLabels = new Dictionary <string, string>(labels);
// pod annotations
var podAnnotations = new Dictionary <string, string>();
podAnnotations.Add(Constants.K8sEdgeOriginalModuleId, moduleIdentity.ModuleId);
// Convert docker labels to annotations because docker labels don't have the same restrictions as
// Kuberenetes labels.
if (moduleWithDockerConfig.Config.CreateOptions?.Labels != null)
{
foreach (KeyValuePair <string, string> label in moduleWithDockerConfig.Config.CreateOptions?.Labels)
{
podAnnotations.Add(KubeUtils.SanitizeAnnotationKey(label.Key), label.Value);
}
}
// Per container settings:
// exposed ports
Option <List <V1ContainerPort> > exposedPortsOption = (moduleWithDockerConfig.Config?.CreateOptions?.ExposedPorts != null)
? this.GetExposedPorts(moduleWithDockerConfig.Config.CreateOptions.ExposedPorts).Map(
servicePorts =>
servicePorts.Select(tuple => new V1ContainerPort(tuple.Port, protocol: tuple.Protocol)).ToList())
: Option.None <List <V1ContainerPort> >();
// privileged container
Option <V1SecurityContext> securityContext = (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Privileged == true) ? Option.Some(new V1SecurityContext(privileged: true)) : Option.None <V1SecurityContext>();
// Environment Variables.
List <V1EnvVar> env = this.CollectEnv(moduleWithDockerConfig, (KubernetesModuleIdentity)moduleIdentity);
// Bind mounts
(List <V1Volume> volumeList, List <V1VolumeMount> proxyMounts, List <V1VolumeMount> volumeMountList) = this.GetVolumesFromModule(moduleWithDockerConfig).GetOrElse((null, null, null));
// Image
string moduleImage = moduleWithDockerConfig.Config.Image;
var containerList = new List <V1Container>()
{
new V1Container(
KubeUtils.SanitizeDNSValue(moduleIdentity.ModuleId),
env: env,
image: moduleImage,
volumeMounts: volumeMountList,
securityContext: securityContext.GetOrElse(() => null),
ports: exposedPortsOption.GetOrElse(() => null)),
// TODO: Add Proxy container here - configmap for proxy configuration.
new V1Container(
"proxy",
env: env, // TODO: check these for validity for proxy.
image: this.proxyImage,
volumeMounts: proxyMounts)
};
Option <List <V1LocalObjectReference> > imageSecret = moduleWithDockerConfig.Config.AuthConfig.Map(
auth =>
{
var secret = new ImagePullSecret(auth);
var authList = new List <V1LocalObjectReference>
{
new V1LocalObjectReference(secret.Name)
};
return(authList);
});
var modulePodSpec = new V1PodSpec(containerList, volumes: volumeList, imagePullSecrets: imageSecret.GetOrElse(() => null));
var objectMeta = new V1ObjectMeta(labels: podLabels, annotations: podAnnotations);
return(new V1PodTemplateSpec(metadata: objectMeta, spec: modulePodSpec));
}
else
{
Events.InvalidModuleType(module);
}
return(new V1PodTemplateSpec());
}
Option <V1Service> PrepareService(IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
// Add annotations from Docker labels. This provides the customer a way to assign annotations to services if they want
// to tie backend services to load balancers via an Ingress Controller.
var annotations = module.Config.CreateOptions.Labels
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
// Entries in the Exposed Port list just tell Docker that this container wants to listen on that port.
// We interpret this as a "ClusterIP" service type listening on that exposed port, backed by this module.
// Users of this Module's exposed port should be able to find the service by connecting to "<module name>:<port>"
Option <List <V1ServicePort> > exposedPorts = module.Config.CreateOptions.ExposedPorts
.Map(ports => ports.GetExposedPorts());
// Entries in Docker portMap wants to expose a port on the host (hostPort) and map it to the container's port (port)
// We interpret that as the pod wants the cluster to expose a port on a public IP (hostPort), and target it to the container's port (port)
Option <List <V1ServicePort> > hostPorts = module.Config.CreateOptions.HostConfig
.FlatMap(config => Option.Maybe(config.PortBindings).Map(ports => ports.GetHostPorts()));
bool onlyExposedPorts = !hostPorts.HasValue;
// override exposed ports with host ports
var servicePorts = new Dictionary <int, V1ServicePort>();
exposedPorts.ForEach(ports => ports.ForEach(port => servicePorts[port.Port] = port));
hostPorts.ForEach(ports => ports.ForEach(port => servicePorts[port.Port] = port));
if (servicePorts.Any())
{
// Selector: by module name and device name, also how we will label this puppy.
string name = identity.DeploymentName();
var serviceMeta = new V1ObjectMeta(
name: name,
labels: labels,
annotations: annotations,
ownerReferences: module.Owner.ToOwnerReferences());
// How we manage this service is dependent on the port mappings user asks for.
// If the user tells us to only use ClusterIP ports, we will always set the type to ClusterIP.
// If all we had were exposed ports, we will assume ClusterIP. Otherwise, we use the given value as the default service type
//
// If the user wants to expose the ClusterIPs port externally, they should manually create a service to expose it.
// This gives the user more control as to how they want this to work.
var serviceType = onlyExposedPorts
? PortMapServiceType.ClusterIP
: this.defaultMapServiceType;
return(Option.Some(new V1Service(metadata: serviceMeta, spec: new V1ServiceSpec(type: serviceType.ToString(), ports: servicePorts.Values.ToList(), selector: labels))));
}
return(Option.None <V1Service>());
}
V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
// Convert docker labels to annotations because docker labels don't have the same restrictions as Kubernetes labels.
Dictionary <string, string> annotations = module.Config.CreateOptions.Labels
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
annotations[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId);
var(proxyContainer, proxyVolumes) = this.PrepareProxyContainer(module);
var(moduleContainer, moduleVolumes) = this.PrepareModuleContainer(name, identity, module);
Option <List <V1LocalObjectReference> > imageSecret = module.Config.AuthConfig
.Map(auth => new List <V1LocalObjectReference> {
new V1LocalObjectReference(auth.Name)
});
Option <IDictionary <string, string> > nodeSelector = Option.Maybe(module.Config.CreateOptions).FlatMap(options => options.NodeSelector);
var modulePodSpec = new V1PodSpec(
containers: new List <V1Container> {
proxyContainer, moduleContainer
},
volumes: proxyVolumes.Concat(moduleVolumes).ToList(),
imagePullSecrets: imageSecret.OrDefault(),
serviceAccountName: name,
nodeSelector: nodeSelector.OrDefault());
var objectMeta = new V1ObjectMeta(name: name, labels: labels, annotations: annotations);
return(new V1PodTemplateSpec(objectMeta, modulePodSpec));
}