TestModule(IModuleIdentity moduleIdentity, string endpointId, IDeviceListener deviceListener, List <IMessage> receivedMessages)
{
this.moduleIdentity = moduleIdentity;
this.outputName = endpointId;
this.deviceListener = deviceListener;
this.receivedMessages = receivedMessages;
}
public async Task TestGetModulesIdentity_EdgeHubTest()
{
const string Name = "filter";
var symmetricKey = new SymmetricKey();
symmetricKey.PrimaryKey = Convert.ToBase64String(Encoding.UTF8.GetBytes("primarySymmetricKey"));
symmetricKey.SecondaryKey = Convert.ToBase64String(Encoding.UTF8.GetBytes("secondarySymmetricKey"));
var serviceModuleFilterIdentity = new Module("device1", Name)
{
Authentication = new AuthenticationMechanism
{
SymmetricKey = symmetricKey
}
};
var serviceModuleEdgeHubIdentity = new Module("device1", Constants.EdgeHubModuleIdentityName)
{
Authentication = new AuthenticationMechanism
{
SymmetricKey = symmetricKey
}
};
var serviceClient = new Mock <IServiceClient>();
string hostname = "hostname";
string deviceId = "deviceId";
string gatewayHostName = "localhost";
Module[] serviceIdentities = { serviceModuleFilterIdentity, serviceModuleEdgeHubIdentity };
serviceClient.Setup(sc => sc.GetModules()).Returns(Task.FromResult(serviceIdentities.AsEnumerable()));
serviceClient.Setup(sc => sc.CreateModules(It.Is <IEnumerable <string> >(m => !m.Any()))).Returns(Task.FromResult(new Module[0]));
serviceClient.Setup(sc => sc.UpdateModules(It.Is <IEnumerable <Module> >(m => !m.Any()))).Returns(Task.FromResult(new Module[0]));
var testMod = new TestModule(Name, "v1", "test", ModuleStatus.Running, new TestConfig("image"), RestartPolicy.OnUnhealthy, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars);
var edgeHubMod = new TestModule(Constants.EdgeHubModuleName, "v1", "test", ModuleStatus.Running, new TestConfig("image"), RestartPolicy.Always, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars);
IImmutableDictionary <string, IModuleIdentity> modulesIdentities = await new ModuleIdentityLifecycleManager(serviceClient.Object, hostname, deviceId, gatewayHostName)
.GetModuleIdentitiesAsync(ModuleSet.Create(testMod, edgeHubMod), ModuleSet.Empty);
serviceClient.Verify(sc => sc.CreateModules(It.Is <IEnumerable <string> >(m => !m.Any())), Times.Once);
serviceClient.Verify(sc => sc.UpdateModules(It.Is <IEnumerable <Module> >(m => !m.Any())), Times.Once);
Assert.True(modulesIdentities.Count == 2);
IModuleIdentity edgeHubModuleIdentity = modulesIdentities[Constants.EdgeHubModuleName];
Assert.NotNull(edgeHubModuleIdentity);
var edgeHubCreds = edgeHubModuleIdentity.Credentials as ConnectionStringCredentials;
Assert.NotNull(edgeHubCreds);
Assert.Equal("HostName=hostname;DeviceId=deviceId;ModuleId=$edgeHub;SharedAccessKey=cHJpbWFyeVN5bW1ldHJpY0tleQ==", edgeHubCreds.ConnectionString);
IModuleIdentity testModuleIdentity = modulesIdentities[Name];
Assert.NotNull(testModuleIdentity);
var testModCreds = testModuleIdentity.Credentials as ConnectionStringCredentials;
Assert.NotNull(testModCreds);
Assert.Equal("HostName=hostname;DeviceId=deviceId;ModuleId=filter;SharedAccessKey=cHJpbWFyeVN5bW1ldHJpY0tleQ==;GatewayHostName=localhost", testModCreds.ConnectionString);
}
public void Test_CreateIdentity_WithEdgelet_SetAuthScheme_ShouldCreateIdentity(
string iotHubHostName,
string deviceId,
string moduleId,
string generationId,
string edgeletUri,
string authScheme = "sasToken")
{
// Arrange
var builder = new ModuleIdentityProviderServiceBuilder(iotHubHostName, deviceId);
// Act
IModuleIdentity identity = builder.Create(moduleId, generationId, edgeletUri, authScheme);
// Assert
Assert.Equal(iotHubHostName, identity.IotHubHostname);
Assert.Equal(deviceId, identity.DeviceId);
Assert.Equal(moduleId, identity.ModuleId);
var creds = identity.Credentials as IdentityProviderServiceCredentials;
Assert.NotNull(creds);
Assert.Equal(edgeletUri, creds.ProviderUri);
Assert.Equal(authScheme, creds.AuthScheme);
Assert.Equal(generationId, creds.ModuleGenerationId);
Assert.Equal(Option.None <string>(), creds.Version);
}
V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
// Convert docker labels to annotations because docker labels don't have the same restrictions as Kubernetes labels.
Dictionary <string, string> annotations = module.Config.CreateOptions.Labels
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
annotations[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId);
var(proxyContainer, proxyVolumes) = this.PrepareProxyContainer(module);
var(moduleContainer, moduleVolumes) = this.PrepareModuleContainer(name, identity, module);
Option <List <V1LocalObjectReference> > imageSecret = module.Config.AuthConfig
.Map(auth => new List <V1LocalObjectReference> {
new V1LocalObjectReference(auth.Name)
});
Option <IDictionary <string, string> > nodeSelector = Option.Maybe(module.Config.CreateOptions).FlatMap(options => options.NodeSelector);
var modulePodSpec = new V1PodSpec(
containers: new List <V1Container> {
proxyContainer, moduleContainer
},
volumes: proxyVolumes.Concat(moduleVolumes).ToList(),
imagePullSecrets: imageSecret.OrDefault(),
serviceAccountName: name,
nodeSelector: nodeSelector.OrDefault());
var objectMeta = new V1ObjectMeta(name: name, labels: labels, annotations: annotations);
return(new V1PodTemplateSpec(objectMeta, modulePodSpec));
}
public static CreateOrUpdateCommand BuildCreate(
IModuleManager moduleManager,
IModule module,
IModuleIdentity identity,
IConfigSource configSource,
object settings) =>
Build(moduleManager, module, identity, configSource, settings, Operation.Create);
public static CreateOrUpdateCommand BuildUpdate(
IModuleManager moduleManager,
IModule module,
IModuleIdentity identity,
IConfigSource configSource,
object settings,
bool start) =>
Build(moduleManager, module, identity, configSource, settings, start ? Operation.UpdateAndStart : Operation.Update);
public Option <V1Service> CreateService(IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
Option <V1Service> service = this.PrepareService(identity, module, labels);
service.ForEach(s => s.Metadata.Annotations[KubernetesConstants.CreationString] = JsonConvert.SerializeObject(s));
return(service);
}
public V1Deployment CreateDeployment(IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
var deployment = this.PrepareDeployment(identity, module, labels);
deployment.Metadata.Annotations[KubernetesConstants.CreationString] = JsonConvert.SerializeObject(deployment);
return(deployment);
}
public static CreateOrUpdateCommand BuildCreate(
IModuleManager moduleManager,
IModule module,
IModuleIdentity identity,
IConfigSource configSource,
object settings,
string edgeDeviceHostname,
Option <string> parentEdgeHostname) =>
Build(moduleManager, module, identity, configSource, settings, Operation.Create, edgeDeviceHostname, parentEdgeHostname);
List <V1EnvVar> CollectEnv(KubernetesModule module, IModuleIdentity identity)
{
var envList = module.Env.Select(env => new V1EnvVar(env.Key, env.Value.Value)).ToList();
char[] envSplit = { '=' };
if (module.Config?.CreateOptions?.Env != null)
{
foreach (string hostEnv in module.Config?.CreateOptions?.Env)
{
string[] keyValue = hostEnv.Split(envSplit, 2);
if (keyValue.Length == 2)
{
envList.Add(new V1EnvVar(keyValue[0], keyValue[1]));
}
}
}
envList.Add(new V1EnvVar(CoreConstants.IotHubHostnameVariableName, identity.IotHubHostname));
envList.Add(new V1EnvVar(CoreConstants.EdgeletAuthSchemeVariableName, "sasToken"));
envList.Add(new V1EnvVar(Logger.RuntimeLogLevelEnvKey, Logger.GetLogLevel().ToString()));
envList.Add(new V1EnvVar(CoreConstants.EdgeletWorkloadUriVariableName, this.workloadUri.ToString()));
if (identity.Credentials is IdentityProviderServiceCredentials creds)
{
envList.Add(new V1EnvVar(CoreConstants.EdgeletModuleGenerationIdVariableName, creds.ModuleGenerationId));
}
envList.Add(new V1EnvVar(CoreConstants.DeviceIdVariableName, identity.DeviceId));
envList.Add(new V1EnvVar(CoreConstants.ModuleIdVariableName, identity.ModuleId));
envList.Add(new V1EnvVar(CoreConstants.EdgeletApiVersionVariableName, this.workloadApiVersion));
if (string.Equals(identity.ModuleId, CoreConstants.EdgeAgentModuleIdentityName))
{
envList.Add(new V1EnvVar(CoreConstants.ModeKey, CoreConstants.KubernetesMode));
envList.Add(new V1EnvVar(CoreConstants.EdgeletManagementUriVariableName, this.managementUri.ToString()));
envList.Add(new V1EnvVar(CoreConstants.NetworkIdKey, "azure-iot-edge"));
envList.Add(new V1EnvVar(KubernetesConstants.ProxyImageEnvKey, this.proxyImage));
envList.Add(new V1EnvVar(KubernetesConstants.ProxyConfigPathEnvKey, this.proxyConfigPath));
envList.Add(new V1EnvVar(KubernetesConstants.ProxyConfigVolumeEnvKey, this.proxyConfigVolumeName));
envList.Add(new V1EnvVar(KubernetesConstants.ProxyConfigMapNameEnvKey, this.proxyConfigMapName));
envList.Add(new V1EnvVar(KubernetesConstants.ProxyTrustBundlePathEnvKey, this.proxyTrustBundlePath));
envList.Add(new V1EnvVar(KubernetesConstants.ProxyTrustBundleVolumeEnvKey, this.proxyTrustBundleVolumeName));
envList.Add(new V1EnvVar(KubernetesConstants.ProxyTrustBundleConfigMapEnvKey, this.proxyTrustBundleConfigMapName));
envList.Add(new V1EnvVar(KubernetesConstants.K8sNamespaceKey, this.deviceNamespace));
}
if (string.Equals(identity.ModuleId, CoreConstants.EdgeAgentModuleIdentityName) ||
string.Equals(identity.ModuleId, CoreConstants.EdgeHubModuleIdentityName))
{
envList.Add(new V1EnvVar(CoreConstants.EdgeDeviceHostNameKey, this.edgeHostname));
}
else
{
envList.Add(new V1EnvVar(CoreConstants.GatewayHostnameVariableName, EdgeHubHostname));
}
return(envList);
}
V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
List <V1EnvVar> envVars = this.CollectEnv(module, identity);
// Convert docker labels to annotations because docker labels don't have the same restrictions as Kubernetes labels.
var annotations = Option.Maybe(module.Config.CreateOptions?.Labels)
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
annotations[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId);
// Per container settings:
// exposed ports
Option <List <V1ContainerPort> > exposedPorts = Option.Maybe(module.Config?.CreateOptions?.ExposedPorts)
.Map(PortExtensions.GetContainerPorts);
// privileged container
Option <V1SecurityContext> securityContext = Option.Maybe(module.Config?.CreateOptions?.HostConfig?.Privileged)
.Map(config => new V1SecurityContext(privileged: true));
// Bind mounts
(List <V1Volume> volumes, List <V1VolumeMount> proxyMounts, List <V1VolumeMount> volumeMounts) = this.GetVolumesFromModule(module);
var containers = new List <V1Container>
{
new V1Container(
name,
env: envVars,
image: module.Config.Image,
volumeMounts: volumeMounts,
securityContext: securityContext.OrDefault(),
ports: exposedPorts.OrDefault()),
new V1Container(
"proxy",
env: envVars, // TODO: check these for validity for proxy.
image: this.proxyImage,
volumeMounts: proxyMounts)
};
Option <List <V1LocalObjectReference> > imageSecret = module.Config.AuthConfig.Map(
auth =>
{
var secret = new ImagePullSecret(auth);
var authList = new List <V1LocalObjectReference>
{
new V1LocalObjectReference(secret.Name)
};
return(authList);
});
var modulePodSpec = new V1PodSpec(containers, volumes: volumes, imagePullSecrets: imageSecret.OrDefault(), serviceAccountName: name);
var objectMeta = new V1ObjectMeta(name: name, labels: labels, annotations: annotations);
return(new V1PodTemplateSpec(objectMeta, modulePodSpec));
}
public V1ServiceAccount CreateServiceAccount(IModuleIdentity identity, IDictionary <string, string> labels)
{
string name = identity.DeploymentName();
var annotations = new Dictionary <string, string>
{
[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId)
};
var metadata = new V1ObjectMeta(annotations, name: name, labels: labels);
return(new V1ServiceAccount(metadata: metadata));
}
Option <V1Service> PrepareService(IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
// Add annotations from Docker labels. This provides the customer a way to assign annotations to services if they want
// to tie backend services to load balancers via an Ingress Controller.
var annotations = module.Config.CreateOptions.Labels
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
// Entries in the Exposed Port list just tell Docker that this container wants to listen on that port.
// We interpret this as a "ClusterIP" service type listening on that exposed port, backed by this module.
// Users of this Module's exposed port should be able to find the service by connecting to "<module name>:<port>"
Option <List <V1ServicePort> > exposedPorts = module.Config.CreateOptions.ExposedPorts
.Map(ports => ports.GetExposedPorts());
// Entries in Docker portMap wants to expose a port on the host (hostPort) and map it to the container's port (port)
// We interpret that as the pod wants the cluster to expose a port on a public IP (hostPort), and target it to the container's port (port)
Option <List <V1ServicePort> > hostPorts = module.Config.CreateOptions.HostConfig
.FlatMap(config => Option.Maybe(config.PortBindings).Map(ports => ports.GetHostPorts()));
bool onlyExposedPorts = !hostPorts.HasValue;
// override exposed ports with host ports
var servicePorts = new Dictionary <int, V1ServicePort>();
exposedPorts.ForEach(ports => ports.ForEach(port => servicePorts[port.Port] = port));
hostPorts.ForEach(ports => ports.ForEach(port => servicePorts[port.Port] = port));
if (servicePorts.Any())
{
// Selector: by module name and device name, also how we will label this puppy.
string name = identity.DeploymentName();
var serviceMeta = new V1ObjectMeta(
name: name,
labels: labels,
annotations: annotations,
ownerReferences: module.Owner.ToOwnerReferences());
// How we manage this service is dependent on the port mappings user asks for.
// If the user tells us to only use ClusterIP ports, we will always set the type to ClusterIP.
// If all we had were exposed ports, we will assume ClusterIP. Otherwise, we use the given value as the default service type
//
// If the user wants to expose the ClusterIPs port externally, they should manually create a service to expose it.
// This gives the user more control as to how they want this to work.
var serviceType = onlyExposedPorts
? PortMapServiceType.ClusterIP
: this.defaultMapServiceType;
return(Option.Some(new V1Service(metadata: serviceMeta, spec: new V1ServiceSpec(type: serviceType.ToString(), ports: servicePorts.Values.ToList(), selector: labels))));
}
return(Option.None <V1Service>());
}
V1Deployment PrepareDeployment(IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
string name = identity.DeploymentName();
var podSpec = this.GetPod(name, identity, module, labels);
var selector = new V1LabelSelector(matchLabels: labels);
var deploymentSpec = new V1DeploymentSpec(replicas: 1, selector: selector, template: podSpec);
var deploymentMeta = new V1ObjectMeta(name: name, labels: labels, annotations: new Dictionary <string, string>());
return(new V1Deployment(metadata: deploymentMeta, spec: deploymentSpec));
}
EdgeHubConnection(IModuleIdentity edgeHubIdentity,
ITwinManager twinManager,
RouteFactory routeFactory,
IMessageConverter <TwinCollection> twinCollectionMessageConverter,
IMessageConverter <Twin> twinMessageConverter,
VersionInfo versionInfo)
{
this.edgeHubIdentity = edgeHubIdentity;
this.twinManager = twinManager;
this.twinCollectionMessageConverter = twinCollectionMessageConverter;
this.twinMessageConverter = twinMessageConverter;
this.routeFactory = routeFactory;
this.versionInfo = versionInfo ?? VersionInfo.Empty;
}
V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
// Convert docker labels to annotations because docker labels don't have the same restrictions as Kubernetes labels.
Dictionary <string, string> annotations = module.Config.CreateOptions.Labels
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
annotations[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId);
var(proxyContainer, proxyVolumes) = this.PrepareProxyContainer(module);
var(moduleContainer, moduleVolumes) = this.PrepareModuleContainer(name, identity, module);
bool?hostIpc = this.IsHostIpc(module.Config.CreateOptions);
var imagePullSecrets = new List <Option <string> > {
this.proxyImagePullSecretName, module.Config.AuthConfig.Map(auth => auth.Name)
}
.FilterMap()
.Distinct()
.Select(pullSecretName => new V1LocalObjectReference(pullSecretName))
.ToList();
V1PodSecurityContext securityContext = module.Config.CreateOptions.SecurityContext.GetOrElse(
() => this.runAsNonRoot
? new V1PodSecurityContext {
RunAsNonRoot = true, RunAsUser = 1000
}
: null);
return(new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Name = name,
Labels = labels,
Annotations = annotations
},
Spec = new V1PodSpec
{
Containers = new List <V1Container> {
proxyContainer, moduleContainer
},
Volumes = proxyVolumes.Concat(moduleVolumes).ToList(),
ImagePullSecrets = imagePullSecrets.Any() ? imagePullSecrets : null,
SecurityContext = securityContext,
ServiceAccountName = name,
NodeSelector = module.Config.CreateOptions.NodeSelector.OrDefault(),
HostIPC = hostIpc,
}
});
}
internal EdgeHubConnection(IModuleIdentity edgeHubIdentity,
ITwinManager twinManager,
RouteFactory routeFactory,
IMessageConverter <TwinCollection> twinCollectionMessageConverter,
IMessageConverter <Twin> twinMessageConverter,
VersionInfo versionInfo,
IDeviceScopeIdentitiesCache deviceScopeIdentitiesCache)
{
this.edgeHubIdentity = edgeHubIdentity;
this.twinManager = twinManager;
this.twinCollectionMessageConverter = twinCollectionMessageConverter;
this.twinMessageConverter = twinMessageConverter;
this.routeFactory = routeFactory;
this.versionInfo = versionInfo ?? VersionInfo.Empty;
this.deviceScopeIdentitiesCache = deviceScopeIdentitiesCache;
}
public static async Task <ICommand> BuildAsync(
IDockerClient client,
DockerModule module,
IModuleIdentity identity,
DockerLoggingConfig defaultDockerLoggerConfig,
IConfigSource configSource,
bool buildForEdgeHub
)
{
// Validate parameters
Preconditions.CheckNotNull(client, nameof(client));
Preconditions.CheckNotNull(module, nameof(module));
Preconditions.CheckNotNull(defaultDockerLoggerConfig, nameof(defaultDockerLoggerConfig));
Preconditions.CheckNotNull(configSource, nameof(configSource));
CreateContainerParameters createContainerParameters = module.Config.CreateOptions ?? new CreateContainerParameters();
// serialize user provided create options to add as docker label, before adding other values
string createOptionsString = JsonConvert.SerializeObject(createContainerParameters);
// Force update parameters with indexing entries
createContainerParameters.Name = module.Name;
createContainerParameters.Image = module.Config.Image;
DeploymentConfigInfo deploymentConfigInfo = await configSource.GetDeploymentConfigInfoAsync();
DeploymentConfig deploymentConfig = deploymentConfigInfo.DeploymentConfig;
Option <DockerRuntimeInfo> dockerRuntimeInfo = deploymentConfig != DeploymentConfig.Empty && deploymentConfig.Runtime is DockerRuntimeInfo
? Option.Some((DockerRuntimeInfo)deploymentConfig.Runtime)
: Option.None <DockerRuntimeInfo>();
// Inject global parameters
InjectCerts(createContainerParameters, configSource, buildForEdgeHub);
InjectConfig(createContainerParameters, identity, buildForEdgeHub, configSource);
InjectPortBindings(createContainerParameters, buildForEdgeHub);
InjectLoggerConfig(createContainerParameters, defaultDockerLoggerConfig, dockerRuntimeInfo.Map(r => r.Config.LoggingOptions));
InjectModuleEnvVars(createContainerParameters, module.Env);
// Inject required Edge parameters
InjectLabels(createContainerParameters, module, createOptionsString);
InjectNetworkAlias(createContainerParameters, configSource, buildForEdgeHub);
return(new CreateCommand(client, createContainerParameters));
}
V1Deployment PrepareDeployment(IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
string name = identity.DeploymentName();
var podSpec = this.GetPod(name, identity, module, labels);
var selector = new V1LabelSelector(matchLabels: labels);
// Desired status in Deployment should only be Running or Stopped. Assume Running if not Stopped
int replicas = (module.DesiredStatus != ModuleStatus.Stopped) ? 1 : 0;
var deploymentSpec = new V1DeploymentSpec(replicas: replicas, selector: selector, template: podSpec);
var deploymentMeta = new V1ObjectMeta(
name: name,
labels: labels,
annotations: new Dictionary <string, string>(),
ownerReferences: module.Owner.ToOwnerReferences());
return(new V1Deployment(metadata: deploymentMeta, spec: deploymentSpec));
}
static CreateOrUpdateCommand Build(
IModuleManager moduleManager,
IModule module,
IModuleIdentity identity,
IConfigSource configSource,
object settings,
Operation operation)
{
Preconditions.CheckNotNull(moduleManager, nameof(moduleManager));
Preconditions.CheckNotNull(module, nameof(module));
Preconditions.CheckNotNull(identity, nameof(identity));
Preconditions.CheckNotNull(configSource, nameof(configSource));
Preconditions.CheckNotNull(settings, nameof(settings));
IEnumerable <EnvVar> envVars = GetEnvVars(module.Env, identity, configSource);
ModuleSpec moduleSpec = BuildModuleSpec(module, envVars, settings);
return(new CreateOrUpdateCommand(moduleManager, moduleSpec, operation));
}
public static async Task <EdgeHubConnection> Create(
IModuleIdentity edgeHubIdentity,
IEdgeHub edgeHub,
ITwinManager twinManager,
IConnectionManager connectionManager,
ICloudProxy cloudProxy,
RouteFactory routeFactory,
IMessageConverter <TwinCollection> twinCollectionMessageConverter,
IMessageConverter <Twin> twinMessageConverter,
VersionInfo versionInfo
)
{
Preconditions.CheckNotNull(edgeHubIdentity, nameof(edgeHubIdentity));
Preconditions.CheckNotNull(edgeHub, nameof(edgeHub));
Preconditions.CheckNotNull(connectionManager, nameof(connectionManager));
Preconditions.CheckNotNull(cloudProxy, nameof(cloudProxy));
Preconditions.CheckNotNull(twinCollectionMessageConverter, nameof(twinCollectionMessageConverter));
Preconditions.CheckNotNull(twinMessageConverter, nameof(twinMessageConverter));
Preconditions.CheckNotNull(routeFactory, nameof(routeFactory));
var edgeHubConnection = new EdgeHubConnection(
edgeHubIdentity, twinManager, routeFactory,
twinCollectionMessageConverter, twinMessageConverter,
versionInfo ?? VersionInfo.Empty
);
cloudProxy.BindCloudListener(new CloudListener(edgeHubConnection));
IDeviceProxy deviceProxy = new EdgeHubDeviceProxy(edgeHubConnection);
await connectionManager.AddDeviceConnection(edgeHubIdentity, deviceProxy);
await edgeHub.AddSubscription(edgeHubIdentity.Id, DeviceSubscription.DesiredPropertyUpdates);
// Clear out all the reported devices.
await edgeHubConnection.ClearDeviceConnectionStatuses();
connectionManager.DeviceConnected += edgeHubConnection.DeviceConnected;
connectionManager.DeviceDisconnected += edgeHubConnection.DeviceDisconnected;
Events.Initialized(edgeHubIdentity);
return(edgeHubConnection);
}
static IEnumerable <EnvVar> GetEnvVars(
IDictionary <string, EnvVal> moduleEnvVars,
IModuleIdentity identity,
IConfigSource configSource,
string edgeDeviceHostname,
Option <string> parentEdgeHostname)
{
List <EnvVar> envVars = moduleEnvVars.Select(m => new EnvVar(m.Key, m.Value.Value)).ToList();
// Inject the connection details as an environment variable
if (identity.Credentials is IdentityProviderServiceCredentials creds)
{
if (!string.IsNullOrWhiteSpace(creds.ProviderUri))
{
envVars.Add(new EnvVar(Constants.EdgeletWorkloadUriVariableName, creds.ProviderUri));
}
if (!string.IsNullOrWhiteSpace(creds.AuthScheme))
{
envVars.Add(new EnvVar(Constants.EdgeletAuthSchemeVariableName, creds.AuthScheme));
}
if (!string.IsNullOrWhiteSpace(creds.ModuleGenerationId))
{
envVars.Add(new EnvVar(Constants.EdgeletModuleGenerationIdVariableName, creds.ModuleGenerationId));
}
}
if (!string.IsNullOrWhiteSpace(identity.IotHubHostname))
{
envVars.Add(new EnvVar(Constants.IotHubHostnameVariableName, identity.IotHubHostname));
}
// In nested edge scenario, EdgeAgent and EdgeHub will use parent edge as upstream gateway,
// Other modules use current edge hub as upstream gateway and have parent edge hostname in environment variable.
if (identity.ModuleId.Equals(Constants.EdgeAgentModuleIdentityName) || identity.ModuleId.Equals(Constants.EdgeHubModuleIdentityName))
{
envVars.Add(new EnvVar(Constants.EdgeDeviceHostNameKey, edgeDeviceHostname));
parentEdgeHostname.ForEach(value => envVars.Add(new EnvVar(Constants.GatewayHostnameVariableName, value)));
}
else
{
envVars.Add(new EnvVar(Constants.GatewayHostnameVariableName, edgeDeviceHostname));
parentEdgeHostname.ForEach(value => envVars.Add(new EnvVar(Constants.ParentEdgeHostnameVariableName, value)));
}
if (!string.IsNullOrWhiteSpace(identity.DeviceId))
{
envVars.Add(new EnvVar(Constants.DeviceIdVariableName, identity.DeviceId));
}
if (!string.IsNullOrWhiteSpace(identity.ModuleId))
{
envVars.Add(new EnvVar(Constants.ModuleIdVariableName, identity.ModuleId));
}
if (!envVars.Exists(e => e.Key == Logger.RuntimeLogLevelEnvKey))
{
envVars.Add(new EnvVar(Logger.RuntimeLogLevelEnvKey, Logger.GetLogLevel().ToString()));
}
configSource.Configuration.GetValue <string>(Constants.UpstreamProtocolKey).ToUpstreamProtocol().ForEach(
u =>
{
if (!envVars.Any(e => e.Key.Equals(Constants.UpstreamProtocolKey, StringComparison.OrdinalIgnoreCase)))
{
envVars.Add(new EnvVar(Constants.UpstreamProtocolKey, u.ToString()));
}
});
if (identity.ModuleId.Equals(Constants.EdgeAgentModuleIdentityName))
{
string managementUri = configSource.Configuration.GetValue <string>(Constants.EdgeletManagementUriVariableName);
if (!string.IsNullOrEmpty(managementUri))
{
envVars.Add(new EnvVar(Constants.EdgeletManagementUriVariableName, managementUri));
}
string networkId = configSource.Configuration.GetValue <string>(Constants.NetworkIdKey);
if (!string.IsNullOrEmpty(networkId))
{
envVars.Add(new EnvVar(Constants.NetworkIdKey, networkId));
}
string workloadListenMnt = configSource.Configuration.GetValue <string>(Constants.EdgeletWorkloadListenMntUriVariableName);
if (!string.IsNullOrEmpty(workloadListenMnt))
{
envVars.Add(new EnvVar(Constants.EdgeletWorkloadListenMntUriVariableName, workloadListenMnt));
}
envVars.Add(new EnvVar(Constants.ModeKey, Constants.IotedgedMode));
}
// Set the edgelet's api version
string apiVersion = configSource.Configuration.GetValue <string>(Constants.EdgeletApiVersionVariableName);
if (!string.IsNullOrEmpty(apiVersion))
{
envVars.Add(new EnvVar(Constants.EdgeletApiVersionVariableName, apiVersion));
}
return(envVars);
}
public static void AuthenticatingWithDeviceIdentity(IModuleIdentity moduleIdentity)
{
Log.LogInformation((int)EventIds.AuthenticatingWithDeviceIdentity, $"Unable to authenticate client {moduleIdentity.Id} with module credentials. Attempting to authenticate using device {moduleIdentity.DeviceId} credentials.");
}
static void InjectConfig(CreateContainerParameters createContainerParameters, IModuleIdentity identity, bool injectForEdgeHub, IConfigSource configSource)
{
var envVars = new List <string>();
// Inject the connection string as an environment variable
if (identity.Credentials is ConnectionStringCredentials creds && !string.IsNullOrWhiteSpace(creds.ConnectionString))
{
string connectionStringKey = injectForEdgeHub ? Constants.IotHubConnectionStringKey : Constants.EdgeHubConnectionStringKey;
envVars.Add($"{connectionStringKey}={creds.ConnectionString}");
}
if (injectForEdgeHub)
{
envVars.Add($"{Logger.RuntimeLogLevelEnvKey}={Logger.GetLogLevel()}");
}
configSource.Configuration.GetValue <string>(Constants.UpstreamProtocolKey).ToUpstreamProtocol().ForEach(
u =>
{
if (createContainerParameters.Env?.Any(e => e.StartsWith("UpstreamProtocol=", StringComparison.OrdinalIgnoreCase)) == false)
{
envVars.Add($"UpstreamProtocol={u}");
}
});
InjectEnvVars(createContainerParameters, envVars);
}
public ModuleWithIdentity(IModule module, IModuleIdentity moduleIdentity)
{
this.module = Preconditions.CheckNotNull(module, nameof(module));
this.moduleIdentity = Preconditions.CheckNotNull(moduleIdentity, nameof(moduleIdentity));
}
public static void InvalidDeviceId(IModuleIdentity moduleIdentity, string edgeDeviceId)
{
Log.LogError((int)EventIds.InvalidDeviceError, Invariant($"Device Id {moduleIdentity.DeviceId} of module {moduleIdentity.ModuleId} is different from the edge device Id {edgeDeviceId}"));
}
private Option <V1Service> GetServiceFromModule(Dictionary <string, string> labels, KubernetesModule <TConfig> module, IModuleIdentity moduleIdentity)
{
var portList = new List <V1ServicePort>();
Option <Dictionary <string, string> > serviceAnnotations = Option.None <Dictionary <string, string> >();
bool onlyExposedPorts = true;
if (module is IModule <AgentDocker.CombinedDockerConfig> moduleWithDockerConfig)
{
if (moduleWithDockerConfig.Config.CreateOptions?.Labels != null)
{
// Add annotations from Docker labels. This provides the customer a way to assign annotations to services if they want
// to tie backend services to load balancers via an Ingress Controller.
var annotations = new Dictionary <string, string>();
foreach (KeyValuePair <string, string> label in moduleWithDockerConfig.Config.CreateOptions?.Labels)
{
annotations.Add(KubeUtils.SanitizeAnnotationKey(label.Key), label.Value);
}
serviceAnnotations = Option.Some(annotations);
}
// Handle ExposedPorts entries
if (moduleWithDockerConfig.Config?.CreateOptions?.ExposedPorts != null)
{
// Entries in the Exposed Port list just tell Docker that this container wants to listen on that port.
// We interpret this as a "ClusterIP" service type listening on that exposed port, backed by this module.
// Users of this Module's exposed port should be able to find the service by connecting to "<module name>:<port>"
this.GetExposedPorts(moduleWithDockerConfig.Config.CreateOptions.ExposedPorts)
.ForEach(
exposedList =>
exposedList.ForEach((item) => portList.Add(new V1ServicePort(item.Port, name: $"ExposedPort-{item.Port}-{item.Protocol.ToLower()}", protocol: item.Protocol))));
}
// Handle HostConfig PortBindings entries
if (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.PortBindings != null)
{
foreach (KeyValuePair <string, IList <DockerModels.PortBinding> > portBinding in moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.PortBindings)
{
string[] portProtocol = portBinding.Key.Split('/');
if (portProtocol.Length == 2)
{
if (int.TryParse(portProtocol[0], out int port) && this.ValidateProtocol(portProtocol[1], out string protocol))
{
// Entries in Docker portMap wants to expose a port on the host (hostPort) and map it to the container's port (port)
// We interpret that as the pod wants the cluster to expose a port on a public IP (hostPort), and target it to the container's port (port)
foreach (DockerModels.PortBinding hostBinding in portBinding.Value)
{
if (int.TryParse(hostBinding.HostPort, out int hostPort))
{
// If a port entry contains the same "port", then remove it and replace with a new ServicePort that contains a target.
var duplicate = portList.SingleOrDefault(a => a.Port == hostPort);
if (duplicate != default(V1ServicePort))
{
portList.Remove(duplicate);
}
portList.Add(new V1ServicePort(hostPort, name: $"HostPort-{port}-{protocol.ToLower()}", protocol: protocol, targetPort: port));
onlyExposedPorts = false;
}
else
{
Events.PortBindingValue(module, portBinding.Key);
}
}
}
}
}
}
}
if (portList.Count > 0)
{
// Selector: by module name and device name, also how we will label this puppy.
var objectMeta = new V1ObjectMeta(annotations: serviceAnnotations.GetOrElse(() => null), labels: labels, name: KubeUtils.SanitizeDNSValue(moduleIdentity.ModuleId));
// How we manage this service is dependent on the port mappings user asks for.
// If the user tells us to only use ClusterIP ports, we will always set the type to ClusterIP.
// If all we had were exposed ports, we will assume ClusterIP. Otherwise, we use the given value as the default service type
//
// If the user wants to expose the ClusterIPs port externally, they should manually create a service to expose it.
// This gives the user more control as to how they want this to work.
string serviceType;
if (onlyExposedPorts)
{
serviceType = "ClusterIP";
}
else
{
serviceType = this.defaultMapServiceType;
}
return(Option.Some(new V1Service(metadata: objectMeta, spec: new V1ServiceSpec(type: serviceType, ports: portList, selector: labels))));
}
else
{
return(Option.None <V1Service>());
}
}
internal static IEnumerable <EnvVar> GetEnvVars(IDictionary <string, EnvVal> moduleEnvVars, IModuleIdentity identity, IConfigSource configSource)
{
List <EnvVar> envVars = moduleEnvVars.Select(m => new EnvVar {
Key = m.Key, Value = m.Value.Value
}).ToList();
// Inject the connection details as an environment variable
if (identity.Credentials is IdentityProviderServiceCredentials creds)
{
if (!string.IsNullOrWhiteSpace(creds.ProviderUri))
{
envVars.Add(new EnvVar {
Key = Constants.EdgeletWorkloadUriVariableName, Value = creds.ProviderUri
});
}
if (!string.IsNullOrWhiteSpace(creds.AuthScheme))
{
envVars.Add(new EnvVar {
Key = Constants.EdgeletAuthSchemeVariableName, Value = creds.AuthScheme
});
}
if (!string.IsNullOrWhiteSpace(creds.ModuleGenerationId))
{
envVars.Add(new EnvVar {
Key = Constants.EdgeletModuleGenerationIdVariableName, Value = creds.ModuleGenerationId
});
}
}
if (!string.IsNullOrWhiteSpace(identity.IotHubHostname))
{
envVars.Add(new EnvVar {
Key = Constants.IotHubHostnameVariableName, Value = identity.IotHubHostname
});
}
if (!string.IsNullOrWhiteSpace(identity.GatewayHostname))
{
if (identity.ModuleId.Equals(Constants.EdgeAgentModuleIdentityName) || identity.ModuleId.Equals(Constants.EdgeHubModuleIdentityName))
{
envVars.Add(
new EnvVar
{
Key = Constants.EdgeDeviceHostNameKey,
Value = identity.GatewayHostname
});
}
else if (!identity.ModuleId.Equals(Constants.EdgeHubModuleIdentityName))
{
envVars.Add(
new EnvVar
{
Key = Constants.GatewayHostnameVariableName,
Value = identity.GatewayHostname
});
}
}
if (!string.IsNullOrWhiteSpace(identity.DeviceId))
{
envVars.Add(new EnvVar {
Key = Constants.DeviceIdVariableName, Value = identity.DeviceId
});
}
if (!string.IsNullOrWhiteSpace(identity.ModuleId))
{
envVars.Add(new EnvVar {
Key = Constants.ModuleIdVariableName, Value = identity.ModuleId
});
}
if (!envVars.Exists(e => e.Key == Logger.RuntimeLogLevelEnvKey))
{
envVars.Add(new EnvVar {
Key = Logger.RuntimeLogLevelEnvKey, Value = Logger.GetLogLevel().ToString()
});
}
configSource.Configuration.GetValue <string>(Constants.UpstreamProtocolKey).ToUpstreamProtocol().ForEach(
u =>
{
if (!envVars.Any(e => e.Key.Equals(Constants.UpstreamProtocolKey, StringComparison.OrdinalIgnoreCase)))
{
envVars.Add(new EnvVar {
Key = Constants.UpstreamProtocolKey, Value = u.ToString()
});
}
});
if (identity.ModuleId.Equals(Constants.EdgeAgentModuleIdentityName))
{
string managementUri = configSource.Configuration.GetValue <string>(Constants.EdgeletManagementUriVariableName);
if (!string.IsNullOrEmpty(managementUri))
{
envVars.Add(new EnvVar {
Key = Constants.EdgeletManagementUriVariableName, Value = managementUri
});
}
string networkId = configSource.Configuration.GetValue <string>(Constants.NetworkIdKey);
if (!string.IsNullOrEmpty(networkId))
{
envVars.Add(new EnvVar {
Key = Constants.NetworkIdKey, Value = networkId
});
}
envVars.Add(new EnvVar {
Key = Constants.ModeKey, Value = Constants.IotedgedMode
});
}
// Set the edgelet's api version
string apiVersion = configSource.Configuration.GetValue <string>(Constants.EdgeletApiVersionVariableName);
if (!string.IsNullOrEmpty(apiVersion))
{
envVars.Add(new EnvVar {
Key = Constants.EdgeletApiVersionVariableName, Value = apiVersion
});
}
return(envVars);
}
public static async Task <ICommand> BuildAsync(IDockerClient client, DockerModule current, DockerModule next, IModuleIdentity identity, DockerLoggingConfig dockerLoggerConfig, IConfigSource configSource)
{
Preconditions.CheckNotNull(client, nameof(client));
Preconditions.CheckNotNull(current, nameof(current));
Preconditions.CheckNotNull(next, nameof(next));
Preconditions.CheckNotNull(dockerLoggerConfig, nameof(dockerLoggerConfig));
var remove = new RemoveCommand(client, current);
ICommand create = await CreateCommand.BuildAsync(client, next, identity, dockerLoggerConfig, configSource, next is EdgeHubDockerModule);
return(new UpdateCommand(remove, create));
}
V1PodTemplateSpec GetPodFromModule(Dictionary <string, string> labels, KubernetesModule <TConfig> module, IModuleIdentity moduleIdentity)
{
if (module is IModule <AgentDocker.CombinedDockerConfig> moduleWithDockerConfig)
{
// pod labels
var podLabels = new Dictionary <string, string>(labels);
// pod annotations
var podAnnotations = new Dictionary <string, string>();
podAnnotations.Add(Constants.K8sEdgeOriginalModuleId, moduleIdentity.ModuleId);
// Convert docker labels to annotations because docker labels don't have the same restrictions as
// Kuberenetes labels.
if (moduleWithDockerConfig.Config.CreateOptions?.Labels != null)
{
foreach (KeyValuePair <string, string> label in moduleWithDockerConfig.Config.CreateOptions?.Labels)
{
podAnnotations.Add(KubeUtils.SanitizeAnnotationKey(label.Key), label.Value);
}
}
// Per container settings:
// exposed ports
Option <List <V1ContainerPort> > exposedPortsOption = (moduleWithDockerConfig.Config?.CreateOptions?.ExposedPorts != null)
? this.GetExposedPorts(moduleWithDockerConfig.Config.CreateOptions.ExposedPorts).Map(
servicePorts =>
servicePorts.Select(tuple => new V1ContainerPort(tuple.Port, protocol: tuple.Protocol)).ToList())
: Option.None <List <V1ContainerPort> >();
// privileged container
Option <V1SecurityContext> securityContext = (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Privileged == true) ? Option.Some(new V1SecurityContext(privileged: true)) : Option.None <V1SecurityContext>();
// Environment Variables.
List <V1EnvVar> env = this.CollectEnv(moduleWithDockerConfig, (KubernetesModuleIdentity)moduleIdentity);
// Bind mounts
(List <V1Volume> volumeList, List <V1VolumeMount> proxyMounts, List <V1VolumeMount> volumeMountList) = this.GetVolumesFromModule(moduleWithDockerConfig).GetOrElse((null, null, null));
// Image
string moduleImage = moduleWithDockerConfig.Config.Image;
var containerList = new List <V1Container>()
{
new V1Container(
KubeUtils.SanitizeDNSValue(moduleIdentity.ModuleId),
env: env,
image: moduleImage,
volumeMounts: volumeMountList,
securityContext: securityContext.GetOrElse(() => null),
ports: exposedPortsOption.GetOrElse(() => null)),
// TODO: Add Proxy container here - configmap for proxy configuration.
new V1Container(
"proxy",
env: env, // TODO: check these for validity for proxy.
image: this.proxyImage,
volumeMounts: proxyMounts)
};
Option <List <V1LocalObjectReference> > imageSecret = moduleWithDockerConfig.Config.AuthConfig.Map(
auth =>
{
var secret = new ImagePullSecret(auth);
var authList = new List <V1LocalObjectReference>
{
new V1LocalObjectReference(secret.Name)
};
return(authList);
});
var modulePodSpec = new V1PodSpec(containerList, volumes: volumeList, imagePullSecrets: imageSecret.GetOrElse(() => null));
var objectMeta = new V1ObjectMeta(labels: podLabels, annotations: podAnnotations);
return(new V1PodTemplateSpec(metadata: objectMeta, spec: modulePodSpec));
}
else
{
Events.InvalidModuleType(module);
}
return(new V1PodTemplateSpec());
}
