Beispiel #1
0
        /// <inheritdoc/>
        async Task <IList <DeployedResource> > IKubernetesApiWrapper.GetDeployedResources(
            ResourceType resourceType,
            string continueParameter,
            bool?allowWatchBookmarks,
            string fieldSelector,
            string labelSelector,
            int?limit,
            string resourceVersion,
            int?timeoutSeconds,
            bool?watch,
            bool?pretty)
        {
            IList <DeployedResource> mappedResources = new List <DeployedResource>();

            switch (resourceType)
            {
            case ResourceType.Deployment:
                V1DeploymentList deployments = await _client.ListNamespacedDeploymentAsync("default", allowWatchBookmarks, continueParameter, fieldSelector, labelSelector, limit, resourceVersion, null, timeoutSeconds, watch, pretty);

                mappedResources = MapDeployments(deployments.Items);
                break;

            case ResourceType.DaemonSet:
                V1DaemonSetList deamonSets = await _client.ListNamespacedDaemonSetAsync("default", allowWatchBookmarks, continueParameter, fieldSelector, labelSelector, limit, resourceVersion, null, timeoutSeconds, watch, pretty);

                mappedResources = MapDaemonSets(deamonSets.Items);
                break;
            }

            return(mappedResources);
        }
Beispiel #2
0
        public async Task PurgeModulesAsync()
        {
            // Delete all services for current edge deployment
            V1ServiceList services = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            var serviceTasks = services.Items
                               .Select(service => this.client.DeleteNamespacedServiceAsync(service.Metadata.Name, this.deviceNamespace, new V1DeleteOptions()));
            await Task.WhenAll(serviceTasks);

            // Delete all deployments for current edge deployment
            V1DeploymentList deployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            var deploymentTasks = deployments.Items
                                  .Select(
                deployment => this.client.DeleteNamespacedDeploymentAsync(
                    deployment.Metadata.Name,
                    this.deviceNamespace,
                    new V1DeleteOptions(propagationPolicy: KubernetesConstants.DefaultDeletePropagationPolicy),
                    propagationPolicy: KubernetesConstants.DefaultDeletePropagationPolicy));
            await Task.WhenAll(deploymentTasks);

            V1PersistentVolumeClaimList pvcs = await this.client.ListNamespacedPersistentVolumeClaimAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            var pvcTasks = pvcs.Items
                           .Select(pvc => this.client.DeleteNamespacedPersistentVolumeClaimAsync(pvc.Metadata.Name, this.deviceNamespace, new V1DeleteOptions()));
            await Task.WhenAll(pvcTasks);

            // Delete the service account for all deployments
            V1ServiceAccountList serviceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            var serviceAccountTasks = serviceAccounts.Items
                                      .Select(service => this.client.DeleteNamespacedServiceAsync(service.Metadata.Name, this.deviceNamespace, new V1DeleteOptions()));
            await Task.WhenAll(serviceAccountTasks);
        }
        public async Task PurgeModulesAsync()
        {
            // Delete all services for current edge deployment
            V1ServiceList services = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            var serviceTasks = services.Items
                               .Select(service => this.client.DeleteNamespacedServiceAsync(service.Metadata.Name, this.deviceNamespace, new V1DeleteOptions()));
            await Task.WhenAll(serviceTasks);

            // Delete all deployments for current edge deployment
            V1DeploymentList deployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            var deploymentTasks = deployments.Items
                                  .Select(
                deployment => this.client.DeleteNamespacedDeployment1Async(
                    deployment.Metadata.Name,
                    this.deviceNamespace,
                    new V1DeleteOptions(propagationPolicy: "Foreground"),
                    propagationPolicy: "Foreground"));
            await Task.WhenAll(deploymentTasks);

            // Delete the service account for all deployments
            V1ServiceAccountList serviceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            var serviceAccountTasks = serviceAccounts.Items
                                      .Select(service => this.client.DeleteNamespacedServiceAsync(service.Metadata.Name, this.deviceNamespace, new V1DeleteOptions()));
            await Task.WhenAll(serviceAccountTasks);
        }
Beispiel #4
0
        private Dictionary <string, string> GetCurrentDeploymentConfig(V1DeploymentList currentDeployments)
        {
            return(currentDeployments.Items.ToDictionary(
                       deployment =>
            {
                if (deployment?.Metadata?.Name != null)
                {
                    return deployment.Metadata.Name;
                }

                Events.InvalidCreationString("deployment", "null deployment");
                throw new NullReferenceException("null deployment in list");
            },
                       deployment =>
            {
                if (deployment == null)
                {
                    Events.InvalidCreationString("deployment", "null deployment");
                    throw new NullReferenceException("null deployment in list");
                }

                if (deployment.Metadata?.Annotations != null &&
                    deployment.Metadata.Annotations.TryGetValue(Constants.CreationString, out string creationString))
                {
                    return creationString;
                }

                Events.InvalidCreationString(deployment.Kind, deployment.Metadata?.Name);
                var deploymentWithoutStatus = new V1Deployment(deployment.ApiVersion, deployment.Kind, deployment.Metadata, deployment.Spec);
                return JsonConvert.SerializeObject(deploymentWithoutStatus);
            }));
        }
        public async Task <ModuleSet> DeployModulesAsync(IReadOnlyList <KubernetesModule> modules, ModuleSet currentModules)
        {
            var desiredModules   = ModuleSet.Create(modules.ToArray());
            var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);

            var labels = modules
                         .ToDictionary(
                module => module.Name,
                module => new Dictionary <string, string>
            {
                [KubernetesConstants.K8sEdgeModuleLabel]  = moduleIdentities[module.Name].DeploymentName(),
                [KubernetesConstants.K8sEdgeDeviceLabel]  = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
                [KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
            });
            var deviceOnlyLabels = new Dictionary <string, string>
            {
                [KubernetesConstants.K8sEdgeDeviceLabel]  = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
                [KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
            };

            var desiredServices = modules
                                  .Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Name], module, labels[module.Name]))
                                  .FilterMap()
                                  .ToList();

            V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            await this.ManageServices(currentServices, desiredServices);

            var desiredDeployments = modules
                                     .Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Name], module, labels[module.Name]))
                                     .ToList();

            V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            await this.ManageDeployments(currentDeployments, desiredDeployments);

            var desiredPvcs = modules
                              .Select(module => this.pvcMapper.CreatePersistentVolumeClaims(module, deviceOnlyLabels))
                              .FilterMap()
                              .SelectMany(x => x)
                              .Distinct(KubernetesPvcByValueEqualityComparer);

            // Modules may use PVCs created by the user, we get all PVCs and then work on ours.
            V1PersistentVolumeClaimList currentPvcList = await this.client.ListNamespacedPersistentVolumeClaimAsync(this.deviceNamespace);

            await this.ManagePvcs(currentPvcList, desiredPvcs);

            var desiredServiceAccounts = modules
                                         .Select(module => this.serviceAccountMapper.CreateServiceAccount(moduleIdentities[module.Name], labels[module.Name]))
                                         .ToList();

            V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);

            return(desiredModules);
        }
Beispiel #6
0
        private async Task WatchDeploymentEventsAsync(WatchEventType type, object custom)
        {
            EdgeDeploymentDefinition <TConfig> edgeDeploymentDefinition;

            try
            {
                string customString = JsonConvert.SerializeObject(custom);
                edgeDeploymentDefinition = this.deploymentSerde.Deserialize(customString);
            }
            catch (Exception e)
            {
                Events.EdgeDeploymentDeserializeFail(e);
                return;
            }

            // only operate on the device that matches this operator.
            if (string.Equals(edgeDeploymentDefinition.Metadata.Name, this.resourceName, StringComparison.OrdinalIgnoreCase))
            {
                using (await this.watchLock.LockAsync())
                {
                    V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.k8sNamespace, labelSelector : this.deploymentSelector);

                    V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.k8sNamespace, labelSelector : this.deploymentSelector);

                    Events.DeploymentStatus(type, this.resourceName);
                    switch (type)
                    {
                    case WatchEventType.Added:
                    case WatchEventType.Modified:
                        await this.ManageDeployments(currentServices, currentDeployments, edgeDeploymentDefinition);

                        break;

                    case WatchEventType.Deleted:
                        await this.DeleteDeployments(currentServices, currentDeployments);

                        break;

                    case WatchEventType.Error:
                        Events.DeploymentError();
                        break;
                    }
                }
            }
            else
            {
                Events.DeploymentNameMismatch(edgeDeploymentDefinition.Metadata.Name, this.resourceName);
            }
        }
        /// <inheritdoc/>
        async Task <IList <Deployment> > IKubernetesApiWrapper.GetDeployments(
            string continueParameter,
            bool?allowWatchBookmarks,
            string fieldSelector,
            string labelSelector,
            int?limit,
            string resourceVersion,
            int?timeoutSeconds,
            bool?watch,
            string pretty)
        {
            V1DeploymentList deployments = await _client.ListNamespacedDeploymentAsync("default", allowWatchBookmarks, continueParameter, fieldSelector, labelSelector, limit, resourceVersion, timeoutSeconds, watch, pretty);

            IList <Deployment> mappedDeployments = MapDeployments(deployments.Items);

            return(mappedDeployments);
        }
Beispiel #8
0
        private async Task DeleteDeployments(V1ServiceList currentServices, V1DeploymentList currentDeployments)
        {
            // Delete the deployment.
            // Delete any services.
            IEnumerable <Task <V1Status> > removeServiceTasks = currentServices.Items.Select(i => this.client.DeleteNamespacedServiceAsync(i.Metadata.Name, this.k8sNamespace, new V1DeleteOptions()));
            await Task.WhenAll(removeServiceTasks);

            IEnumerable <Task <V1Status> > removeDeploymentTasks = currentDeployments.Items.Select(
                d => this.client.DeleteNamespacedDeployment1Async(
                    d.Metadata.Name,
                    this.k8sNamespace,
                    new V1DeleteOptions(propagationPolicy: "Foreground"),
                    propagationPolicy: "Foreground"));
            await Task.WhenAll(removeDeploymentTasks);

            this.currentModules = ModuleSet.Empty;
        }
Beispiel #9
0
        private List <V1Deployment> GetDeploymentsByImageName(string imageName, V1DeploymentList deployments)
        {
            var matchingDeployments = deployments?.Items?.ToList().FindAll(x =>
            {
                foreach (var container in x.Spec?.Template?.Spec?.Containers)
                {
                    if (container.Image.ToLowerInvariant().Contains(imageName.ToLowerInvariant()))
                    {
                        _logger.LogDebug("Found deployment with name {Name} in namespace {Namespace}", x?.Metadata?.Name, x?.Metadata?.NamespaceProperty);

                        return(true);
                    }
                }

                return(false);
            });

            return(matchingDeployments);
        }
        public async Task <ModuleSet> DeployModulesAsync(IList <KubernetesModule> modules, ModuleSet currentModules)
        {
            var desiredModules   = ModuleSet.Create(modules.ToArray());
            var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);

            var labels = modules
                         .ToDictionary(
                module => module.Name,
                module => new Dictionary <string, string>
            {
                [KubernetesConstants.K8sEdgeModuleLabel]  = moduleIdentities[module.Name].DeploymentName(),
                [KubernetesConstants.K8sEdgeDeviceLabel]  = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
                [KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
            });

            var desiredServices = modules
                                  .Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Name], module, labels[module.Name]))
                                  .Where(service => service.HasValue)
                                  .Select(service => service.OrDefault())
                                  .ToList();

            V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            await this.ManageServices(currentServices, desiredServices);

            var desiredDeployments = modules
                                     .Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Name], module, labels[module.Name]))
                                     .ToList();

            V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            await this.ManageDeployments(currentDeployments, desiredDeployments);

            var desiredServiceAccounts = modules
                                         .Select(module => this.serviceAccountMapper.CreateServiceAccount(moduleIdentities[module.Name], labels[module.Name]))
                                         .ToList();

            V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

            await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);

            return(desiredModules);
        }
Beispiel #11
0
        async Task ManageDeployments(V1DeploymentList existing, IEnumerable <V1Deployment> desired)
        {
            // find difference between desired and existing deployments
            var diff = FindDeploymentDiff(desired, existing.Items);

            var updatingTask = diff.Updated
                               .Select(
                update =>
            {
                Events.UpdateDeployment(update.To);

                this.deploymentMapper.UpdateDeployment(update.To, update.From);
                return(this.client.ReplaceNamespacedDeploymentAsync(update.To, update.To.Metadata.Name, this.deviceNamespace));
            });
            await Task.WhenAll(updatingTask);

            // Delete all existing deployments that are not in desired list
            var removingTasks = diff.Removed
                                .Select(
                name =>
            {
                Events.DeleteDeployment(name);
                return(this.client.DeleteNamespacedDeploymentAsync(
                           name,
                           this.deviceNamespace,
                           propagationPolicy: KubernetesConstants.DefaultDeletePropagationPolicy,
                           body: new V1DeleteOptions(propagationPolicy: KubernetesConstants.DefaultDeletePropagationPolicy)));
            });
            await Task.WhenAll(removingTasks);

            // Add all new deployments from desired list
            var addingTasks = diff.Added
                              .Select(
                deployment =>
            {
                Events.CreateDeployment(deployment);
                return(this.client.CreateNamespacedDeploymentAsync(deployment, this.deviceNamespace));
            });
            await Task.WhenAll(addingTasks);
        }
Beispiel #12
0
        private async void AssertNoMatchingDeployments(string deviceSelector, string moduleName)
        {
            V1DeploymentList currentDeployments = await this.client.ListDeploymentsAsync(deviceSelector);

            Assert.Single(currentDeployments.Items, d => d.Metadata.Name == moduleName);
        }
Beispiel #13
0
        private async void AssertNoDeploymentsExist(string deviceSelector)
        {
            V1DeploymentList currentDeployments = await this.client.ListDeploymentsAsync(deviceSelector);

            Assert.Empty(currentDeployments.Items);
        }
Beispiel #14
0
        public async Task <EdgeDeploymentStatus> DeployModulesAsync(ModuleSet desiredModules, ModuleSet currentModules)
        {
            try
            {
                var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, currentModules);

                // having desired modules an no module identities means that we are unable to obtain a list of module identities
                if (desiredModules.Modules.Any() && !moduleIdentities.Any())
                {
                    Events.NoModuleIdentities();
                    return(EdgeDeploymentStatus.Failure("Unable to obtain identities for desired modules"));
                }

                var labels = desiredModules.Modules
                             .ToDictionary(
                    module => module.Key,
                    module => new Dictionary <string, string>
                {
                    [KubernetesConstants.K8sEdgeModuleLabel]  = moduleIdentities[module.Key].DeploymentName(),
                    [KubernetesConstants.K8sEdgeDeviceLabel]  = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
                    [KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
                });
                var deviceOnlyLabels = new Dictionary <string, string>
                {
                    [KubernetesConstants.K8sEdgeDeviceLabel]  = KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId),
                    [KubernetesConstants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.resourceName.Hostname)
                };

                var desiredServices = desiredModules.Modules
                                      .Select(module => this.serviceMapper.CreateService(moduleIdentities[module.Key], (KubernetesModule)module.Value, labels[module.Key]))
                                      .FilterMap()
                                      .ToList();

                V1ServiceList currentServices = await this.client.ListNamespacedServiceAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

                await this.ManageServices(currentServices, desiredServices);

                var desiredDeployments = desiredModules.Modules
                                         .Select(module => this.deploymentMapper.CreateDeployment(moduleIdentities[module.Key], (KubernetesModule)module.Value, labels[module.Key]))
                                         .ToList();

                V1DeploymentList currentDeployments = await this.client.ListNamespacedDeploymentAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

                await this.ManageDeployments(currentDeployments, desiredDeployments);

                var desiredPvcs = desiredModules.Modules
                                  .Select(module => this.pvcMapper.CreatePersistentVolumeClaims((KubernetesModule)module.Value, deviceOnlyLabels))
                                  .FilterMap()
                                  .SelectMany(x => x)
                                  .Distinct(KubernetesPvcByValueEqualityComparer);

                // Modules may use PVCs created by the user, we get all PVCs and then work on ours.
                V1PersistentVolumeClaimList currentPvcList = await this.client.ListNamespacedPersistentVolumeClaimAsync(this.deviceNamespace);

                await this.ManagePvcs(currentPvcList, desiredPvcs);

                var desiredServiceAccounts = desiredModules.Modules
                                             .Select(module => this.serviceAccountMapper.CreateServiceAccount((KubernetesModule)module.Value, moduleIdentities[module.Key], labels[module.Key]))
                                             .ToList();

                V1ServiceAccountList currentServiceAccounts = await this.client.ListNamespacedServiceAccountAsync(this.deviceNamespace, labelSelector : this.deploymentSelector);

                await this.ManageServiceAccounts(currentServiceAccounts, desiredServiceAccounts);

                return(EdgeDeploymentStatus.Success("Successfully deployed"));
            }
            catch (HttpOperationException e)
            {
                Events.DeployModulesException(e);
                return(EdgeDeploymentStatus.Failure(e));
            }
        }
Beispiel #15
0
        private async Task ManageDeployments(V1ServiceList currentServices, V1DeploymentList currentDeployments, EdgeDeploymentDefinition <TConfig> customObject)
        {
            var desiredModules   = ModuleSet.Create(customObject.Spec.ToArray());
            var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, this.currentModules);

            // Pull current configuration from annotations.
            Dictionary <string, string> currentV1ServicesFromAnnotations = this.GetCurrentServiceConfig(currentServices);
            // strip out edgeAgent so edgeAgent doesn't update itself.
            // TODO: remove this filter.
            var agentDeploymentName = this.DeploymentName(CoreConstants.EdgeAgentModuleName);
            Dictionary <string, string> currentDeploymentsFromAnnotations = this.GetCurrentDeploymentConfig(currentDeployments)
                                                                            .ToDictionary(
                pair => pair.Key,
                pair => pair.Value);

            var desiredServices    = new List <V1Service>();
            var desiredDeployments = new List <V1Deployment>();

            foreach (KubernetesModule <TConfig> module in customObject.Spec)
            {
                var moduleId = moduleIdentities[module.Name];
                if (string.Equals(module.Type, "docker"))
                {
                    // Default labels
                    var labels = new Dictionary <string, string>
                    {
                        [Constants.K8sEdgeModuleLabel]  = KubeUtils.SanitizeLabelValue(moduleId.ModuleId),
                        [Constants.K8sEdgeDeviceLabel]  = KubeUtils.SanitizeLabelValue(this.deviceId),
                        [Constants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.iotHubHostname)
                    };

                    // Create a Service for every network interface of each module. (label them with hub, device and module id)
                    Option <V1Service> moduleService = this.GetServiceFromModule(labels, module, moduleId);
                    moduleService.ForEach(service => desiredServices.Add(service));

                    // Create a Pod for each module, and a proxy container.
                    V1PodTemplateSpec v1PodSpec = this.GetPodFromModule(labels, module, moduleId);

                    // if this is the edge agent's deployment then it needs to run under a specific service account
                    if (moduleIdentities[module.Name].ModuleId == CoreConstants.EdgeAgentModuleIdentityName)
                    {
                        v1PodSpec.Spec.ServiceAccountName = this.serviceAccountName;
                    }

                    // Bundle into a deployment
                    string deploymentName = this.DeploymentName(moduleIdentities[module.Name].ModuleId);
                    // Deployment data
                    var deploymentMeta = new V1ObjectMeta(name: deploymentName, labels: labels);

                    var selector       = new V1LabelSelector(matchLabels: labels);
                    var deploymentSpec = new V1DeploymentSpec(replicas: 1, selector: selector, template: v1PodSpec);

                    desiredDeployments.Add(new V1Deployment(metadata: deploymentMeta, spec: deploymentSpec));
                }
                else
                {
                    Events.InvalidModuleType(module);
                }
            }

            // Find current Services/Deployments which need to be removed and updated
            var servicesRemoved = new List <V1Service>(currentServices.Items);

            servicesRemoved.RemoveAll(s => desiredServices.Exists(i => string.Equals(i.Metadata.Name, s.Metadata.Name)));
            var deploymentsRemoved = new List <V1Deployment>(currentDeployments.Items);

            deploymentsRemoved.RemoveAll(
                d =>
            {
                return(desiredDeployments.Exists(i => string.Equals(i.Metadata.Name, d.Metadata.Name)));
            });

            var newServices = new List <V1Service>();

            desiredServices.ForEach(
                s =>
            {
                string creationString = JsonConvert.SerializeObject(s);

                if (currentV1ServicesFromAnnotations.ContainsKey(s.Metadata.Name))
                {
                    string serviceAnnotation = currentV1ServicesFromAnnotations[s.Metadata.Name];
                    // If configuration matches, no need to update service
                    if (string.Equals(serviceAnnotation, creationString))
                    {
                        return;
                    }

                    if (s.Metadata.Annotations == null)
                    {
                        s.Metadata.Annotations = new Dictionary <string, string>();
                    }

                    s.Metadata.Annotations[Constants.CreationString] = creationString;

                    servicesRemoved.Add(s);
                    newServices.Add(s);
                    Events.UpdateService(s.Metadata.Name);
                }
                else
                {
                    if (s.Metadata.Annotations == null)
                    {
                        s.Metadata.Annotations = new Dictionary <string, string>();
                    }

                    s.Metadata.Annotations[Constants.CreationString] = creationString;

                    newServices.Add(s);
                    Events.CreateService(s.Metadata.Name);
                }
            });
            var deploymentsUpdated = new List <V1Deployment>();
            var newDeployments     = new List <V1Deployment>();
            List <V1Deployment> currentDeploymentsList = currentDeployments.Items.ToList();

            desiredDeployments.ForEach(
                d =>
            {
                if (currentDeploymentsFromAnnotations.ContainsKey(d.Metadata.Name))
                {
                    V1Deployment current         = currentDeploymentsList.Find(i => string.Equals(i.Metadata.Name, d.Metadata.Name));
                    string currentFromAnnotation = currentDeploymentsFromAnnotations[d.Metadata.Name];
                    string creationString        = JsonConvert.SerializeObject(d);

                    // If configuration matches, or this is edgeAgent deployment and the images match,
                    // no need to do update deployment
                    if (string.Equals(currentFromAnnotation, creationString) ||
                        (string.Equals(d.Metadata.Name, this.DeploymentName(CoreConstants.EdgeAgentModuleName)) && V1DeploymentEx.ImageEquals(current, d)))
                    {
                        return;
                    }

                    d.Metadata.ResourceVersion = current.Metadata.ResourceVersion;
                    if (d.Metadata.Annotations == null)
                    {
                        var annotations = new Dictionary <string, string>
                        {
                            [Constants.CreationString] = creationString
                        };
                        d.Metadata.Annotations = annotations;
                    }
                    else
                    {
                        d.Metadata.Annotations[Constants.CreationString] = creationString;
                    }

                    deploymentsUpdated.Add(d);
                    Events.UpdateDeployment(d.Metadata.Name);
                }
                else
                {
                    string creationString = JsonConvert.SerializeObject(d);
                    var annotations       = new Dictionary <string, string>
                    {
                        [Constants.CreationString] = creationString
                    };
                    d.Metadata.Annotations = annotations;
                    newDeployments.Add(d);
                    Events.CreateDeployment(d.Metadata.Name);
                }
            });

            // Remove the old
            IEnumerable <Task <V1Status> > removeServiceTasks = servicesRemoved.Select(
                i =>
            {
                Events.DeletingService(i);
                return(this.client.DeleteNamespacedServiceAsync(i.Metadata.Name, this.k8sNamespace, new V1DeleteOptions()));
            });
            await Task.WhenAll(removeServiceTasks);

            IEnumerable <Task <V1Status> > removeDeploymentTasks = deploymentsRemoved.Select(
                d =>
            {
                Events.DeletingDeployment(d);
                return(this.client.DeleteNamespacedDeployment1Async(d.Metadata.Name, this.k8sNamespace, new V1DeleteOptions(propagationPolicy: "Foreground"), propagationPolicy: "Foreground"));
            });
            await Task.WhenAll(removeDeploymentTasks);

            // Create the new.
            IEnumerable <Task <V1Service> > createServiceTasks = newServices.Select(
                s =>
            {
                Events.CreatingService(s);
                return(this.client.CreateNamespacedServiceAsync(s, this.k8sNamespace));
            });
            await Task.WhenAll(createServiceTasks);

            IEnumerable <Task <V1Deployment> > createDeploymentTasks = newDeployments.Select(
                deployment =>
            {
                Events.CreatingDeployment(deployment);
                return(this.client.CreateNamespacedDeploymentAsync(deployment, this.k8sNamespace));
            });
            await Task.WhenAll(createDeploymentTasks);

            // Update the existing - should only do this when different.
            IEnumerable <Task <V1Deployment> > updateDeploymentTasks = deploymentsUpdated.Select(deployment => this.client.ReplaceNamespacedDeploymentAsync(deployment, deployment.Metadata.Name, this.k8sNamespace));
            await Task.WhenAll(updateDeploymentTasks);

            this.currentModules = desiredModules;
        }