public async void ChangeFileAndSeeChange()
{
// Set up initial config file and create `FileConfigSource`
File.WriteAllText(this.tempFileName, ValidJson1);
Diff validDiff1To2 = ValidSet2.Diff(ValidSet1);
using (FileConfigSource configSource = await FileConfigSource.Create(this.tempFileName, this.config, this.serde))
{
Assert.NotNull(configSource);
DeploymentConfigInfo deploymentConfigInfo = await configSource.GetDeploymentConfigInfoAsync();
Assert.NotNull(deploymentConfigInfo);
ModuleSet initialModuleSet = deploymentConfigInfo.DeploymentConfig.GetModuleSet();
Diff emptyDiff = ValidSet1.Diff(initialModuleSet);
Assert.True(emptyDiff.IsEmpty);
// Modify the config file by writing new content.
File.WriteAllText(this.tempFileName, ValidJson2);
await Task.Delay(TimeSpan.FromSeconds(20));
DeploymentConfigInfo updatedAgentConfig = await configSource.GetDeploymentConfigInfoAsync();
Assert.NotNull(updatedAgentConfig);
ModuleSet updatedModuleSet = updatedAgentConfig.DeploymentConfig.GetModuleSet();
Diff newDiff = updatedModuleSet.Diff(initialModuleSet);
Assert.False(newDiff.IsEmpty);
Assert.Equal(newDiff, validDiff1To2);
}
}
async Task <IEnumerable <ICommand> > ProcessDesiredAndCurrentSets(
ModuleSet desired, ModuleSet current, IRuntimeInfo runtimeInfo, IImmutableDictionary <string, IModuleIdentity> moduleIdentities)
{
Diff diff = desired.Diff(current);
IEnumerable <Task <ICommand> > stopTasks = current.Modules.Select(m => this.commandFactory.StopAsync(m.Value));
IEnumerable <ICommand> stop = await Task.WhenAll(stopTasks);
IEnumerable <Task <ICommand> > removeTasks = diff.Removed.Select(name => this.commandFactory.RemoveAsync(current.Modules[name]));
IEnumerable <ICommand> remove = await Task.WhenAll(removeTasks);
// Only update changed modules
IList <Task <ICommand> > updateTasks = diff.AddedOrUpdated
.Select(m => this.CreateOrUpdate(current, m, runtimeInfo, moduleIdentities))
.ToList();
IEnumerable <ICommand> update = await Task.WhenAll(updateTasks);
IEnumerable <Task <ICommand> > startTasks = desired.Modules.Values
.Where(m => m.DesiredStatus == ModuleStatus.Running)
.Select(m => this.commandFactory.StartAsync(m));
IEnumerable <ICommand> start = await Task.WhenAll(startTasks);
return(stop
.Concat(remove)
.Concat(update)
.Concat(start));
}
public async void CreateSuccess()
{
File.WriteAllText(this.tempFileName, ValidJson1);
using (FileConfigSource configSource = await FileConfigSource.Create(this.tempFileName, this.config, this.serde))
{
Assert.NotNull(configSource);
DeploymentConfigInfo deploymentConfigInfo = await configSource.GetDeploymentConfigInfoAsync();
Assert.NotNull(deploymentConfigInfo);
Assert.NotNull(deploymentConfigInfo.DeploymentConfig);
ModuleSet moduleSet = deploymentConfigInfo.DeploymentConfig.GetModuleSet();
Diff emptyDiff = ValidSet1.Diff(moduleSet);
Assert.True(emptyDiff.IsEmpty);
}
}
public async Task <Plan> PlanAsync(
ModuleSet desired,
ModuleSet current,
IRuntimeInfo runtimeInfo,
IImmutableDictionary <string, IModuleIdentity> moduleIdentities)
{
Events.LogDesired(desired);
Events.LogCurrent(current);
Events.LogIdentities(moduleIdentities);
// Check that module names sanitize and remain unique.
var groupedModules = desired.Modules.ToLookup(pair => KubeUtils.SanitizeK8sValue(pair.Key));
if (groupedModules.Any(c => c.Count() > 1))
{
string nameList = groupedModules
.Where(c => c.Count() > 1)
.SelectMany(g => g, (pairs, pair) => pair.Key)
.Join(",");
throw new InvalidIdentityException($"Deployment will cause a name collision in Kubernetes namespace, modules: [{nameList}]");
}
// TODO: improve this so it is generic for all potential module types.
if (!desired.Modules.Values.All(p => p is IModule <DockerConfig>))
{
throw new InvalidModuleException($"Kubernetes deployment currently only handles type={typeof(DockerConfig).FullName}");
}
// This is a workaround for K8s Public Preview Refresh
// TODO: remove this workaround when merging to the main release
desired = new ModuleSet(desired.Modules.Remove(Constants.EdgeAgentModuleName));
current = new ModuleSet(current.Modules.Remove(Constants.EdgeAgentModuleName));
Diff moduleDifference = desired.Diff(current);
Plan plan;
if (!moduleDifference.IsEmpty)
{
// The "Plan" here is very simple - if we have any change, publish all desired modules to a EdgeDeployment CRD.
// The CRD allows us to give the customer a Kubernetes-centric way to see the deployment
// and the status of that deployment through the "edgedeployments" API.
var crdCommand = new EdgeDeploymentCommand(this.deviceNamespace, this.resourceName, this.client, desired.Modules.Values, runtimeInfo, this.configProvider);
var planCommand = await this.commandFactory.WrapAsync(crdCommand);
var planList = new List <ICommand>
{
planCommand
};
Events.PlanCreated(planList);
plan = new Plan(planList);
}
else
{
plan = Plan.Empty;
}
return(plan);
}
public async Task <Plan> PlanAsync(ModuleSet desired, ModuleSet current, IRuntimeInfo runtimeInfo, IImmutableDictionary <string, IModuleIdentity> moduleIdentities)
{
Diff diff = desired.Diff(current);
Plan plan = diff.IsEmpty
? Plan.Empty
: await this.CreatePlan(desired, current, runtimeInfo, moduleIdentities);
return(plan);
}
DiffState ProcessDiff(ModuleSet desired, ModuleSet current)
{
Diff diff = desired.Diff(current);
IList <IModule> added = diff.Added.ToList();
IList <IRuntimeModule> removed = diff.Removed.Select(name => (IRuntimeModule)current.Modules[name]).ToList();
// We are interested in 3 kinds of "updated" modules:
//
// [1] someone pushed a new deployment for this device that changed something
// for an existing module
// [2] someone pushed a new deployment for this device that changed the desired
// status of a module (running to stopped, or stopped to running)
// [3] something changed in the runtime state of the module - for example, it
// had a tragic untimely death
// We need to be able to distinguish between the three cases because we handle them differently
IList <IModule> updateDeployed = diff.Updated.ToList();
IList <IModule> desiredStatusUpdated = diff.DesiredStatusUpdated.ToList();
// Get the list of modules unaffected by deployment
ISet <string> modulesInDeployment = diff.AddedOrUpdated
.Concat(diff.DesiredStatusUpdated)
.Select(m => m.Name)
.Concat(diff.Removed)
.ToImmutableHashSet();
IList <IRuntimeModule> currentRuntimeModules = current.Modules.Values
.Select(m => (IRuntimeModule)m)
.Where(m => !modulesInDeployment.Contains(m.Name))
.Except(updateDeployed.Select(m => current.Modules[m.Name] as IRuntimeModule)).ToList();
// Find the modules whose desired and runtime status are not the same
IList <IRuntimeModule> updateStateChanged = currentRuntimeModules
.Where(m => m.DesiredStatus != m.RuntimeStatus && m.RuntimeStatus != ModuleStatus.Dead).ToList();
// Identify dead modules
IList <IRuntimeModule> dead = currentRuntimeModules
.Where(m => m.RuntimeStatus == ModuleStatus.Dead).ToList();
// Apart from all of the lists above, there can be modules in "current" where neither
// the desired state has changed nor the runtime state has changed. For example, a module
// that is expected to be "running" continues to run just fine. This won't show up in
// any of the lists above. But we are still interested in these because we want to clear
// the restart stats on them when they have been behaving well for "intensiveCareTime".
//
// Note that we are only interested in "running" modules. If there's a module that was
// expected to be in the "stopped" state and continues to be in the "stopped" state, that
// is not very interesting to us.
IList <IRuntimeModule> runningGreat = currentRuntimeModules
.Where(m => m.DesiredStatus == ModuleStatus.Running && m.RuntimeStatus == ModuleStatus.Running).ToList();
return(added, updateDeployed, desiredStatusUpdated, updateStateChanged, removed, dead, runningGreat);
}
// Modules in IoTHub can be created in one of two ways - 1. single deployment:
// This can be done using the registry manager's
// ApplyConfigurationContentOnDeviceAsync method. This call will create all
// modules in the provided deployment json, but does not create the module
// credentials. After a single deployment, GetModuleIdentitiesAsync will update
// such modules in the service by calling UpdateModuleAsync, prompting the
// service to create and return credentials for them. The single deployment also
// stamps each module with its twin (provided by the deployment json) at module
// creation time. 2. at-scale deployment: This can be done via the portal on the
// Edge blade. This type of deployment waits for a module identity to be
// created, before stamping it with its twin. In this type of deployment, the
// EdgeAgent needs to create the modules identities. This is also handled in
// GetModuleIdentitiesAsync. When the deployment detects that a module has been
// created, it stamps it with the deployed twin. The service creates the
// $edgeAgent and $edgeHub twin when it creates the Edge Device, so their twins
// are always available for stamping with either a single deployment or at-scale
// deployment.
public async Task <IImmutableDictionary <string, IModuleIdentity> > GetModuleIdentitiesAsync(ModuleSet desired, ModuleSet current)
{
Diff diff = desired.Diff(current);
if (diff.IsEmpty)
{
return(ImmutableDictionary <string, IModuleIdentity> .Empty);
}
// System modules have different module names and identity names. We need to convert module names to module identity names
// and vice versa, to make sure the right values are being used.
// TODO - This will fail if the user adds modules with the same module name as a system module - for example a module called
// edgeHub. We might have to catch such cases and flag them as error (or handle them in some other way).
IEnumerable <string> updatedModuleIdentites = diff.Updated.Select(m => ModuleIdentityHelper.GetModuleIdentityName(m.Name));
IEnumerable <string> removedModuleIdentites = diff.Removed.Select(m => ModuleIdentityHelper.GetModuleIdentityName(m));
List <Module> modules = (await this.serviceClient.GetModules()).ToList();
ImmutableDictionary <string, Module> modulesDict = modules.ToImmutableDictionary(p => p.Id);
IEnumerable <string> createIdentities = updatedModuleIdentites.Where(m => !modulesDict.ContainsKey(m));
IEnumerable <string> removeIdentities = removedModuleIdentites.Where(m => modulesDict.ContainsKey(m) &&
string.Equals(modulesDict.GetValueOrDefault(m).ManagedBy, Constants.ModuleIdentityEdgeManagedByValue, StringComparison.OrdinalIgnoreCase));
// Update any identities that don't have SAS auth type or where the keys are null (this will happen for single device deployments,
// where the identities of modules are created, but the auth keys are not set).
IEnumerable <Module> updateIdentities = modules.Where(
m => m.Authentication == null ||
m.Authentication.Type != AuthenticationType.Sas ||
m.Authentication.SymmetricKey == null ||
(m.Authentication.SymmetricKey.PrimaryKey == null && m.Authentication.SymmetricKey.SecondaryKey == null))
.Select(
m =>
{
m.Authentication = new AuthenticationMechanism
{
Type = AuthenticationType.Sas
};
return(m);
}).ToList();
List <Module> updatedModulesIndentity = (await this.UpdateServiceModulesIdentityAsync(removeIdentities, createIdentities, updateIdentities)).ToList();
ImmutableDictionary <string, Module> updatedDict = updatedModulesIndentity.ToImmutableDictionary(p => p.Id);
IEnumerable <IModuleIdentity> moduleIdentities = updatedModulesIndentity.Concat(modules.Where(p => !updatedDict.ContainsKey(p.Id))).Select(p =>
{
string connectionString = this.GetModuleConnectionString(p);
return(new ModuleIdentity(this.iothubHostName, this.gatewayHostName, this.deviceId, p.Id, new ConnectionStringCredentials(connectionString)));
});
return(moduleIdentities.ToImmutableDictionary(m => ModuleIdentityHelper.GetModuleName(m.ModuleId)));
}
public async Task <Plan> PlanAsync(
ModuleSet desired,
ModuleSet current,
IRuntimeInfo runtimeInfo,
IImmutableDictionary <string, IModuleIdentity> moduleIdentities)
{
Events.LogDesired(desired);
Events.LogCurrent(current);
Events.LogIdentities(moduleIdentities);
// Check that module names sanitize and remain unique.
var groupedModules = desired.Modules.GroupBy(pair => KubeUtils.SanitizeK8sValue(pair.Key)).ToArray();
if (groupedModules.Any(c => c.Count() > 1))
{
string nameList = groupedModules.Where(c => c.Count() > 1).SelectMany(g => g, (pairs, pair) => pair.Key).Join(",");
throw new InvalidIdentityException($"Deployment will cause a name collision in Kubernetes namespace, modules: [{nameList}]");
}
// TODO: improve this so it is generic for all potential module types.
if (!desired.Modules.Values.All(p => p is IModule <DockerConfig>))
{
throw new InvalidModuleException($"Kubernetes deployment currently only handles type={typeof(T).FullName}");
}
Diff moduleDifference = desired.Diff(current);
Plan plan;
if (!moduleDifference.IsEmpty)
{
// The "Plan" here is very simple - if we have any change, publish all desired modules to a CRD.
// The CRD allows us to give the customer a Kubernetes-centric way to see the deployment
// and the status of that deployment through the "edgedeployments" API.
var k8sModules = desired.Modules.Select(m => new KubernetesModule <DockerConfig>(m.Value as IModule <DockerConfig>));
var crdCommand = new KubernetesCrdCommand <CombinedDockerConfig>(this.deviceNamespace, this.iotHubHostname, this.deviceId, this.client, k8sModules.ToArray(), Option.Some(runtimeInfo), this.combinedConfigProvider as ICombinedConfigProvider <CombinedDockerConfig>);
var planCommand = await this.commandFactory.WrapAsync(crdCommand);
var planList = new List <ICommand>
{
planCommand
};
Events.PlanCreated(planList);
plan = new Plan(planList);
}
else
{
plan = Plan.Empty;
}
return(plan);
}
public async Task <IImmutableDictionary <string, IModuleIdentity> > GetModuleIdentitiesAsync(ModuleSet desired, ModuleSet current)
{
Diff diff = desired.Diff(current);
if (diff.IsEmpty)
{
return(ImmutableDictionary <string, IModuleIdentity> .Empty);
}
IList <string> updatedModuleNames = diff.Updated.Select(m => ModuleIdentityHelper.GetModuleIdentityName(m.Name)).ToList();
IEnumerable <string> removedModuleNames = diff.Removed.Select(m => ModuleIdentityHelper.GetModuleIdentityName(m));
IImmutableDictionary <string, Identity> identities = (await this.identityManager.GetIdentities()).ToImmutableDictionary(i => i.ModuleId);
// Create identities for all modules that are in the deployment but aren't in iotedged.
IEnumerable <string> createIdentities = updatedModuleNames.Where(m => !identities.ContainsKey(m));
// Update identities for all modules that are in the deployment and are in iotedged (except for Edge Agent which gets special
// treatment in iotedged).
//
// NOTE: This update can potentiatlly be made more efficient by checking that an update is actually needed, i.e. if auth type
// is not SAS and/or if the credentials are not what iotedged expects it to be.
IEnumerable <Identity> updateIdentities = updatedModuleNames
.Where(m => identities.ContainsKey(m) && m != Constants.EdgeAgentModuleIdentityName)
.Select(m => identities[m]);
// Remove identities which exist in iotedged but don't exist in the deployment anymore. We exclude however, identities that
// aren't managed by Edge since these have been created by some out-of-band process and Edge doesn't "own" the identity.
IEnumerable <string> removeIdentities = removedModuleNames.Where(m => identities.ContainsKey(m) &&
Constants.ModuleIdentityEdgeManagedByValue.Equals(identities[m].ManagedBy, StringComparison.OrdinalIgnoreCase));
// First remove identities (so that we don't go over the IoTHub limit).
await Task.WhenAll(removeIdentities.Select(i => this.identityManager.DeleteIdentityAsync(i)));
// Create/update identities.
IEnumerable <Task <Identity> > createTasks = createIdentities.Select(i => this.identityManager.CreateIdentityAsync(i, Constants.ModuleIdentityEdgeManagedByValue));
IEnumerable <Task <Identity> > updateTasks = updateIdentities.Select(i => this.identityManager.UpdateIdentityAsync(i.ModuleId, i.GenerationId, i.ManagedBy));
Identity[] upsertedIdentities = await Task.WhenAll(createTasks.Concat(updateTasks));
List <IModuleIdentity> moduleIdentities = upsertedIdentities.Select(m => this.GetModuleIdentity(m)).ToList();
// Add the module identity for Edge Agent in the returned dictionary
// because is was excluded from the update call.
if (identities.ContainsKey(Constants.EdgeAgentModuleIdentityName))
{
moduleIdentities.Add(this.GetModuleIdentity(identities[Constants.EdgeAgentModuleIdentityName]));
}
return(moduleIdentities.ToImmutableDictionary(m => ModuleIdentityHelper.GetModuleName(m.ModuleId)));
}
DiffState ProcessDiff(ModuleSet desired, ModuleSet current)
{
Diff diff = desired.Diff(current);
IList <IModule> added = diff.Updated.Where(m => !current.TryGetModule(m.Name, out _)).ToList();
IList <IRuntimeModule> removed = diff.Removed.Select(name => (IRuntimeModule)current.Modules[name]).ToList();
// We are interested in 2 kinds of "updated" modules:
//
// [1] someone pushed a new deployment for this device that changed something
// for an existing module
// [2] something changed in the runtime state of the module - for example, it
// had a tragic untimely death
//
// We need to be able to distinguish between the two cases because for the latter
// we want to apply the restart policy and for the former we want to simply
// re-deploy.
IList <IModule> updateDeployed = diff.Updated.Except(added).ToList(); // TODO: Should we do module name comparisons below instead of object comparisons? Faster?
IList <IRuntimeModule> currentRuntimeModules = current.Modules.Values
.Select(m => (IRuntimeModule)m)
.Except(removed) // TODO: Should we do module name comparisons below instead of object comparisons? Faster?
.Except(updateDeployed.Select(m => current.Modules[m.Name] as IRuntimeModule)).ToList();
IList <IRuntimeModule> updateStateChanged = currentRuntimeModules
.Where(m => m.DesiredStatus == ModuleStatus.Running && m.RuntimeStatus != ModuleStatus.Running).ToList();
// Apart from all of the lists above, there can be modules in "current" where neither
// the desired state has changed nor the runtime state has changed. For example, a module
// that is expected to be "running" continues to run just fine. This won't show up in
// any of the lists above. But we are still interested in these because we want to clear
// the restart stats on them when they have been behaving well for "intensiveCareTime".
//
// Note that we are only interested in "running" modules. If there's a module that was
// expected to be in the "stopped" state and continues to be in the "stopped" state, that
// is not very interesting to us.
IList <IRuntimeModule> runningGreat = currentRuntimeModules
.Where(m => m.DesiredStatus == ModuleStatus.Running && m.RuntimeStatus == ModuleStatus.Running).ToList();
return(added, updateDeployed, updateStateChanged, removed, runningGreat);
}
public async Task <IImmutableDictionary <string, IModuleIdentity> > GetModuleIdentitiesAsync(ModuleSet desired, ModuleSet current)
{
Diff diff = desired.Diff(current);
if (diff.IsEmpty)
{
return(ImmutableDictionary <string, IModuleIdentity> .Empty);
}
try
{
IImmutableDictionary <string, IModuleIdentity> moduleIdentities = await this.GetModuleIdentitiesAsync(diff);
return(moduleIdentities);
}
catch (Exception ex)
{
Events.ErrorGettingModuleIdentities(ex);
return(ImmutableDictionary <string, IModuleIdentity> .Empty);
}
}
public async Task <Plan> PlanAsync(
ModuleSet desired,
ModuleSet current,
IRuntimeInfo runtimeInfo,
IImmutableDictionary <string, IModuleIdentity> moduleIdentities)
{
Events.LogDesired(desired);
// We receive current ModuleSet from Agent based on what it reports (i.e. pods).
// We need to rebuild the current ModuleSet based on deployments (i.e. CRD).
Option <EdgeDeploymentDefinition> activeDeployment = await this.GetCurrentEdgeDeploymentDefinitionAsync();
ModuleSet currentModules =
activeDeployment.Match(
a => ModuleSet.Create(a.Spec.ToArray()),
() => ModuleSet.Empty);
Events.LogCurrent(currentModules);
// Check that module names sanitize and remain unique.
var groupedModules = desired.Modules.ToLookup(pair => KubeUtils.SanitizeK8sValue(pair.Key));
if (groupedModules.Any(c => c.Count() > 1))
{
string nameList = groupedModules
.Where(c => c.Count() > 1)
.SelectMany(g => g, (pairs, pair) => pair.Key)
.Join(",");
throw new InvalidIdentityException($"Deployment will cause a name collision in Kubernetes namespace, modules: [{nameList}]");
}
// TODO: improve this so it is generic for all potential module types.
if (!desired.Modules.Values.All(p => p is IModule <DockerConfig>))
{
throw new InvalidModuleException($"Kubernetes deployment currently only handles type={typeof(DockerConfig).FullName}");
}
Diff moduleDifference = desired.Diff(currentModules);
Plan plan;
if (!moduleDifference.IsEmpty)
{
// The "Plan" here is very simple - if we have any change, publish all desired modules to a EdgeDeployment CRD.
var crdCommand = new EdgeDeploymentCommand(this.resourceName, this.deviceSelector, this.deviceNamespace, this.client, desired.Modules.Values, activeDeployment, runtimeInfo, this.configProvider, this.moduleOwner);
var planCommand = await this.commandFactory.WrapAsync(crdCommand);
var planList = new List <ICommand>
{
planCommand
};
Events.PlanCreated(planList);
plan = new Plan(planList);
}
else
{
plan = Plan.Empty;
}
return(plan);
}
public void TestDiff(ModuleSet start, ModuleSet end, Diff expected)
{
Diff setDifference = end.Diff(start);
Assert.Equal(setDifference, expected);
}
