public KubernetesModule(
string iotHubHostname,
string deviceId,
string edgeDeviceHostName,
string proxyImage,
Option <string> proxyImagePullSecretName,
string proxyConfigPath,
string proxyConfigVolumeName,
string proxyConfigMapName,
string proxyTrustBundlePath,
string proxyTrustBundleVolumeName,
string proxyTrustBundleConfigMapName,
string apiVersion,
string deviceNamespace,
Uri managementUri,
Uri workloadUri,
IEnumerable <global::Docker.DotNet.Models.AuthConfig> dockerAuthConfig,
Option <UpstreamProtocol> upstreamProtocol,
Option <string> productInfo,
PortMapServiceType defaultMapServiceType,
bool enableServiceCallTracing,
string persistentVolumeName,
string storageClassName,
uint persistentVolumeClaimSizeMb,
Option <IWebProxy> proxy,
bool closeOnIdleTimeout,
TimeSpan idleTimeout,
KubernetesExperimentalFeatures experimentalFeatures,
KubernetesModuleOwner moduleOwner,
bool runAsNonRoot)
{
this.resourceName = new ResourceName(iotHubHostname, deviceId);
this.edgeDeviceHostName = Preconditions.CheckNonWhiteSpace(edgeDeviceHostName, nameof(edgeDeviceHostName));
this.proxyImage = Preconditions.CheckNonWhiteSpace(proxyImage, nameof(proxyImage));
this.proxyImagePullSecretName = proxyImagePullSecretName;
this.proxyConfigPath = Preconditions.CheckNonWhiteSpace(proxyConfigPath, nameof(proxyConfigPath));
this.proxyConfigVolumeName = Preconditions.CheckNonWhiteSpace(proxyConfigVolumeName, nameof(proxyConfigVolumeName));
this.proxyConfigMapName = Preconditions.CheckNonWhiteSpace(proxyConfigMapName, nameof(proxyConfigMapName));
this.proxyTrustBundlePath = Preconditions.CheckNonWhiteSpace(proxyTrustBundlePath, nameof(proxyTrustBundlePath));
this.proxyTrustBundleVolumeName = Preconditions.CheckNonWhiteSpace(proxyTrustBundleVolumeName, nameof(proxyTrustBundleVolumeName));
this.proxyTrustBundleConfigMapName = Preconditions.CheckNonWhiteSpace(proxyTrustBundleConfigMapName, nameof(proxyTrustBundleConfigMapName));
this.apiVersion = Preconditions.CheckNonWhiteSpace(apiVersion, nameof(apiVersion));
this.deviceNamespace = Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace));
this.managementUri = Preconditions.CheckNotNull(managementUri, nameof(managementUri));
this.workloadUri = Preconditions.CheckNotNull(workloadUri, nameof(workloadUri));
this.dockerAuthConfig = Preconditions.CheckNotNull(dockerAuthConfig, nameof(dockerAuthConfig));
this.upstreamProtocol = Preconditions.CheckNotNull(upstreamProtocol, nameof(upstreamProtocol));
this.productInfo = productInfo;
this.defaultMapServiceType = defaultMapServiceType;
this.enableServiceCallTracing = enableServiceCallTracing;
this.persistentVolumeName = persistentVolumeName;
this.storageClassName = storageClassName;
this.persistentVolumeClaimSizeMb = persistentVolumeClaimSizeMb;
this.proxy = proxy;
this.closeOnIdleTimeout = closeOnIdleTimeout;
this.idleTimeout = idleTimeout;
this.experimentalFeatures = experimentalFeatures;
this.moduleOwner = moduleOwner;
this.runAsNonRoot = runAsNonRoot;
}
static KubernetesModule CreateKubernetesModule(CreatePodParameters podParameters)
{
var config = new KubernetesConfig("image", podParameters, Option.None <AuthConfig>());
var docker = new DockerModule("module1", "v1", ModuleStatus.Running, RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, Constants.DefaultStartupOrder, DefaultConfigurationInfo, EnvVarsDict);
var owner = new KubernetesModuleOwner("v1", "Owner", "an-owner", "a-uid");
return(new KubernetesModule(docker, config, owner));
}
public static List <V1OwnerReference> ToOwnerReferences(this KubernetesModuleOwner kubeModuleOwner) =>
new List <V1OwnerReference>
{
new V1OwnerReference(
apiVersion: kubeModuleOwner.ApiVersion,
kind: kubeModuleOwner.Kind,
name: kubeModuleOwner.Name,
uid: kubeModuleOwner.Uid)
};
public KubernetesPlanner(
string deviceNamespace,
ResourceName resourceName,
IKubernetes client,
ICommandFactory commandFactory,
ICombinedConfigProvider <CombinedKubernetesConfig> configProvider,
KubernetesModuleOwner moduleOwner)
{
this.resourceName = Preconditions.CheckNotNull(resourceName, nameof(resourceName));
this.deviceNamespace = Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace));
this.client = Preconditions.CheckNotNull(client, nameof(client));
this.commandFactory = Preconditions.CheckNotNull(commandFactory, nameof(commandFactory));
this.configProvider = Preconditions.CheckNotNull(configProvider, nameof(configProvider));
this.moduleOwner = Preconditions.CheckNotNull(moduleOwner);
}
public KubernetesPlanner(
ResourceName resourceName,
string deviceSelector,
string deviceNamespace,
IKubernetes client,
ICommandFactory commandFactory,
ICombinedConfigProvider <CombinedKubernetesConfig> configProvider,
KubernetesModuleOwner moduleOwner)
{
this.resourceName = Preconditions.CheckNotNull(resourceName, nameof(resourceName));
this.deviceSelector = Preconditions.CheckNonWhiteSpace(deviceSelector, nameof(deviceSelector));
this.deviceNamespace = Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace));
this.client = Preconditions.CheckNotNull(client, nameof(client));
this.commandFactory = Preconditions.CheckNotNull(commandFactory, nameof(commandFactory));
this.configProvider = Preconditions.CheckNotNull(configProvider, nameof(configProvider));
this.moduleOwner = Preconditions.CheckNotNull(moduleOwner);
this.serializerSettings = EdgeDeploymentSerialization.SerializerSettings;
}
public EdgeDeploymentCommand(
string deviceNamespace,
ResourceName resourceName,
IKubernetes client,
IEnumerable <IModule> desiredmodules,
Option <EdgeDeploymentDefinition> activeDeployment,
IRuntimeInfo runtimeInfo,
ICombinedConfigProvider <CombinedKubernetesConfig> configProvider,
KubernetesModuleOwner moduleOwner)
{
this.deviceNamespace = KubeUtils.SanitizeK8sValue(Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace)));
this.resourceName = Preconditions.CheckNotNull(resourceName, nameof(resourceName));
this.client = Preconditions.CheckNotNull(client, nameof(client));
this.modules = Preconditions.CheckNotNull(desiredmodules, nameof(desiredmodules)).ToList();
this.activeDeployment = activeDeployment;
this.runtimeInfo = Preconditions.CheckNotNull(runtimeInfo, nameof(runtimeInfo));
this.configProvider = Preconditions.CheckNotNull(configProvider, nameof(configProvider));
this.id = new Lazy <string>(() => this.modules.Aggregate(string.Empty, (prev, module) => module.Name + prev));
this.serializerSettings = EdgeDeploymentSerialization.SerializerSettings;
this.moduleOwner = Preconditions.CheckNotNull(moduleOwner, nameof(moduleOwner));
}
public static async Task <int> MainAsync(IConfiguration configuration)
{
// Bring up the logger before anything else so we can log errors ASAP
ILogger logger = SetupLogger(configuration);
logger.LogInformation("Initializing Edge Agent.");
VersionInfo versionInfo = VersionInfo.Get(VersionInfoFileName);
if (versionInfo != VersionInfo.Empty)
{
logger.LogInformation($"Version - {versionInfo.ToString(true)}");
}
LogLogo(logger);
string mode;
string configSourceConfig;
string backupConfigFilePath;
int maxRestartCount;
TimeSpan intensiveCareTime;
int coolOffTimeUnitInSeconds;
bool usePersistentStorage;
string storagePath;
bool enableNonPersistentStorageBackup;
Option <string> storageBackupPath = Option.None <string>();
string edgeDeviceHostName;
string dockerLoggingDriver;
Dictionary <string, string> dockerLoggingOptions;
IEnumerable <global::Docker.DotNet.Models.AuthConfig> dockerAuthConfig;
int configRefreshFrequencySecs;
ExperimentalFeatures experimentalFeatures;
MetricsConfig metricsConfig;
DiagnosticConfig diagnosticConfig;
bool useServerHeartbeat;
try
{
mode = configuration.GetValue(Constants.ModeKey, "docker");
configSourceConfig = configuration.GetValue <string>("ConfigSource");
backupConfigFilePath = configuration.GetValue <string>("BackupConfigFilePath");
maxRestartCount = configuration.GetValue <int>("MaxRestartCount");
intensiveCareTime = TimeSpan.FromMinutes(configuration.GetValue <int>("IntensiveCareTimeInMinutes"));
coolOffTimeUnitInSeconds = configuration.GetValue("CoolOffTimeUnitInSeconds", 10);
usePersistentStorage = configuration.GetValue("UsePersistentStorage", true);
useServerHeartbeat = configuration.GetValue("UseServerHeartbeat", true);
// Note: Keep in sync with iotedge-check's edge-agent-storage-mounted-from-host check (edgelet/iotedge/src/check/checks/storage_mounted_from_host.rs)
storagePath = GetOrCreateDirectoryPath(configuration.GetValue <string>("StorageFolder"), EdgeAgentStorageFolder);
enableNonPersistentStorageBackup = configuration.GetValue("EnableNonPersistentStorageBackup", false);
if (enableNonPersistentStorageBackup)
{
storageBackupPath = Option.Some(GetOrCreateDirectoryPath(configuration.GetValue <string>("BackupFolder"), EdgeAgentStorageBackupFolder));
}
backupConfigFilePath = GetFullBackupFilePath(storagePath, backupConfigFilePath);
edgeDeviceHostName = configuration.GetValue <string>(Constants.EdgeDeviceHostNameKey);
dockerLoggingDriver = configuration.GetValue <string>("DockerLoggingDriver");
dockerLoggingOptions = configuration.GetSection("DockerLoggingOptions").Get <Dictionary <string, string> >() ?? new Dictionary <string, string>();
dockerAuthConfig = configuration.GetSection("DockerRegistryAuth").Get <List <global::Docker.DotNet.Models.AuthConfig> >() ?? new List <global::Docker.DotNet.Models.AuthConfig>();
NestedEdgeParentUriParser parser = new NestedEdgeParentUriParser();
dockerAuthConfig = dockerAuthConfig.Select(c =>
{
c.Password = parser.ParseURI(c.Password).GetOrElse(c.Password);
return(c);
})
.ToList();
configRefreshFrequencySecs = configuration.GetValue("ConfigRefreshFrequencySecs", 3600);
}
catch (Exception ex)
{
logger.LogCritical(AgentEventIds.Agent, ex, "Fatal error reading the Agent's configuration.");
return(1);
}
IContainer container;
try
{
var builder = new ContainerBuilder();
builder.RegisterModule(new LoggingModule(dockerLoggingDriver, dockerLoggingOptions));
string productInfo =
versionInfo != VersionInfo.Empty ?
$"{Constants.IoTEdgeAgentProductInfoIdentifier}/{versionInfo}" :
Constants.IoTEdgeAgentProductInfoIdentifier;
Option <UpstreamProtocol> upstreamProtocol = configuration.GetValue <string>(Constants.UpstreamProtocolKey).ToUpstreamProtocol();
Option <IWebProxy> proxy = Proxy.Parse(configuration.GetValue <string>("https_proxy"), logger);
bool closeOnIdleTimeout = configuration.GetValue(Constants.CloseOnIdleTimeout, false);
int idleTimeoutSecs = configuration.GetValue(Constants.IdleTimeoutSecs, 300);
TimeSpan idleTimeout = TimeSpan.FromSeconds(idleTimeoutSecs);
experimentalFeatures = ExperimentalFeatures.Create(configuration.GetSection("experimentalFeatures"), logger);
Option <ulong> storageTotalMaxWalSize = GetConfigIfExists <ulong>(Constants.StorageMaxTotalWalSize, configuration, logger);
Option <ulong> storageMaxManifestFileSize = GetConfigIfExists <ulong>(Constants.StorageMaxManifestFileSize, configuration, logger);
Option <int> storageMaxOpenFiles = GetConfigIfExists <int>(Constants.StorageMaxOpenFiles, configuration, logger);
Option <StorageLogLevel> storageLogLevel = GetConfigIfExists <StorageLogLevel>(Constants.StorageLogLevel, configuration, logger);
string iothubHostname;
string deviceId;
string apiVersion = "2018-06-28";
Option <X509Certificate2> manifestTrustBundle = Option.None <X509Certificate2>();
switch (mode.ToLowerInvariant())
{
case Constants.DockerMode:
var dockerUri = new Uri(configuration.GetValue <string>("DockerUri"));
string deviceConnectionString = configuration.GetValue <string>("DeviceConnectionString");
IotHubConnectionStringBuilder connectionStringParser = IotHubConnectionStringBuilder.Create(deviceConnectionString);
deviceId = connectionStringParser.DeviceId;
iothubHostname = connectionStringParser.HostName;
builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxManifestFileSize, storageMaxOpenFiles, storageLogLevel));
builder.RegisterModule(new DockerModule(deviceConnectionString, edgeDeviceHostName, dockerUri, dockerAuthConfig, upstreamProtocol, proxy, productInfo, closeOnIdleTimeout, idleTimeout, useServerHeartbeat, backupConfigFilePath));
break;
case Constants.IotedgedMode:
string managementUri = configuration.GetValue <string>(Constants.EdgeletManagementUriVariableName);
string workloadUri = configuration.GetValue <string>(Constants.EdgeletWorkloadUriVariableName);
iothubHostname = configuration.GetValue <string>(Constants.IotHubHostnameVariableName);
deviceId = configuration.GetValue <string>(Constants.DeviceIdVariableName);
string moduleId = configuration.GetValue(Constants.ModuleIdVariableName, Constants.EdgeAgentModuleIdentityName);
string moduleGenerationId = configuration.GetValue <string>(Constants.EdgeletModuleGenerationIdVariableName);
apiVersion = configuration.GetValue <string>(Constants.EdgeletApiVersionVariableName);
TimeSpan performanceMetricsUpdateFrequency = configuration.GetTimeSpan("PerformanceMetricsUpdateFrequency", TimeSpan.FromMinutes(5));
builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxManifestFileSize, storageMaxOpenFiles, storageLogLevel));
builder.RegisterModule(new EdgeletModule(iothubHostname, deviceId, new Uri(managementUri), new Uri(workloadUri), apiVersion, dockerAuthConfig, upstreamProtocol, proxy, productInfo, closeOnIdleTimeout, idleTimeout, performanceMetricsUpdateFrequency, useServerHeartbeat, backupConfigFilePath));
IEnumerable <X509Certificate2> trustBundle =
await CertificateHelper.GetTrustBundleFromEdgelet(new Uri(workloadUri), apiVersion, Constants.WorkloadApiVersion, moduleId, moduleGenerationId);
CertificateHelper.InstallCertificates(trustBundle, logger);
manifestTrustBundle = await CertificateHelper.GetManifestTrustBundleFromEdgelet(new Uri(workloadUri), apiVersion, Constants.WorkloadApiVersion, moduleId, moduleGenerationId);
break;
case Constants.KubernetesMode:
managementUri = configuration.GetValue <string>(Constants.EdgeletManagementUriVariableName);
workloadUri = configuration.GetValue <string>(Constants.EdgeletWorkloadUriVariableName);
moduleId = configuration.GetValue(Constants.ModuleIdVariableName, Constants.EdgeAgentModuleIdentityName);
moduleGenerationId = configuration.GetValue <string>(Constants.EdgeletModuleGenerationIdVariableName);
apiVersion = configuration.GetValue <string>(Constants.EdgeletApiVersionVariableName);
iothubHostname = configuration.GetValue <string>(Constants.IotHubHostnameVariableName);
deviceId = configuration.GetValue <string>(Constants.DeviceIdVariableName);
string proxyImage = configuration.GetValue <string>(K8sConstants.ProxyImageEnvKey);
Option <string> proxyImagePullSecretName = Option.Maybe(configuration.GetValue <string>(K8sConstants.ProxyImagePullSecretNameEnvKey));
string proxyConfigPath = configuration.GetValue <string>(K8sConstants.ProxyConfigPathEnvKey);
string proxyConfigVolumeName = configuration.GetValue <string>(K8sConstants.ProxyConfigVolumeEnvKey);
string proxyConfigMapName = configuration.GetValue <string>(K8sConstants.ProxyConfigMapNameEnvKey);
string proxyTrustBundlePath = configuration.GetValue <string>(K8sConstants.ProxyTrustBundlePathEnvKey);
string proxyTrustBundleVolumeName = configuration.GetValue <string>(K8sConstants.ProxyTrustBundleVolumeEnvKey);
string proxyTrustBundleConfigMapName = configuration.GetValue <string>(K8sConstants.ProxyTrustBundleConfigMapEnvKey);
PortMapServiceType mappedServiceDefault = GetDefaultServiceType(configuration);
bool enableServiceCallTracing = configuration.GetValue <bool>(K8sConstants.EnableK8sServiceCallTracingName);
bool useMountSourceForVolumeName = configuration.GetValue <bool>(K8sConstants.UseMountSourceForVolumeNameKey, false);
string storageClassName = configuration.GetValue <string>(K8sConstants.StorageClassNameKey);
Option <uint> persistentVolumeClaimDefaultSizeMb = Option.Maybe(configuration.GetValue <uint?>(K8sConstants.PersistentVolumeClaimDefaultSizeInMbKey));
string deviceNamespace = configuration.GetValue <string>(K8sConstants.K8sNamespaceKey);
var kubernetesExperimentalFeatures = KubernetesExperimentalFeatures.Create(configuration.GetSection("experimentalFeatures"), logger);
var moduleOwner = new KubernetesModuleOwner(
configuration.GetValue <string>(K8sConstants.EdgeK8sObjectOwnerApiVersionKey),
configuration.GetValue <string>(K8sConstants.EdgeK8sObjectOwnerKindKey),
configuration.GetValue <string>(K8sConstants.EdgeK8sObjectOwnerNameKey),
configuration.GetValue <string>(K8sConstants.EdgeK8sObjectOwnerUidKey));
bool runAsNonRoot = configuration.GetValue <bool>(K8sConstants.RunAsNonRootKey);
builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxManifestFileSize, storageMaxOpenFiles, storageLogLevel));
builder.RegisterModule(
new KubernetesModule(
iothubHostname,
deviceId,
edgeDeviceHostName,
proxyImage,
proxyImagePullSecretName,
proxyConfigPath,
proxyConfigVolumeName,
proxyConfigMapName,
proxyTrustBundlePath,
proxyTrustBundleVolumeName,
proxyTrustBundleConfigMapName,
apiVersion,
deviceNamespace,
new Uri(managementUri),
new Uri(workloadUri),
dockerAuthConfig,
upstreamProtocol,
Option.Some(productInfo),
mappedServiceDefault,
enableServiceCallTracing,
useMountSourceForVolumeName,
storageClassName,
persistentVolumeClaimDefaultSizeMb,
proxy,
closeOnIdleTimeout,
idleTimeout,
useServerHeartbeat,
kubernetesExperimentalFeatures,
moduleOwner,
runAsNonRoot));
IEnumerable <X509Certificate2> k8sTrustBundle = await CertificateHelper.GetTrustBundleFromEdgelet(new Uri(workloadUri), apiVersion, Constants.WorkloadApiVersion, moduleId, moduleGenerationId);
CertificateHelper.InstallCertificates(k8sTrustBundle, logger);
break;
default:
throw new InvalidOperationException($"Mode '{mode}' not supported.");
}
switch (configSourceConfig.ToLowerInvariant())
{
case "twin":
bool enableStreams = configuration.GetValue(Constants.EnableStreams, false);
int requestTimeoutSecs = configuration.GetValue(Constants.RequestTimeoutSecs, 600);
builder.RegisterModule(
new TwinConfigSourceModule(
iothubHostname,
deviceId,
configuration,
versionInfo,
TimeSpan.FromSeconds(configRefreshFrequencySecs),
enableStreams,
TimeSpan.FromSeconds(requestTimeoutSecs),
experimentalFeatures,
manifestTrustBundle));
break;
case "local":
string localConfigFilePath = GetLocalConfigFilePath(configuration, logger);
builder.RegisterModule(new FileConfigSourceModule(localConfigFilePath, configuration, versionInfo));
break;
default:
throw new InvalidOperationException($"ConfigSource '{configSourceConfig}' not supported.");
}
metricsConfig = new MetricsConfig(configuration);
builder.RegisterModule(new MetricsModule(metricsConfig, iothubHostname, deviceId));
bool diagnosticsEnabled = configuration.GetValue("SendRuntimeQualityTelemetry", true);
diagnosticConfig = new DiagnosticConfig(diagnosticsEnabled, storagePath, configuration);
builder.RegisterModule(new DiagnosticsModule(diagnosticConfig));
container = builder.Build();
}
catch (Exception ex)
{
logger.LogCritical(AgentEventIds.Agent, ex, "Fatal error building application.");
return(1);
}
// TODO move this code to Agent
if (mode.ToLowerInvariant().Equals(Constants.KubernetesMode))
{
// Block agent startup routine until proxy sidecar container is ready
string managementUri = configuration.GetValue <string>(Constants.EdgeletManagementUriVariableName);
string apiVersion = configuration.GetValue <string>(Constants.EdgeletApiVersionVariableName);
ProxyReadinessProbe probe = new ProxyReadinessProbe(new Uri(managementUri), apiVersion);
CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(5));
await probe.WaitUntilProxyIsReady(tokenSource.Token);
// Start environment operator
IKubernetesEnvironmentOperator environmentOperator = container.Resolve <IKubernetesEnvironmentOperator>();
environmentOperator.Start();
// Start the edge deployment operator
IEdgeDeploymentOperator edgeDeploymentOperator = container.Resolve <IEdgeDeploymentOperator>();
edgeDeploymentOperator.Start();
}
// Initialize metrics
if (metricsConfig.Enabled)
{
container.Resolve <IMetricsListener>().Start(logger);
container.Resolve <ISystemResourcesMetrics>().Start(logger);
await container.Resolve <MetadataMetrics>().Start(logger, versionInfo.ToString(true), Newtonsoft.Json.JsonConvert.SerializeObject(experimentalFeatures));
}
// Initialize metric uploading
if (diagnosticConfig.Enabled)
{
MetricsWorker worker = await container.Resolve <Task <MetricsWorker> >();
worker.Start(diagnosticConfig.ScrapeInterval, diagnosticConfig.UploadInterval);
Console.WriteLine($"Scraping frequency: {diagnosticConfig.ScrapeInterval}\nUpload Frequency: {diagnosticConfig.UploadInterval}");
}
(CancellationTokenSource cts, ManualResetEventSlim completed, Option <object> handler)
= ShutdownHandler.Init(ShutdownWaitPeriod, logger);
// Register request handlers
await RegisterRequestHandlers(container);
// Initialize stream request listener
IStreamRequestListener streamRequestListener = await container.Resolve <Task <IStreamRequestListener> >();
streamRequestListener.InitPump();
int returnCode;
using (IConfigSource unused = await container.Resolve <Task <IConfigSource> >())
{
Option <Agent> agentOption = Option.None <Agent>();
try
{
Agent agent = await container.Resolve <Task <Agent> >();
agentOption = Option.Some(agent);
while (!cts.Token.IsCancellationRequested)
{
try
{
await agent.ReconcileAsync(cts.Token).TimeoutAfter(ReconcileTimeout);
}
catch (Exception ex) when(!ex.IsFatal())
{
logger.LogWarning(AgentEventIds.Agent, ex, "Agent reconcile concluded with errors.");
}
await Task.Delay(TimeSpan.FromSeconds(5), cts.Token);
}
logger.LogInformation("Closing module management agent.");
returnCode = 0;
}
catch (OperationCanceledException)
{
logger.LogInformation("Main thread terminated");
returnCode = 0;
}
catch (Exception ex)
{
logger.LogCritical(AgentEventIds.Agent, ex, "Fatal error starting Agent.");
returnCode = 1;
}
// Attempt to report shutdown of Agent
await Cleanup(agentOption, logger);
await CloseDbStoreProviderAsync(container);
if (metricsConfig.Enabled && returnCode == 0)
{
container.Resolve <IDeploymentMetrics>().IndicateCleanShutdown();
}
completed.Set();
}
handler.ForEach(h => GC.KeepAlive(h));
return(returnCode);
}
public static async Task <int> MainAsync(IConfiguration configuration)
{
// Bring up the logger before anything else so we can log errors ASAP
ILogger logger = SetupLogger(configuration);
logger.LogInformation("Initializing Edge Agent.");
VersionInfo versionInfo = VersionInfo.Get(VersionInfoFileName);
if (versionInfo != VersionInfo.Empty)
{
logger.LogInformation($"Version - {versionInfo.ToString(true)}");
}
LogLogo(logger);
string mode;
string configSourceConfig;
string backupConfigFilePath;
int maxRestartCount;
TimeSpan intensiveCareTime;
int coolOffTimeUnitInSeconds;
bool usePersistentStorage;
string storagePath;
bool enableNonPersistentStorageBackup;
Option <string> storageBackupPath = Option.None <string>();
string edgeDeviceHostName;
string dockerLoggingDriver;
Dictionary <string, string> dockerLoggingOptions;
IEnumerable <global::Docker.DotNet.Models.AuthConfig> dockerAuthConfig;
int configRefreshFrequencySecs;
MetricsConfig metricsConfig;
try
{
mode = configuration.GetValue(Constants.ModeKey, "docker");
configSourceConfig = configuration.GetValue <string>("ConfigSource");
backupConfigFilePath = configuration.GetValue <string>("BackupConfigFilePath");
maxRestartCount = configuration.GetValue <int>("MaxRestartCount");
intensiveCareTime = TimeSpan.FromMinutes(configuration.GetValue <int>("IntensiveCareTimeInMinutes"));
coolOffTimeUnitInSeconds = configuration.GetValue("CoolOffTimeUnitInSeconds", 10);
usePersistentStorage = configuration.GetValue("UsePersistentStorage", true);
// Note: Keep in sync with iotedge-check's edge-agent-storage-mounted-from-host check (edgelet/iotedge/src/check/checks/storage_mounted_from_host.rs)
storagePath = GetOrCreateDirectoryPath(configuration.GetValue <string>("StorageFolder"), EdgeAgentStorageFolder);
enableNonPersistentStorageBackup = configuration.GetValue("EnableNonPersistentStorageBackup", false);
if (enableNonPersistentStorageBackup)
{
storageBackupPath = Option.Some(GetOrCreateDirectoryPath(configuration.GetValue <string>("BackupFolder"), EdgeAgentStorageBackupFolder));
}
edgeDeviceHostName = configuration.GetValue <string>(Constants.EdgeDeviceHostNameKey);
dockerLoggingDriver = configuration.GetValue <string>("DockerLoggingDriver");
dockerLoggingOptions = configuration.GetSection("DockerLoggingOptions").Get <Dictionary <string, string> >() ?? new Dictionary <string, string>();
dockerAuthConfig = configuration.GetSection("DockerRegistryAuth").Get <List <global::Docker.DotNet.Models.AuthConfig> >() ?? new List <global::Docker.DotNet.Models.AuthConfig>();
configRefreshFrequencySecs = configuration.GetValue("ConfigRefreshFrequencySecs", 3600);
}
catch (Exception ex)
{
logger.LogCritical(AgentEventIds.Agent, ex, "Fatal error reading the Agent's configuration.");
return(1);
}
IContainer container;
try
{
var builder = new ContainerBuilder();
builder.RegisterModule(new LoggingModule(dockerLoggingDriver, dockerLoggingOptions));
string productInfo = versionInfo != VersionInfo.Empty ? $"{Constants.IoTEdgeAgentProductInfoIdentifier}/{versionInfo}" : Constants.IoTEdgeAgentProductInfoIdentifier;
Option <UpstreamProtocol> upstreamProtocol = configuration.GetValue <string>(Constants.UpstreamProtocolKey).ToUpstreamProtocol();
Option <IWebProxy> proxy = Proxy.Parse(configuration.GetValue <string>("https_proxy"), logger);
bool closeOnIdleTimeout = configuration.GetValue(Constants.CloseOnIdleTimeout, false);
int idleTimeoutSecs = configuration.GetValue(Constants.IdleTimeoutSecs, 300);
TimeSpan idleTimeout = TimeSpan.FromSeconds(idleTimeoutSecs);
ExperimentalFeatures experimentalFeatures = ExperimentalFeatures.Create(configuration.GetSection("experimentalFeatures"), logger);
metricsConfig = new MetricsConfig(experimentalFeatures.EnableMetrics, MetricsListenerConfig.Create(configuration));
string iothubHostname;
string deviceId;
switch (mode.ToLowerInvariant())
{
case Constants.DockerMode:
var dockerUri = new Uri(configuration.GetValue <string>("DockerUri"));
string deviceConnectionString = configuration.GetValue <string>("DeviceConnectionString");
IotHubConnectionStringBuilder connectionStringParser = IotHubConnectionStringBuilder.Create(deviceConnectionString);
deviceId = connectionStringParser.DeviceId;
iothubHostname = connectionStringParser.HostName;
builder.RegisterInstance(metricsConfig.Enabled
? new MetricsProvider("edgeagent", iothubHostname, deviceId)
: new NullMetricsProvider() as IMetricsProvider);
builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, enableNonPersistentStorageBackup, storageBackupPath));
builder.RegisterModule(new DockerModule(deviceConnectionString, edgeDeviceHostName, dockerUri, dockerAuthConfig, upstreamProtocol, proxy, productInfo, closeOnIdleTimeout, idleTimeout));
break;
case Constants.IotedgedMode:
string managementUri = configuration.GetValue <string>(Constants.EdgeletManagementUriVariableName);
string workloadUri = configuration.GetValue <string>(Constants.EdgeletWorkloadUriVariableName);
iothubHostname = configuration.GetValue <string>(Constants.IotHubHostnameVariableName);
deviceId = configuration.GetValue <string>(Constants.DeviceIdVariableName);
string moduleId = configuration.GetValue(Constants.ModuleIdVariableName, Constants.EdgeAgentModuleIdentityName);
string moduleGenerationId = configuration.GetValue <string>(Constants.EdgeletModuleGenerationIdVariableName);
string apiVersion = configuration.GetValue <string>(Constants.EdgeletApiVersionVariableName);
builder.RegisterInstance(metricsConfig.Enabled
? new MetricsProvider("edgeagent", iothubHostname, deviceId)
: new NullMetricsProvider() as IMetricsProvider);
builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath));
builder.RegisterModule(new EdgeletModule(iothubHostname, edgeDeviceHostName, deviceId, new Uri(managementUri), new Uri(workloadUri), apiVersion, dockerAuthConfig, upstreamProtocol, proxy, productInfo, closeOnIdleTimeout, idleTimeout));
break;
case Constants.KubernetesMode:
managementUri = configuration.GetValue <string>(Constants.EdgeletManagementUriVariableName);
workloadUri = configuration.GetValue <string>(Constants.EdgeletWorkloadUriVariableName);
moduleId = configuration.GetValue(Constants.ModuleIdVariableName, Constants.EdgeAgentModuleIdentityName);
moduleGenerationId = configuration.GetValue <string>(Constants.EdgeletModuleGenerationIdVariableName);
apiVersion = configuration.GetValue <string>(Constants.EdgeletApiVersionVariableName);
iothubHostname = configuration.GetValue <string>(Constants.IotHubHostnameVariableName);
deviceId = configuration.GetValue <string>(Constants.DeviceIdVariableName);
string proxyImage = configuration.GetValue <string>(K8sConstants.ProxyImageEnvKey);
Option <string> proxyImagePullSecretName = Option.Maybe(configuration.GetValue <string>(K8sConstants.ProxyImagePullSecretNameEnvKey));
string proxyConfigPath = configuration.GetValue <string>(K8sConstants.ProxyConfigPathEnvKey);
string proxyConfigVolumeName = configuration.GetValue <string>(K8sConstants.ProxyConfigVolumeEnvKey);
string proxyConfigMapName = configuration.GetValue <string>(K8sConstants.ProxyConfigMapNameEnvKey);
string proxyTrustBundlePath = configuration.GetValue <string>(K8sConstants.ProxyTrustBundlePathEnvKey);
string proxyTrustBundleVolumeName = configuration.GetValue <string>(K8sConstants.ProxyTrustBundleVolumeEnvKey);
string proxyTrustBundleConfigMapName = configuration.GetValue <string>(K8sConstants.ProxyTrustBundleConfigMapEnvKey);
PortMapServiceType mappedServiceDefault = GetDefaultServiceType(configuration);
bool enableServiceCallTracing = configuration.GetValue <bool>(K8sConstants.EnableK8sServiceCallTracingName);
string persistentVolumeName = configuration.GetValue <string>(K8sConstants.PersistentVolumeNameKey);
string storageClassName = configuration.GetValue <string>(K8sConstants.StorageClassNameKey);
Option <uint> persistentVolumeClaimDefaultSizeMb = Option.Maybe(configuration.GetValue <uint?>(K8sConstants.PersistentVolumeClaimDefaultSizeInMbKey));
string deviceNamespace = configuration.GetValue <string>(K8sConstants.K8sNamespaceKey);
var kubernetesExperimentalFeatures = KubernetesExperimentalFeatures.Create(configuration.GetSection("experimentalFeatures"), logger);
var moduleOwner = new KubernetesModuleOwner(
configuration.GetValue <string>(K8sConstants.EdgeK8sObjectOwnerApiVersionKey),
configuration.GetValue <string>(K8sConstants.EdgeK8sObjectOwnerKindKey),
configuration.GetValue <string>(K8sConstants.EdgeK8sObjectOwnerNameKey),
configuration.GetValue <string>(K8sConstants.EdgeK8sObjectOwnerUidKey));
bool runAsNonRoot = configuration.GetValue <bool>(K8sConstants.RunAsNonRootKey);
builder.RegisterInstance(metricsConfig.Enabled
? new MetricsProvider("edgeagent", iothubHostname, deviceId)
: new NullMetricsProvider() as IMetricsProvider);
builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath));
builder.RegisterModule(new KubernetesModule(
iothubHostname,
deviceId,
edgeDeviceHostName,
proxyImage,
proxyImagePullSecretName,
proxyConfigPath,
proxyConfigVolumeName,
proxyConfigMapName,
proxyTrustBundlePath,
proxyTrustBundleVolumeName,
proxyTrustBundleConfigMapName,
apiVersion,
deviceNamespace,
new Uri(managementUri),
new Uri(workloadUri),
dockerAuthConfig,
upstreamProtocol,
Option.Some(productInfo),
mappedServiceDefault,
enableServiceCallTracing,
persistentVolumeName,
storageClassName,
persistentVolumeClaimDefaultSizeMb,
proxy,
closeOnIdleTimeout,
idleTimeout,
kubernetesExperimentalFeatures,
moduleOwner,
runAsNonRoot));
break;
default:
throw new InvalidOperationException($"Mode '{mode}' not supported.");
}
switch (configSourceConfig.ToLowerInvariant())
{
case "twin":
bool enableStreams = configuration.GetValue(Constants.EnableStreams, false);
int requestTimeoutSecs = configuration.GetValue(Constants.RequestTimeoutSecs, 600);
builder.RegisterModule(
new TwinConfigSourceModule(
iothubHostname,
deviceId,
backupConfigFilePath,
configuration,
versionInfo,
TimeSpan.FromSeconds(configRefreshFrequencySecs),
enableStreams,
TimeSpan.FromSeconds(requestTimeoutSecs),
experimentalFeatures));
break;
case "local":
string localConfigFilePath = GetLocalConfigFilePath(configuration, logger);
builder.RegisterModule(new FileConfigSourceModule(localConfigFilePath, configuration, versionInfo));
break;
default:
throw new InvalidOperationException($"ConfigSource '{configSourceConfig}' not supported.");
}
container = builder.Build();
}
catch (Exception ex)
{
logger.LogCritical(AgentEventIds.Agent, ex, "Fatal error building application.");
return(1);
}
// Initialize metrics
var metricsLifetimes = new List <IDisposable>();
if (metricsConfig.Enabled)
{
var metricsListener = new MetricsListener(metricsConfig.ListenerConfig, container.Resolve <IMetricsProvider>());
metricsListener.Start(logger);
metricsLifetimes.Add(metricsListener);
}
Dictionary <Type, string> recognizedExceptions = new Dictionary <Type, string>
{
// TODO: Decide what exceptions to recognize and ignore
{ typeof(Newtonsoft.Json.JsonSerializationException), "json_serialization" },
{ typeof(ArgumentException), "argument" },
{ typeof(Rest.HttpOperationException), "http" },
};
HashSet <Type> ignoredExceptions = new HashSet <Type>
{
typeof(TaskCanceledException),
typeof(OperationCanceledException),
};
metricsLifetimes.Add(new ExceptionCounter(recognizedExceptions, ignoredExceptions, container.Resolve <IMetricsProvider>()));
// TODO move this code to Agent
if (mode.ToLowerInvariant().Equals(Constants.KubernetesMode))
{
// Start environment operator
IKubernetesEnvironmentOperator environmentOperator = container.Resolve <IKubernetesEnvironmentOperator>();
environmentOperator.Start();
// Start the edge deployment operator
IEdgeDeploymentOperator edgeDeploymentOperator = container.Resolve <IEdgeDeploymentOperator>();
edgeDeploymentOperator.Start();
}
(CancellationTokenSource cts, ManualResetEventSlim completed, Option <object> handler)
= ShutdownHandler.Init(ShutdownWaitPeriod, logger);
// Register request handlers
await RegisterRequestHandlers(container);
// Initialize stream request listener
IStreamRequestListener streamRequestListener = await container.Resolve <Task <IStreamRequestListener> >();
streamRequestListener.InitPump();
int returnCode;
using (IConfigSource unused = await container.Resolve <Task <IConfigSource> >())
{
Option <Agent> agentOption = Option.None <Agent>();
try
{
Agent agent = await container.Resolve <Task <Agent> >();
agentOption = Option.Some(agent);
while (!cts.Token.IsCancellationRequested)
{
try
{
await agent.ReconcileAsync(cts.Token).TimeoutAfter(ReconcileTimeout);
}
catch (Exception ex) when(!ex.IsFatal())
{
logger.LogWarning(AgentEventIds.Agent, ex, "Agent reconcile concluded with errors.");
}
await Task.Delay(TimeSpan.FromSeconds(5), cts.Token);
}
logger.LogInformation("Closing module management agent.");
returnCode = 0;
}
catch (OperationCanceledException)
{
logger.LogInformation("Main thread terminated");
returnCode = 0;
}
catch (Exception ex)
{
logger.LogCritical(AgentEventIds.Agent, ex, "Fatal error starting Agent.");
returnCode = 1;
}
metricsLifetimes.ForEach(m => m.Dispose());
// Attempt to report shutdown of Agent
await Cleanup(agentOption, logger);
await CloseDbStoreProviderAsync(container);
completed.Set();
}
handler.ForEach(h => GC.KeepAlive(h));
return(returnCode);
}
public void CompareModule()
{
Dictionary <string, EnvVal> goodEnv = new Dictionary <string, EnvVal>();
Dictionary <string, EnvVal> newEnv = new Dictionary <string, EnvVal> {
["a"] = new EnvVal("B")
};
IReadOnlyList <string> dockerEnv = new List <string> {
"c=d"
};
KubernetesConfig goodConfig = new KubernetesConfig("image:tag", CreatePodParameters.Create(), Option.None <AuthConfig>());
KubernetesConfig imageDifferent = new KubernetesConfig("image:newtag", CreatePodParameters.Create(), Option.None <AuthConfig>());
var auth1 = new AuthConfig("secret1");
KubernetesConfig auth1Config = new KubernetesConfig("image:tag", CreatePodParameters.Create(), Option.Some(auth1));
var auth2 = new AuthConfig("secret2");
KubernetesConfig auth2Config = new KubernetesConfig("image:tag", CreatePodParameters.Create(), Option.Some(auth2));
var auth3 = new AuthConfig("secret3");
KubernetesConfig auth3Config = new KubernetesConfig("image:tag", CreatePodParameters.Create(), Option.Some(auth3));
var auth4 = new AuthConfig("secret4");
KubernetesConfig auth4Config = new KubernetesConfig("image:tag", CreatePodParameters.Create(), Option.Some(auth4));
KubernetesConfig createContainerConfigDifferent = new KubernetesConfig("image:tag", CreatePodParameters.Create(dockerEnv), Option.None <AuthConfig>());
ConfigurationInfo goodInfo = new ConfigurationInfo(string.Empty);
KubernetesModuleOwner moduleOwner = new KubernetesModuleOwner("v1", "Deployment", "iotedged", "123");
var m1 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, goodConfig, ImagePullPolicy.OnCreate, moduleOwner);
var m2 = new KubernetesModule("name2", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, goodConfig, ImagePullPolicy.OnCreate, moduleOwner);
var m3 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, goodConfig, ImagePullPolicy.OnCreate, moduleOwner);
var m4 = new KubernetesModule("name1", "v2", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, goodConfig, ImagePullPolicy.OnCreate, moduleOwner);
var m5 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, goodConfig, ImagePullPolicy.OnCreate, moduleOwner);
var m6 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Stopped, RestartPolicy.Always, goodInfo, goodEnv, goodConfig, ImagePullPolicy.OnCreate, moduleOwner);
var m7 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, goodConfig, ImagePullPolicy.OnCreate, moduleOwner);
var m8 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Never, goodInfo, goodEnv, goodConfig, ImagePullPolicy.OnCreate, moduleOwner);
var m9 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, imageDifferent, ImagePullPolicy.OnCreate, moduleOwner);
var m10 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, auth1Config, ImagePullPolicy.OnCreate, moduleOwner);
var m11 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, auth2Config, ImagePullPolicy.OnCreate, moduleOwner);
var m12 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, auth3Config, ImagePullPolicy.OnCreate, moduleOwner);
var m13 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, auth4Config, ImagePullPolicy.OnCreate, moduleOwner);
var m14 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, goodEnv, createContainerConfigDifferent, ImagePullPolicy.OnCreate, moduleOwner);
var m15 = new KubernetesModule("name1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, goodInfo, newEnv, goodConfig, ImagePullPolicy.OnCreate, moduleOwner);
Assert.NotEqual(m1, m2);
Assert.NotEqual(m3, m4);
Assert.NotEqual(m5, m6);
Assert.NotEqual(m7, m8);
Assert.NotEqual(m1, m9);
Assert.NotEqual(m9, m1);
Assert.NotEqual(m10, m9);
Assert.NotEqual(m9, m10);
Assert.NotEqual(m10, m11);
Assert.NotEqual(m11, m10);
Assert.NotEqual(m10, m12);
Assert.NotEqual(m10, m13);
Assert.NotEqual(m11, m14);
Assert.NotEqual(m11, m15);
Assert.True(m5.IsOnlyModuleStatusChanged(m6));
Assert.False(m1.IsOnlyModuleStatusChanged(m2));
Assert.False(m1.IsOnlyModuleStatusChanged(m9));
}
