/// <summary>
/// Deploys the log related containers to a node.
/// </summary>
/// <param name="node">The target hive node.</param>
/// <param name="stepDelay">The step delay if the operation hasn't already been completed.</param>
public void DeployContainers(SshProxy <NodeDefinition> node, TimeSpan stepDelay)
{
Thread.Sleep(stepDelay);
ServiceHelper.StartContainer(node, "neon-log-host", hive.Definition.Image.LogHost, RunOptions.FaultOnError,
new CommandBundle(
"docker run",
"--name", "neon-log-host",
"--detach",
"--restart", "always",
"--volume", "/etc/neon/host-env:/etc/neon/host-env:ro",
"--volume", "/var/log:/hostfs/var/log",
"--network", "host",
"--log-driver", "json-file", // Ensure that we don't log to the pipeline to avoid cascading events.
ServiceHelper.ImagePlaceholderArg));
ServiceHelper.StartContainer(node, "neon-log-metricbeat", hive.Definition.Image.Metricbeat, RunOptions.FaultOnError,
new CommandBundle(
"docker run",
"--name", "neon-log-metricbeat",
"--detach",
"--net", "host",
"--restart", "always",
"--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
"--volume", "/etc/neon/host-env:/etc/neon/host-env:ro",
"--volume", "/proc:/hostfs/proc:ro",
"--volume", "/:/hostfs:ro",
"--env", $"ELASTICSEARCH_URL={hive.Definition.LogEsDataUri}",
"--log-driver", "json-file", // Ensure that we don't log to the pipeline to avoid cascading events.
ServiceHelper.ImagePlaceholderArg));
}
/// <summary>
/// Deploys hive containers to a node.
/// </summary>
/// <param name="node">The target hive node.</param>
/// <param name="stepDelay">The step delay if the operation hasn't already been completed.</param>
public void DeployContainers(SshProxy <NodeDefinition> node, TimeSpan stepDelay)
{
Thread.Sleep(stepDelay);
// NOTE: We only need to deploy the proxy bridges to the pet nodes,
// because these will be deployed as global services on the
// swarm nodes.
if (node.Metadata.IsPet)
{
ServiceHelper.StartContainer(node, "neon-proxy-public-bridge", hive.Definition.Image.Proxy, RunOptions.FaultOnError,
new CommandBundle(
"docker run",
"--detach",
"--name", "neon-proxy-public-bridge",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public-bridge/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public-bridge/proxy-hash",
"--env", "WARN_SECONDS=300",
"--env", "POLL_SECONDS=15",
"--env", "START_SECONDS=10",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--env", "VAULT_SKIP_VERIFY=true",
"--network", "host",
"--restart", "always",
ServiceHelper.ImagePlaceholderArg));
ServiceHelper.StartContainer(node, "neon-proxy-private-bridge", hive.Definition.Image.Proxy, RunOptions.FaultOnError,
new CommandBundle(
"docker run",
"--detach",
"--name", "neon-proxy-private-bridge",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private-bridge/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private-bridge/proxy-hash",
"--env", "WARN_SECONDS=300",
"--env", "POLL_SECONDS=15",
"--env", "START_SECONDS=10",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--env", "VAULT_SKIP_VERIFY=true",
"--network", "host",
"--restart", "always",
ServiceHelper.ImagePlaceholderArg));
}
}
/// <summary>
/// Performs the Docker registry cache related configuration of the node.
/// </summary>
public void Configure(SshProxy <NodeDefinition> node)
{
// NOTE:
//
// We're going to configure the certificates even if the registry cache
// isn't enabled so it'll be easier to upgrade the hive later.
// For managers, upload the individual cache certificate and
// private key files for managers [cache.crt] and [cache.key] at
// [/etc/neon-registry-cache/]. This directory will be
// mapped into the cache container.
//
// Then create the cache's data volume and start the manager's
// Registry cache container.
if (node.Metadata.IsManager)
{
node.InvokeIdempotentAction("setup/registrycache",
() =>
{
// Copy the registry cache certificate and private key to
//
// /etc/neon-registry-cache
node.Status = "run: registry-cache-server-certs.sh";
var copyCommand = new CommandBundle("./registry-cache-server-certs.sh");
var sbCopyScript = new StringBuilder();
sbCopyScript.AppendLine("mkdir -p /etc/neon-registry-cache");
sbCopyScript.AppendLine("chmod 750 /etc/neon-registry-cache");
copyCommand.AddFile($"cache.crt", hive.HiveLogin.HiveCertificate.CertPem);
copyCommand.AddFile($"cache.key", hive.HiveLogin.HiveCertificate.KeyPem);
sbCopyScript.AppendLine($"cp cache.crt /etc/neon-registry-cache/cache.crt");
sbCopyScript.AppendLine($"cp cache.key /etc/neon-registry-cache/cache.key");
sbCopyScript.AppendLine($"chmod 640 /etc/neon-registry-cache/*");
copyCommand.AddFile("registry-cache-server-certs.sh", sbCopyScript.ToString(), isExecutable: true);
node.SudoCommand(copyCommand);
// Upload the cache certificates to every hive node at:
//
// /etc/docker/certs.d/<hostname>:{HiveHostPorts.RegistryCache}/ca.crt
//
// and then have Linux reload the trusted certificates.
node.InvokeIdempotentAction("setup/registrycache-cert",
() =>
{
node.Status = "upload: registry cache certs";
var uploadCommand = new CommandBundle("./registry-cache-client-certs.sh");
var sbUploadScript = new StringBuilder();
uploadCommand.AddFile($"hive-neon-registry-cache.crt", hive.HiveLogin.HiveCertificate.CertPem);
foreach (var manager in hive.Definition.SortedManagers)
{
var cacheHostName = hive.Definition.GetRegistryCacheHost(manager);
sbUploadScript.AppendLine($"mkdir -p /etc/docker/certs.d/{cacheHostName}:{HiveHostPorts.DockerRegistryCache}");
sbUploadScript.AppendLine($"cp hive-neon-registry-cache.crt /etc/docker/certs.d/{cacheHostName}:{HiveHostPorts.DockerRegistryCache}/ca.crt");
}
uploadCommand.AddFile("registry-cache-client-certs.sh", sbUploadScript.ToString(), isExecutable: true);
node.SudoCommand(uploadCommand);
});
// Start the registry cache containers if enabled for the hive.
if (hive.Definition.Docker.RegistryCache)
{
// Create the registry data volume.
node.Status = "create: registry cache volume";
node.SudoCommand(new CommandBundle("docker-volume-create \"neon-registry-cache\""));
// Start the registry cache using the required Docker public registry
// credentials, if any.
var publicRegistryCredentials = hive.Definition.Docker.Registries.SingleOrDefault(r => HiveHelper.IsDockerPublicRegistry(r.Registry));
publicRegistryCredentials = publicRegistryCredentials ?? new RegistryCredentials()
{
Registry = HiveConst.DockerPublicRegistry
};
publicRegistryCredentials.Username = publicRegistryCredentials.Username ?? string.Empty;
publicRegistryCredentials.Password = publicRegistryCredentials.Password ?? string.Empty;
node.Status = "start: neon-registry-cache";
var registry = publicRegistryCredentials.Registry;
if (string.IsNullOrEmpty(registry) || registry.Equals("docker.io", StringComparison.InvariantCultureIgnoreCase))
{
registry = "registry-1.docker.io";
}
ServiceHelper.StartContainer(node, "neon-registry-cache", hive.Definition.Image.RegistryCache, RunOptions.FaultOnError | hive.SecureRunOptions,
new CommandBundle(
"docker run",
"--name", "neon-registry-cache",
"--detach",
"--restart", "always",
"--publish", $"{HiveHostPorts.DockerRegistryCache}:5000",
"--volume", "/etc/neon-registry-cache:/etc/neon-registry-cache:ro", // Registry cache certificates folder
"--volume", "neon-registry-cache:/var/lib/neon-registry-cache",
"--env", $"HOSTNAME={node.Name}.{hive.Definition.Hostnames.RegistryCache}",
"--env", $"REGISTRY=https://{registry}",
"--env", $"USERNAME={publicRegistryCredentials.Username}",
"--env", $"PASSWORD={publicRegistryCredentials.Password}",
"--env", "LOG_LEVEL=info",
ServiceHelper.ImagePlaceholderArg));
}
});
node.Status = string.Empty;
}
}
/// <summary>
/// Deploys RabbitMQ to a cluster node as a container.
/// </summary>
/// <param name="node">The target hive node.</param>
private void DeployHiveMQ(SshProxy <NodeDefinition> node)
{
// Deploy RabbitMQ only on the labeled nodes.
if (node.Metadata.Labels.HiveMQ)
{
// Build a comma separated list of fully qualified RabbitMQ hostnames so we
// can pass them as the CLUSTER environment variable.
var rabbitNodes = hive.Definition.SortedNodes.Where(n => n.Labels.HiveMQ).ToList();
var sbCluster = new StringBuilder();
foreach (var rabbitNode in rabbitNodes)
{
sbCluster.AppendWithSeparator($"{rabbitNode.Name}@{rabbitNode.Name}.{hive.Definition.Hostnames.HiveMQ}", ",");
}
var hipeCompileArgs = new List <string>();
if (hive.Definition.HiveMQ.Precompile)
{
hipeCompileArgs.Add("--env");
hipeCompileArgs.Add("RABBITMQ_HIPE_COMPILE=1");
}
var managementPluginArgs = new List <string>();
if (node.Metadata.Labels.HiveMQManager)
{
hipeCompileArgs.Add("--env");
hipeCompileArgs.Add("MANAGEMENT_PLUGIN=true");
}
// $todo(jeff.lill):
//
// I was unable to get TLS working correctly for RabbitMQ. I'll come back
// and revisit this later:
//
// https://github.com/jefflill/NeonForge/issues/319
ServiceHelper.StartContainer(node, "neon-hivemq", hive.Definition.Image.HiveMQ, RunOptions.FaultOnError,
new CommandBundle(
"docker run",
"--detach",
"--name", "neon-hivemq",
"--env", $"CLUSTER_NAME={hive.Definition.Name}",
"--env", $"CLUSTER_NODES={sbCluster}",
"--env", $"CLUSTER_PARTITION_MODE=autoheal",
"--env", $"NODENAME={node.Name}@{node.Name}.{hive.Definition.Hostnames.HiveMQ}",
"--env", $"RABBITMQ_USE_LONGNAME=true",
"--env", $"RABBITMQ_DEFAULT_USER=sysadmin",
"--env", $"RABBITMQ_DEFAULT_PASS=password",
"--env", $"RABBITMQ_NODE_PORT={HiveHostPorts.HiveMQAMQP}",
"--env", $"RABBITMQ_DIST_PORT={HiveHostPorts.HiveMQDIST}",
"--env", $"RABBITMQ_MANAGEMENT_PORT={HiveHostPorts.HiveMQManagement}",
"--env", $"RABBITMQ_ERLANG_COOKIE={hive.Definition.HiveMQ.ErlangCookie}",
"--env", $"RABBITMQ_VM_MEMORY_HIGH_WATERMARK={hive.Definition.HiveMQ.RamHighWatermark}",
hipeCompileArgs,
managementPluginArgs,
"--env", $"RABBITMQ_DISK_FREE_LIMIT={HiveDefinition.ValidateSize(hive.Definition.HiveMQ.DiskFreeLimit, typeof(HiveMQOptions), nameof(hive.Definition.HiveMQ.DiskFreeLimit))}",
//"--env", $"RABBITMQ_SSL_CERTFILE=/etc/neon/certs/hive.crt",
//"--env", $"RABBITMQ_SSL_KEYFILE=/etc/neon/certs/hive.key",
"--env", $"ERL_EPMD_PORT={HiveHostPorts.HiveMQEPMD}",
"--mount", "type=volume,source=neon-hivemq,target=/var/lib/rabbitmq",
"--mount", "type=bind,source=/etc/neon/certs,target=/etc/neon/certs,readonly",
"--publish", $"{HiveHostPorts.HiveMQEPMD}:{HiveHostPorts.HiveMQEPMD}",
"--publish", $"{HiveHostPorts.HiveMQAMQP}:{HiveHostPorts.HiveMQAMQP}",
"--publish", $"{HiveHostPorts.HiveMQDIST}:{HiveHostPorts.HiveMQDIST}",
"--publish", $"{HiveHostPorts.HiveMQManagement}:{HiveHostPorts.HiveMQManagement}",
"--memory", HiveDefinition.ValidateSize(hive.Definition.HiveMQ.RamLimit, typeof(HiveMQOptions), nameof(hive.Definition.HiveMQ.RamLimit)),
"--restart", "always",
ServiceHelper.ImagePlaceholderArg));
// Wait for the RabbitMQ node to report that it's ready.
var timeout = TimeSpan.FromMinutes(4);
var pollTime = TimeSpan.FromSeconds(2);
node.Status = "hivemq: waiting";
try
{
NeonHelper.WaitFor(
() =>
{
var readyReponse = node.SudoCommand($"docker exec neon-hivemq rabbitmqctl node_health_check -n {node.Name}@{node.Name}.{hive.Definition.Hostnames.HiveMQ}", node.DefaultRunOptions & ~RunOptions.FaultOnError);
return(readyReponse.ExitCode == 0);
},
timeout: timeout,
pollTime: pollTime);
}
catch (TimeoutException)
{
node.Fault($"RabbitMQ not ready after waiting [{timeout}].");
return;
}
node.Status = "hivemq: ready";
}
}
