/// <summary>
/// Verify that the XenServer is ready to provision the hive virtual machines.
/// </summary>
/// <param name="xenSshProxy">The XenServer SSH proxy.</param>
private void VerifyReady(SshProxy <XenClient> xenSshProxy)
{
// $todo(jeff.lill):
//
// It would be nice to verify that XenServer actually has enough
// resources (RAM, DISK, and perhaps CPU) here as well.
var xenHost = xenSshProxy.Metadata;
var nodes = GetHostedNodes(xenHost);
xenSshProxy.Status = "check virtual machines";
var vmNames = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase);
foreach (var vm in xenHost.Machine.List())
{
vmNames.Add(vm.NameLabel);
}
foreach (var hostedNode in nodes)
{
var vmName = GetVmName(hostedNode);
if (vmNames.Contains(vmName))
{
xenSshProxy.Fault($"XenServer [{xenHost.Name}] already hosts a virtual machine named [{vmName}].");
return;
}
}
}
/// <summary>
/// Uploads the configuration files for the target operating system to the server.
/// </summary>
/// <typeparam name="Metadata">The node metadata type.</typeparam>
/// <param name="node">The remote node.</param>
/// <param name="hiveDefinition">The hive definition or <c>null</c>.</param>
public static void UploadConfigFiles <Metadata>(this SshProxy <Metadata> node, HiveDefinition hiveDefinition = null)
where Metadata : class
{
Covenant.Requires <ArgumentNullException>(node != null);
// Clear the contents of the configuration folder.
node.Status = $"clear: {HiveHostFolders.Config}";
node.SudoCommand($"rm -rf {HiveHostFolders.Config}/*.*");
// Upload the files.
node.Status = "upload: config files";
foreach (var file in Program.LinuxFolder.GetFolder("conf").Files())
{
node.UploadFile(hiveDefinition, file, $"{HiveHostFolders.Config}/{file.Name}");
}
// Secure the files and make the scripts executable.
node.SudoCommand($"chmod 644 {HiveHostFolders.Config}/*.*");
node.SudoCommand($"chmod 744 {HiveHostFolders.Config}/*.sh");
node.Status = "copied";
}
/// <summary>
/// Updates docker on a hive node.
/// </summary>
/// <param name="hive">The target hive.</param>
/// <param name="node">The target node.</param>
/// <param name="dockerPackageUri">The Docker Debian package URI.</param>
private static void UpdateDocker(HiveProxy hive, SshProxy <NodeDefinition> node, string dockerPackageUri)
{
try
{
if (node.Metadata.InSwarm)
{
node.Status = "swarm: drain services";
hive.Docker.DrainNode(node.Name);
}
node.Status = "stop: docker";
node.SudoCommand("systemctl stop docker").EnsureSuccess();
node.Status = "download: docker package";
node.SudoCommand($"curl {Program.CurlOptions} {dockerPackageUri} -o /tmp/docker.deb").EnsureSuccess();
node.Status = "update: docker";
node.SudoCommand("gdebi /tmp/docker.deb").EnsureSuccess();
node.SudoCommand("rm /tmp/docker.deb");
node.Status = "restart: docker";
node.SudoCommand("systemctl start docker").EnsureSuccess();
if (node.Metadata.InSwarm)
{
node.Status = "swarm: activate";
hive.Docker.ActivateNode(node.Name);
}
}
catch (Exception e)
{
node.Fault($"[docker] update failed: {NeonHelper.ExceptionError(e)}");
}
}
/// <summary>
/// Updates Linux on a specific node.
/// </summary>
/// <param name="node">The target node.</param>
/// <param name="stepDelay">The step delay.</param>
private void UpdateLinux(SshProxy <NodeDefinition> node, TimeSpan stepDelay)
{
if (node.Metadata.InSwarm)
{
node.Status = "swarm: drain services";
hive.Docker.DrainNode(node.Name);
}
node.Status = "run: safe-apt-get dist-upgrade -yq";
node.SudoCommand("safe-apt-get dist-upgrade -yq");
node.Reboot();
if (node.Metadata.InSwarm)
{
// Put the node back into ACTIVE mode (from DRAIN).
node.Status = "swarm: activate";
hive.Docker.ActivateNode(node.Name);
}
// Give the node a chance to become active again in the swarm
// for containers to restart and for service tasks to redeploy
node.Status = $"stabilizing ({Program.WaitSeconds}s)";
Thread.Sleep(TimeSpan.FromSeconds(Program.WaitSeconds));
}
/// <summary>
/// Configures the hive logging related services.
/// </summary>
/// <param name="firstManager">The first hive proxy manager.</param>
public void Configure(SshProxy <NodeDefinition> firstManager)
{
if (!hive.Definition.Log.Enabled)
{
return;
}
firstManager.InvokeIdempotentAction("setup/log-services",
() =>
{
var steps = new ConfigStepList();
AddElasticsearchSteps(steps);
if (hive.Definition.Dashboard.Kibana)
{
AddKibanaSteps(steps);
}
AddCollectorSteps(steps);
hive.Configure(steps);
firstManager.Status = string.Empty;
});
}
/// <summary>
/// Verifies that a hive worker or pet node is healthy.
/// </summary>
/// <param name="node">The server node.</param>
/// <param name="hiveDefinition">The hive definition.</param>
public static void CheckWorkersOrPet(SshProxy <NodeDefinition> node, HiveDefinition hiveDefinition)
{
Covenant.Requires <ArgumentNullException>(node != null);
Covenant.Requires <ArgumentException>(node.Metadata.IsWorker || node.Metadata.IsPet);
Covenant.Requires <ArgumentNullException>(hiveDefinition != null);
if (!node.IsFaulted)
{
CheckWorkerNtp(node, hiveDefinition);
}
if (!node.IsFaulted)
{
CheckDocker(node, hiveDefinition);
}
if (!node.IsFaulted)
{
CheckConsul(node, hiveDefinition);
}
if (!node.IsFaulted)
{
CheckVault(node, hiveDefinition);
}
node.Status = "healthy";
}
/// <summary>
/// Starts a neonHIVE related Docker container on a node and also uploads a script
/// to make it easy to restart the container manually or for hive updates.
/// </summary>
/// <param name="node">The target hive node.</param>
/// <param name="containerName">Identifies the container.</param>
/// <param name="image">The Docker image to be used by the container.</param>
/// <param name="runOptions">Optional run options (defaults to <see cref="RunOptions.FaultOnError"/>).</param>
/// <param name="commands">The commands required to start the container.</param>
/// <remarks>
/// <para>
/// This method performs the following steps:
/// </para>
/// <list type="number">
/// <item>
/// Passes <paramref name="image"/> to <see cref="Program.ResolveDockerImage(string)"/> to
/// obtain the actual image to be started.
/// </item>
/// <item>
/// Generates the first few lines of the script file that sets the
/// default image as the <c>TARGET_IMAGE</c> macro and then overrides
/// this with the script parameter (if there is one).
/// </item>
/// <item>
/// Appends the commands to the script, replacing any text that matches
/// <see cref="ImagePlaceholderArg"/> with <c>${TARGET_IMAGE}</c> to make it easy
/// for services to be upgraded later.
/// </item>
/// <item>
/// Starts the container.
/// </item>
/// <item>
/// Uploads the generated script to the node to [<see cref="HiveHostFolders.Scripts"/>/<paramref name="containerName"/>.sh].
/// </item>
/// </list>
/// </remarks>
public static void StartContainer(SshProxy <NodeDefinition> node, string containerName, string image, RunOptions runOptions = RunOptions.FaultOnError, params IBashCommandFormatter[] commands)
{
Covenant.Requires <ArgumentNullException>(node != null);
Covenant.Requires <ArgumentNullException>(!string.IsNullOrWhiteSpace(containerName));
Covenant.Requires <ArgumentNullException>(!string.IsNullOrWhiteSpace(image));
Covenant.Requires <ArgumentNullException>(commands != null);
Covenant.Requires <ArgumentNullException>(commands.Length > 0);
node.Status = $"start: {containerName}";
// Generate the container start script.
var script = CreateStartScript(containerName, image, true, commands);
// Upload the script to the target node and set permissions.
var scriptPath = LinuxPath.Combine(HiveHostFolders.Scripts, $"{containerName}.sh");
node.UploadText(scriptPath, script);
node.SudoCommand($"chmod 740 {scriptPath}");
// Run the script without a parameter to start the container.
node.IdempotentDockerCommand($"setup/{containerName}", null, runOptions, scriptPath);
node.Status = string.Empty;
}
/// <summary>
/// Executes a Vault command on a specific node using the root Vault token.
/// </summary>
/// <param name="node">The target node.</param>
/// <param name="commandLine">The Vault command.</param>
private void ExecuteOnNode(SshProxy <NodeDefinition> node, CommandLine commandLine)
{
var response = node.SudoCommand($"export VAULT_TOKEN={vaultCredentials.RootToken} && {remoteVaultPath} {commandLine}", RunOptions.IgnoreRemotePath | RunOptions.Redact);
Console.WriteLine(response.AllText);
Program.Exit(response.ExitCode);
}
/// <summary>
/// Verifies Consul health.
/// </summary>
/// <param name="node">The manager node.</param>
/// <param name="hiveDefinition">The hive definition.</param>
private static void CheckConsul(SshProxy <NodeDefinition> node, HiveDefinition hiveDefinition)
{
node.Status = "checking: consul";
// Verify that the daemon is running.
switch (Program.ServiceManager)
{
case ServiceManager.Systemd:
{
var output = node.SudoCommand("systemctl status consul", RunOptions.LogOutput).OutputText;
if (!output.Contains("Active: active (running)"))
{
node.Fault($"Consul deamon is not running.");
return;
}
}
break;
default:
throw new NotImplementedException();
}
}
/// <summary>
/// Deploys the log related containers to a node.
/// </summary>
/// <param name="node">The target hive node.</param>
/// <param name="stepDelay">The step delay if the operation hasn't already been completed.</param>
public void DeployContainers(SshProxy <NodeDefinition> node, TimeSpan stepDelay)
{
Thread.Sleep(stepDelay);
ServiceHelper.StartContainer(node, "neon-log-host", hive.Definition.Image.LogHost, RunOptions.FaultOnError,
new CommandBundle(
"docker run",
"--name", "neon-log-host",
"--detach",
"--restart", "always",
"--volume", "/etc/neon/host-env:/etc/neon/host-env:ro",
"--volume", "/var/log:/hostfs/var/log",
"--network", "host",
"--log-driver", "json-file", // Ensure that we don't log to the pipeline to avoid cascading events.
ServiceHelper.ImagePlaceholderArg));
ServiceHelper.StartContainer(node, "neon-log-metricbeat", hive.Definition.Image.Metricbeat, RunOptions.FaultOnError,
new CommandBundle(
"docker run",
"--name", "neon-log-metricbeat",
"--detach",
"--net", "host",
"--restart", "always",
"--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
"--volume", "/etc/neon/host-env:/etc/neon/host-env:ro",
"--volume", "/proc:/hostfs/proc:ro",
"--volume", "/:/hostfs:ro",
"--env", $"ELASTICSEARCH_URL={hive.Definition.LogEsDataUri}",
"--log-driver", "json-file", // Ensure that we don't log to the pipeline to avoid cascading events.
ServiceHelper.ImagePlaceholderArg));
}
/// <summary>
/// Edits the [neon-proxy-public-bridge.sh] and [neon-proxy-private-bridge.sh]
/// scripts to remove the [VAULT_CREDENTIALS] environment variable so the new
/// .NET based proxy bridge image will work properly.
/// </summary>
/// <param name="node">The target node.</param>
private void UpdateProxyBridgeScripts(SshProxy <NodeDefinition> node)
{
var scriptNames =
new string[]
{
"neon-proxy-public-bridge.sh",
"neon-proxy-private-bridge.sh"
};
foreach (var scriptName in scriptNames)
{
var scriptPath = LinuxPath.Combine(HiveHostFolders.Scripts, scriptName);
var scriptText = node.DownloadText(scriptName);
var sbEdited = new StringBuilder();
using (var reader = new StringReader(scriptText))
{
foreach (var line in reader.Lines())
{
if (!line.Contains("--env VAULT_CREDENTIALS="))
{
sbEdited.AppendLineLinux(line);
}
}
}
node.UploadText(scriptPath, sbEdited.ToString(), permissions: "700");
}
}
/// <summary>
/// <para>
/// Edits the [/etc/hosts] file on all hive nodes so that the line:
/// </para>
/// <code>
/// 127.0.1.1 {hostname}
/// </code>
/// <para>
/// is changed to:
/// </para>
/// <code>
/// {node.PrivateAddress} {hostname}
/// </code>
/// <para>
/// Hashicorp Vault cannot restart with the old setting, complaining about a
/// <b>""missing API address</b>.
/// </para>
/// </summary>
/// <param name="node">The target node.</param>
private void EditEtcHosts(SshProxy <NodeDefinition> node)
{
node.InvokeIdempotentAction(GetIdempotentTag("edit-etc-hosts"),
() =>
{
var etcHosts = node.DownloadText("/etc/hosts");
var sbEtcHosts = new StringBuilder();
using (var reader = new StringReader(etcHosts))
{
foreach (var line in reader.Lines())
{
if (line.StartsWith("127.0.1.1"))
{
var nodeAddress = node.PrivateAddress.ToString();
var separator = new string(' ', Math.Max(16 - nodeAddress.Length, 1));
sbEtcHosts.AppendLine($"{nodeAddress}{separator}{node.Name}");
}
else
{
sbEtcHosts.AppendLine(line);
}
}
}
node.UploadText("/etc/hosts", sbEtcHosts.ToString(), permissions: "644");
node.SudoCommand("systemctl restart vault");
});
}
/// <summary>
/// Updates Docker on a specific node.
/// </summary>
/// <param name="node">The target node.</param>
/// <param name="stepDelay">The step delay.</param>
private void UpdateDocker(SshProxy <NodeDefinition> node, TimeSpan stepDelay)
{
if (node.GetDockerVersion() >= (SemanticVersion)version)
{
return; // Already updated
}
if (node.Metadata.InSwarm)
{
node.Status = "swarm: drain services";
hive.Docker.DrainNode(node.Name);
}
node.Status = "run: safe-apt-get update";
node.SudoCommand("safe-apt-get update");
node.Status = $"run: safe-apt-get install -yq {dockerPackageUri}";
node.SudoCommand($"safe-apt-get install -yq {dockerPackageUri}");
node.Status = $"restart: docker";
node.SudoCommand("systemctl restart docker");
if (node.Metadata.InSwarm)
{
// Put the node back into ACTIVE mode (from DRAIN).
node.Status = "swarm: activate";
hive.Docker.ActivateNode(node.Name);
}
node.Status = $"stabilizing ({Program.WaitSeconds}s)";
Thread.Sleep(TimeSpan.FromSeconds(Program.WaitSeconds));
}
/// <summary>
/// Releases any resources associated with the instance.
/// </summary>
public void Dispose()
{
if (SshProxy == null)
{
SshProxy.Dispose();
SshProxy = null;
}
}
/// <summary>
/// Removes the Docker python module from all nodes because it conflicts with
/// Docker related Ansible playbooks.
/// </summary>
/// <param name="node">The target node.</param>
private void RemoveDockerPython(SshProxy <NodeDefinition> node)
{
node.InvokeIdempotentAction(GetIdempotentTag("remove-docker-py"),
() =>
{
node.SudoCommand("su sysadmin -c 'pip uninstall -y docker'", RunOptions.LogOnErrorOnly);
});
}
/// <summary>
/// Returns steps that upload a text file to a cluster node.
/// </summary>
/// <param name="node">The cluster node to receive the upload.</param>
/// <param name="path">The target path on the Linux node.</param>
/// <param name="text">The input text.</param>
/// <param name="tabStop">Optionally expands TABs into spaces when non-zero.</param>
/// <param name="outputEncoding">Optionally specifies the output text encoding (defaults to UTF-8).</param>
/// <param name="permissions">Optionally specifies target file permissions (must be <c>chmod</c> compatible).</param>
/// <returns>The steps.</returns>
public IEnumerable <ConfigStep> GetFileUploadSteps(SshProxy <NodeDefinition> node, string path, string text, int tabStop = 0, Encoding outputEncoding = null, string permissions = null)
{
Covenant.Requires <ArgumentNullException>(node != null);
return(GetFileUploadSteps(new List <SshProxy <NodeDefinition> >()
{
node
}, path, text, tabStop, outputEncoding, permissions));
}
/// <summary>
/// Ensures that the Docker <b>config.json</b> file for the node's root
/// user matches that for the sysadmin user.
/// </summary>
private void SyncDockerConf(SshProxy <NodeDefinition> node)
{
// We also need to manage the login for the [root] account due
// to issue
//
// https://github.com/jefflill/NeonForge/issues/265
// $hack(jeff.lill):
//
// We're simply going ensure that the [/root/.docker/config.json]
// file matches the equivalent file for the node sysadmin account,
// removing the root file if this was deleted for sysadmin.
//
// This is a bit of a hack because it assumes that the Docker config
// for the root and sysadmin account never diverge, which is probably
// a reasonable assumption given that these are managed hosts.
//
// We're also going to ensure that these directories and files have the
// correct owners and permissions.
var bundle = new CommandBundle("./sync.sh");
bundle.AddFile("sync.sh",
$@"#!/bin/bash
if [ ! -d /root/.docker ] ; then
mkdir -p /root/.docker
fi
if [ -f /home/{node.Username}/.docker/config.json ] ; then
cp /home/{node.Username}/.docker/config.json /root/.docker/config.json
else
if [ -f /root/.docker/config.json ] ; then
rm /root/.docker/config.json
fi
fi
if [ -d /root/.docker ] ; then
chown -R root:root /root/.docker
chmod 660 /root/.docker/*
fi
if [ -d /home/{node.Username}/.docker ] ; then
chown -R {node.Username}:{node.Username} /home/{node.Username}/.docker
chmod 660 /home/{node.Username}/.docker/*
fi
",
isExecutable: true);
var response = node.SudoCommand(bundle);
if (response.ExitCode != 0)
{
throw new HiveException(response.ErrorSummary);
}
}
/// <summary>
/// Verifies that the node has the correct operating system installed.
/// </summary>
/// <param name="node">The target cluster node.</param>
/// <param name="stepDelay">Ignored.</param>
public static void VerifyOS(SshProxy <NodeDefinition> node, TimeSpan stepDelay)
{
node.Status = "check: OS";
// $todo(jeff.lill): We're currently hardcoded to Ubuntu 18.04.x
if (!node.OsName.Equals("Ubuntu", StringComparison.InvariantCultureIgnoreCase) || node.OsVersion < Version.Parse("18.04"))
{
node.Fault("Expected: Ubuntu 18.04.x");
}
}
/// <summary>
/// Updates a service or container start script on a hive node with a new image.
/// </summary>
/// <param name="node">The target hive node.</param>
/// <param name="scriptName">The script name (without the <b>.sh</b>).</param>
/// <param name="image">The fully qualified image name.</param>
private static void UpdateStartScript(SshProxy <NodeDefinition> node, string scriptName, string image)
{
var scriptPath = LinuxPath.Combine(HiveHostFolders.Scripts, $"{scriptName}.sh");
node.Status = $"edit: {scriptPath}";
if (node.FileExists(scriptPath))
{
var curScript = node.DownloadText(scriptPath);
var sbNewScript = new StringBuilder();
// Scan for the generated code section and then replace the first
// line that looks like:
//
// TARGET_IMAGE=OLD-IMAGE
//
// with the new image and then upload the change.
using (var reader = new StringReader(curScript))
{
var inGenerated = false;
var wasEdited = false;
foreach (var line in reader.Lines())
{
if (wasEdited)
{
sbNewScript.AppendLine(line);
continue;
}
if (!inGenerated && line.StartsWith(ServiceHelper.ParamSectionMarker))
{
inGenerated = true;
}
if (line.StartsWith("TARGET_IMAGE="))
{
sbNewScript.AppendLine($"TARGET_IMAGE={image}");
wasEdited = true;
}
else
{
sbNewScript.AppendLine(line);
}
}
}
node.UploadText(scriptPath, sbNewScript.ToString(), permissions: "740");
}
node.Status = string.Empty;
}
/// <summary>
/// Verifies that a cluster worker node is healthy.
/// </summary>
/// <param name="node">The server node.</param>
/// <param name="clusterDefinition">The cluster definition.</param>
public static void CheckWorker(SshProxy <NodeDefinition> node, ClusterDefinition clusterDefinition)
{
Covenant.Requires <ArgumentNullException>(node != null);
Covenant.Requires <ArgumentException>(node.Metadata.IsWorker);
Covenant.Requires <ArgumentNullException>(clusterDefinition != null);
if (!node.IsFaulted)
{
CheckWorkerNtp(node, clusterDefinition);
}
node.Status = "healthy";
}
/// <summary>
/// Install the virtual machine template on the XenServer if it's not already present.
/// </summary>
/// <param name="xenSshProxy">The XenServer SSH proxy.</param>
private void CheckVmTemplate(SshProxy <XenClient> xenSshProxy)
{
var xenHost = xenSshProxy.Metadata;
var templateName = hive.Definition.Hosting.XenServer.TemplateName;
xenSshProxy.Status = "check template";
if (xenHost.Template.Find(templateName) == null)
{
xenSshProxy.Status = "download vm template (slow)";
xenHost.Template.Install(hive.Definition.Hosting.XenServer.HostXvaUri, templateName, hive.Definition.Hosting.XenServer.StorageRepository);
}
}
/// <summary>
/// Invokes a low-level <b>xe CLI</b> command on the remote XenServer host
/// that returns text, throwing an exception on failure.
/// </summary>
/// <param name="command">The <b>xe CLI</b> command.</param>
/// <param name="args">The optional arguments formatted as <b>name=value</b>.</param>
/// <returns>The command response.</returns>
/// <exception cref="XenException">Thrown if the operation failed.</exception>
public CommandResponse SafeInvoke(string command, params string[] args)
{
VerifyNotDisposed();
var response = SshProxy.RunCommand($"xe {command}", runOptions, args);
if (response.ExitCode != 0)
{
throw new XenException($"XE-COMMAND: {command} MESSAGE: {response.ErrorText}");
}
return(response);
}
/// <summary>
/// Verifies Docker health.
/// </summary>
/// <param name="node">The target hive node.</param>
/// <param name="hiveDefinition">The hive definition.</param>
private static void CheckDocker(SshProxy <NodeDefinition> node, HiveDefinition hiveDefinition)
{
node.Status = "checking: docker";
// This is a super simple ping to verify that Docker appears to be running.
var response = node.SudoCommand("docker info");
if (response.ExitCode != 0)
{
node.Fault($"Docker: {response.AllText}");
}
}
/// <summary>
/// Deploys hive containers to a node.
/// </summary>
/// <param name="node">The target hive node.</param>
/// <param name="stepDelay">The step delay if the operation hasn't already been completed.</param>
public void DeployContainers(SshProxy <NodeDefinition> node, TimeSpan stepDelay)
{
Thread.Sleep(stepDelay);
// NOTE: We only need to deploy the proxy bridges to the pet nodes,
// because these will be deployed as global services on the
// swarm nodes.
if (node.Metadata.IsPet)
{
ServiceHelper.StartContainer(node, "neon-proxy-public-bridge", hive.Definition.Image.Proxy, RunOptions.FaultOnError,
new CommandBundle(
"docker run",
"--detach",
"--name", "neon-proxy-public-bridge",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/public-bridge/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/public-bridge/proxy-hash",
"--env", "WARN_SECONDS=300",
"--env", "POLL_SECONDS=15",
"--env", "START_SECONDS=10",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--env", "VAULT_SKIP_VERIFY=true",
"--network", "host",
"--restart", "always",
ServiceHelper.ImagePlaceholderArg));
ServiceHelper.StartContainer(node, "neon-proxy-private-bridge", hive.Definition.Image.Proxy, RunOptions.FaultOnError,
new CommandBundle(
"docker run",
"--detach",
"--name", "neon-proxy-private-bridge",
"--mount", "type=bind,src=/etc/neon/host-env,dst=/etc/neon/host-env,readonly=true",
"--mount", "type=bind,src=/usr/local/share/ca-certificates,dst=/mnt/host/ca-certificates,readonly=true",
"--env", "CONFIG_KEY=neon/service/neon-proxy-manager/proxies/private-bridge/proxy-conf",
"--env", "CONFIG_HASH_KEY=neon/service/neon-proxy-manager/proxies/private-bridge/proxy-hash",
"--env", "WARN_SECONDS=300",
"--env", "POLL_SECONDS=15",
"--env", "START_SECONDS=10",
"--env", "LOG_LEVEL=INFO",
"--env", "DEBUG=false",
"--env", "VAULT_SKIP_VERIFY=true",
"--network", "host",
"--restart", "always",
ServiceHelper.ImagePlaceholderArg));
}
}
/// <summary>
/// Uploads a resource file to the remote server after performing any necessary preprocessing.
/// </summary>
/// <typeparam name="TMetadata">The node metadata type.</typeparam>
/// <param name="node">The remote node.</param>
/// <param name="hiveDefinition">The hive definition or <c>null</c>.</param>
/// <param name="file">The resource file.</param>
/// <param name="targetPath">The target path on the remote server.</param>
private static void UploadFile <TMetadata>(this SshProxy <TMetadata> node, HiveDefinition hiveDefinition, ResourceFiles.File file, string targetPath)
where TMetadata : class
{
using (var input = file.ToStream())
{
if (file.HasVariables)
{
// We need to expand any variables. Note that if we don't have a
// hive definition or for undefined variables, we're going to
// have the variables expand to the empty string.
using (var msExpanded = new MemoryStream())
{
using (var writer = new StreamWriter(msExpanded))
{
var preprocessReader =
new PreprocessReader(new StreamReader(input))
{
DefaultVariable = string.Empty,
ExpandVariables = true,
ProcessCommands = false,
StripComments = false
};
if (hiveDefinition != null)
{
SetHiveVariables(preprocessReader, hiveDefinition, node.Metadata as NodeDefinition);
}
foreach (var line in preprocessReader.Lines())
{
writer.WriteLine(line);
}
writer.Flush();
msExpanded.Position = 0;
node.UploadText(targetPath, msExpanded, tabStop: 4, outputEncoding: Encoding.UTF8);
}
}
}
else
{
node.UploadText(targetPath, input, tabStop: 4, outputEncoding: Encoding.UTF8);
}
}
}
/// <summary>
/// Uploads the setup and other scripts and tools for the target operating system to the server.
/// </summary>
/// <typeparam name="TMetadata">The server's metadata type.</typeparam>
/// <param name="server">The remote server.</param>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <param name="kubeSetupInfo">The Kubernetes setup details.</param>
public static void UploadResources <TMetadata>(this SshProxy <TMetadata> server, ClusterDefinition clusterDefinition, KubeSetupInfo kubeSetupInfo)
where TMetadata : class
{
Covenant.Requires <ArgumentNullException>(server != null, nameof(server));
Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));
Covenant.Requires <ArgumentNullException>(kubeSetupInfo != null, nameof(kubeSetupInfo));
//-----------------------------------------------------------------
// Upload resource files to the setup folder.
server.Status = $"clear: {KubeHostFolders.Setup}";
server.SudoCommand($"rm -rf {KubeHostFolders.Setup}/*.*");
// Upload the setup files.
server.Status = "upload: setup scripts";
foreach (var file in Program.LinuxFolder.GetFolder("setup").Files())
{
server.UploadFile(clusterDefinition, kubeSetupInfo, file, $"{KubeHostFolders.Setup}/{file.Name}");
}
// Make the setup scripts executable.
server.SudoCommand($"chmod 744 {KubeHostFolders.Setup}/*");
//-----------------------------------------------------------------
// Upload files to the bin folder.
server.Status = $"clear: {KubeHostFolders.Bin}";
server.SudoCommand($"rm -rf {KubeHostFolders.Bin}/*.*");
// Upload the tool files. Note that we're going to strip out the [.sh]
// file type to make these easier to run.
server.Status = "upload: binary files";
foreach (var file in Program.LinuxFolder.GetFolder("binary").Files())
{
server.UploadFile(clusterDefinition, kubeSetupInfo, file, $"{KubeHostFolders.Bin}/{file.Name.Replace(".sh", string.Empty)}");
}
// Make the scripts executable.
server.SudoCommand($"chmod 744 {KubeHostFolders.Bin}/*");
}
/// <summary>
/// Executes a <b>docker config create</b> command.
/// </summary>
/// <param name="node">The target node.</param>
/// <param name="rightCommandLine">The right split of the command line.</param>
private void ConfigCreate(SshProxy <NodeDefinition> node, CommandLine rightCommandLine)
{
// We're expecting a command like:
//
// docker config create [OPTIONS] CONFIG file|-
//
// where CONFIG is the name of the configuration and and [file]
// is the path to the config file or [-] indicates that
// the config is streaming in on stdin.
//
// We're going to run this as a command bundle that includes
// the config file.
if (rightCommandLine.Arguments.Length != 4)
{
Console.Error.WriteLine("*** ERROR: Expected: docker config create [OPTIONS] CONFIG file|-");
Program.Exit(0);
}
string fileArg = rightCommandLine.Arguments[3];
byte[] configData;
if (fileArg == "-")
{
configData = NeonHelper.ReadStandardInputBytes();
}
else
{
configData = File.ReadAllBytes(fileArg);
}
// Create and execute a command bundle. Note that we're going to hardcode
// the config data path to [config.data].
rightCommandLine.Items[rightCommandLine.Items.Length - 1] = "config.data";
var bundle = new CommandBundle("docker", rightCommandLine.Items);
bundle.AddFile("config.data", configData);
var response = node.SudoCommand(bundle, RunOptions.None);
Console.Write(response.AllText);
Program.Exit(response.ExitCode);
}
/// <summary>
/// Update the Elasticsearch container launch scripts to enable automatic
/// memory settings based on any cgroup limits.
/// </summary>
/// <param name="node">The target node.</param>
private void UpdateElasticsearch(SshProxy <NodeDefinition> node)
{
// This method is called for all cluster nodes, even those
// that aren't currently hosting Elasticsearch, so we can
// update any scripts that may have been orphaned (for
// consistency).
//
// The update consists of replacing the script line that
// sets the [ES_JAVA_OPTS] environment variable with:
//
// --env ES_JAVA_OPTS=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
//
// To ensure that this feature is enabled in favor of the
// old hacked memory level settings.
var scriptPath = LinuxPath.Combine(HiveHostFolders.Scripts, "neon-log-esdata.sh");
node.InvokeIdempotentAction(GetIdempotentTag("neon-log-esdata"),
() =>
{
if (node.FileExists(scriptPath))
{
node.Status = $"edit: {scriptPath}";
var orgScript = node.DownloadText(scriptPath);
var newScript = new StringBuilder();
foreach (var line in new StringReader(orgScript).Lines())
{
if (line.Contains("ES_JAVA_OPTS="))
{
newScript.AppendLine(" --env \"ES_JAVA_OPTS=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap\" \\");
}
else
{
newScript.AppendLine(line);
}
}
node.UploadText(scriptPath, newScript.ToString(), permissions: "");
node.Status = string.Empty;
}
});
}
/// <summary>
/// Verifies Vault health for a node.
/// </summary>
/// <param name="node">The node.</param>
/// <param name="hiveDefinition">The hive definition.</param>
private static void CheckVault(SshProxy <NodeDefinition> node, HiveDefinition hiveDefinition)
{
// $todo(jeff.lill): Implement this.
return;
node.Status = "checking: vault";
// This is a minimal health test that just verifies that Vault
// is listening for requests. We're going to ping the local
// Vault instance at [/v1/sys/health].
//
// Note that this should return a 500 status code with some
// JSON content. The reason for this is because we have not
// yet initialized and unsealed the vault.
var targetUrl = $"https://{node.Metadata.PrivateAddress}:{hiveDefinition.Vault.Port}/v1/sys/health?standbycode=200";
using (var client = new HttpClient())
{
try
{
var response = client.GetAsync(targetUrl).Result;
if (response.StatusCode != HttpStatusCode.OK &&
response.StatusCode != HttpStatusCode.InternalServerError)
{
node.Fault($"Vault: Unexpected HTTP response status [{(int) response.StatusCode}={response.StatusCode}]");
return;
}
if (!response.Content.Headers.ContentType.MediaType.Equals("application/json", StringComparison.OrdinalIgnoreCase))
{
node.Fault($"Vault: Unexpected content type [{response.Content.Headers.ContentType.MediaType}]");
return;
}
}
catch (Exception e)
{
node.Fault($"Vault: {NeonHelper.ExceptionError(e)}");
}
}
}
/// <summary>
/// Updates the <b>/etc/systemd/system/ceph-fuse-hivefs.service</b> to adjust restart
/// behavior: https://github.com/jefflill/NeonForge/issues/364
/// </summary>
/// <param name="node">The target node.</param>
private void UpdateCephFuse(SshProxy <NodeDefinition> node)
{
node.InvokeIdempotentAction(GetIdempotentTag("ceph-fuse"),
() =>
{
node.UploadText("/etc/systemd/system/ceph-fuse-hivefs.service",
@"[Unit]
Description=Ceph FUSE client (for /mnt/hivefs)
After=network-online.target local-fs.target time-sync.target
Wants=network-online.target local-fs.target time-sync.target
Conflicts=umount.target
PartOf=ceph-fuse.target
[Service]
EnvironmentFile=-/etc/default/ceph
Environment=CLUSTER=ceph
ExecStart=/usr/bin/ceph-fuse -f -o nonempty --cluster ${CLUSTER} /mnt/hivefs
TasksMax=infinity
# These settings configure the service to restart always after
# waiting 5 seconds between attempts for up to a 365 days (effectively
# forever). [StartLimitIntervalSec] is set to the number of seconds
# in a year and [StartLimitBurst] is set to the number of 5 second
# intervals in [StartLimitIntervalSec].
Restart=always
RestartSec=5
StartLimitIntervalSec=31536000
StartLimitBurst=6307200
[Install]
WantedBy=ceph-fuse.target
WantedBy=docker.service
",
permissions: "644");
// Tell systemd to regenerate its configuration.
node.SudoCommand("systemctl daemon-reload");
});
}
