/// <summary>
/// Executes a <b>docker config create</b> command.
/// </summary>
/// <param name="node">The target node.</param>
/// <param name="rightCommandLine">The right split of the command line.</param>
private void ConfigCreate(SshProxy <NodeDefinition> node, CommandLine rightCommandLine)
{
// We're expecting a command like:
//
// docker config create [OPTIONS] CONFIG file|-
//
// where CONFIG is the name of the configuration and and [file]
// is the path to the config file or [-] indicates that
// the config is streaming in on stdin.
//
// We're going to run this as a command bundle that includes
// the config file.
if (rightCommandLine.Arguments.Length != 4)
{
Console.Error.WriteLine("*** ERROR: Expected: docker config create [OPTIONS] CONFIG file|-");
Program.Exit(0);
}
string fileArg = rightCommandLine.Arguments[3];
byte[] configData;
if (fileArg == "-")
{
configData = NeonHelper.ReadStandardInputBytes();
}
else
{
configData = File.ReadAllBytes(fileArg);
}
// Create and execute a command bundle. Note that we're going to hardcode
// the config data path to [config.data].
rightCommandLine.Items[rightCommandLine.Items.Length - 1] = "config.data";
var bundle = new CommandBundle("docker", rightCommandLine.Items);
bundle.AddFile("config.data", configData);
var response = node.SudoCommand(bundle, RunOptions.None);
Console.Write(response.AllText);
Program.Exit(response.ExitCode);
}
/// <summary>
/// Updates Consul on a specific node.
/// </summary>
/// <param name="node">The target node.</param>
/// <param name="stepDelay">The step delay.</param>
private void UpdateConsul(SshProxy <NodeDefinition> node, TimeSpan stepDelay)
{
if (node.GetConsulVersion() >= (SemanticVersion)version)
{
return; // Already updated
}
node.Status = $"stop: consul";
node.SudoCommand("systemctl stop consul");
node.Status = $"update: consul";
var bundle = new CommandBundle("./install.sh", version);
bundle.AddFile("install.sh",
$@"#!/bin/bash
set -euo pipefail
curl {Program.CurlOptions} https://releases.hashicorp.com/consul/$1/consul_$1_linux_amd64.zip -o /tmp/consul.zip 1>&2
unzip -u /tmp/consul.zip -d /tmp
cp /tmp/consul /usr/local/bin
chmod 770 /usr/local/bin/consul
rm /tmp/consul.zip
rm /tmp/consul
",
isExecutable: true);
node.SudoCommand(bundle);
node.Status = $"restart: consul";
node.SudoCommand("systemctl restart consul");
if (node.Metadata.IsManager)
{
node.Status = $"stabilizing ({Program.WaitSeconds}s)";
Thread.Sleep(TimeSpan.FromSeconds(Program.WaitSeconds));
}
}
/// <summary>
/// Updates Vault on a specific node.
/// </summary>
/// <param name="node">The target node.</param>
/// <param name="stepDelay">The step delay.</param>
private void UpdateVault(SshProxy <NodeDefinition> node, TimeSpan stepDelay)
{
if (node.GetVaultVersion() >= (SemanticVersion)version)
{
return; // Already updated
}
node.Status = $"update: vault";
var bundle = new CommandBundle("./install.sh", version);
bundle.AddFile("install.sh",
$@"#!/bin/bash
set -euo pipefail
curl {Program.CurlOptions} https://releases.hashicorp.com/vault/$1/vault_$1_linux_amd64.zip -o /tmp/vault.zip 1>&2
unzip -o /tmp/vault.zip -d /tmp
rm /tmp/vault.zip
mv /tmp/vault /usr/local/bin/vault
chmod 700 /usr/local/bin/vault
",
isExecutable: true);
node.SudoCommand(bundle);
if (node.Metadata.IsManager)
{
node.Status = $"restart: vault";
node.SudoCommand("systemctl restart vault");
node.Status = $"unseal: vault";
hive.Vault.Unseal();
node.Status = $"stabilizing ({Program.WaitSeconds}s)";
Thread.Sleep(TimeSpan.FromSeconds(Program.WaitSeconds));
}
}
/// <summary>
/// Executes a <b>docker deploy</b> or <b>docker stack deploy</b> command.
/// </summary>
/// <param name="node">The target node.</param>
/// <param name="rightCommandLine">The right split of the command line.</param>
private void Deploy(SshProxy <NodeDefinition> node, CommandLine rightCommandLine)
{
string path = null;
// We're going to upload the file specified by the first
// [--bundle-file], [--compose-file], or [-c] option.
for (int i = 0; i < rightCommandLine.Items.Length; i++)
{
switch (rightCommandLine.Items[i])
{
case "--bundle-file":
case "--compose-file":
case "-c":
path = rightCommandLine.Items.Skip(i + 1).FirstOrDefault();
break;
}
if (path != null)
{
// Convert the command line argument to a bundle relative path.
rightCommandLine.Items[i + 1] = Path.GetFileName(rightCommandLine.Items[i + 1]);
break;
}
}
if (path == null)
{
// If that didn't work, try looking for arguments like:
//
// --bundle-file=PATH
var patterns =
new string[]
{
"--bundle-file=",
"--compose-file=",
"-c="
};
for (int i = 0; i < rightCommandLine.Items.Length; i++)
{
var item = rightCommandLine.Items[i];
foreach (var pattern in patterns)
{
if (item.StartsWith(pattern))
{
path = item.Substring(pattern.Length);
// Convert the command line argument to a bundle relative path.
rightCommandLine.Items[i] = pattern + Path.GetFileName(path);
break;
}
}
if (path != null)
{
break;
}
}
}
if (path == null)
{
Console.Error.WriteLine("*** ERROR: No DAB or compose file specified.");
Program.Exit(0);
}
var bundle = new CommandBundle("docker", rightCommandLine.Items);
bundle.AddFile(Path.GetFileName(path), File.ReadAllText(path));
var response = node.SudoCommand(bundle);
Console.Write(response.AllText);
Program.Exit(response.ExitCode);
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
var split = commandLine.Split(SplitItem);
var leftCommandLine = split.Left;
var rightCommandLine = split.Right;
// Basic initialization.
if (leftCommandLine.HasHelpOption)
{
Console.WriteLine(usage);
Program.Exit(0);
}
// Initialize the hive.
var hiveLogin = Program.ConnectHive();
hive = new HiveProxy(hiveLogin);
// Determine which node we're going to target.
var node = (SshProxy <NodeDefinition>)null;
var nodeName = leftCommandLine.GetOption("--node", null);
if (!string.IsNullOrEmpty(nodeName))
{
node = hive.GetNode(nodeName);
}
else
{
node = hive.GetReachableManager();
}
if (rightCommandLine == null)
{
Console.Error.WriteLine("*** ERROR: The [consul] command requires the \"--\" argument.");
Program.Exit(1);
}
var command = rightCommandLine.Arguments.FirstOrDefault();
switch (command)
{
case "watch":
Console.Error.WriteLine("*** ERROR: [neon consul watch] is not supported.");
Program.Exit(1);
break;
case "monitor":
// We'll just relay the output we receive from the remote command
// until the user kills this process.
using (var shell = node.CreateSudoShell())
{
shell.WriteLine($"sudo {remoteConsulPath} {rightCommandLine}");
while (true)
{
var line = shell.ReadLine();
if (line == null)
{
break; // Just being defensive
}
Console.WriteLine(line);
}
}
break;
default:
if (rightCommandLine.Items.LastOrDefault() == "-")
{
// This is the special case where we need to pipe the standard input sent
// to this command on to Consul on the remote machine. We're going to use
// a CommandBundle by uploading the standard input data as a file.
var bundle = new CommandBundle($"cat stdin.dat | {remoteConsulPath} {rightCommandLine}");
bundle.AddFile("stdin.dat", NeonHelper.ReadStandardInputBytes());
var response = node.SudoCommand(bundle, RunOptions.IgnoreRemotePath);
Console.WriteLine(response.AllText);
Program.Exit(response.ExitCode);
}
else if (rightCommandLine.StartsWithArgs("kv", "put") && rightCommandLine.Arguments.Length == 4 && rightCommandLine.Arguments[3].StartsWith("@"))
{
// We're going to special case PUT when saving a file
// whose name is prefixed with "@".
var filePath = rightCommandLine.Arguments[3].Substring(1);
var bundle = new CommandBundle($"{remoteConsulPath} {rightCommandLine}");
bundle.AddFile(filePath, File.ReadAllBytes(filePath));
var response = node.SudoCommand(bundle, RunOptions.IgnoreRemotePath);
Console.Write(response.AllText);
Program.Exit(response.ExitCode);
}
else
{
// All we need to do is to execute the command remotely. We're going to special case
// the [consul kv get ...] command to process the result as binary.
CommandResponse response;
if (rightCommandLine.ToString().StartsWith("kv get"))
{
response = node.SudoCommand($"{remoteConsulPath} {rightCommandLine}", RunOptions.IgnoreRemotePath | RunOptions.BinaryOutput);
using (var remoteStandardOutput = response.OpenOutputBinaryStream())
{
if (response.ExitCode != 0)
{
// Looks like Consul writes its errors to standard output, so
// I'm going to open a text reader and write those lines
// to standard error.
using (var reader = new StreamReader(remoteStandardOutput))
{
foreach (var line in reader.Lines())
{
Console.Error.WriteLine(line);
}
}
}
else
{
// Write the remote binary output to standard output.
using (var output = Console.OpenStandardOutput())
{
var buffer = new byte[8192];
int cb;
while (true)
{
cb = remoteStandardOutput.Read(buffer, 0, buffer.Length);
if (cb == 0)
{
break;
}
output.Write(buffer, 0, cb);
}
}
}
}
}
else
{
response = node.SudoCommand($"{remoteConsulPath} {rightCommandLine}", RunOptions.IgnoreRemotePath);
Console.WriteLine(response.AllText);
}
Program.Exit(response.ExitCode);
}
break;
}
}
/// <summary>
/// Performs the Docker registry cache related configuration of the node.
/// </summary>
public void Configure(SshProxy <NodeDefinition> node)
{
// NOTE:
//
// We're going to configure the certificates even if the registry cache
// isn't enabled so it'll be easier to upgrade the hive later.
// For managers, upload the individual cache certificate and
// private key files for managers [cache.crt] and [cache.key] at
// [/etc/neon-registry-cache/]. This directory will be
// mapped into the cache container.
//
// Then create the cache's data volume and start the manager's
// Registry cache container.
if (node.Metadata.IsManager)
{
node.InvokeIdempotentAction("setup/registrycache",
() =>
{
// Copy the registry cache certificate and private key to
//
// /etc/neon-registry-cache
node.Status = "run: registry-cache-server-certs.sh";
var copyCommand = new CommandBundle("./registry-cache-server-certs.sh");
var sbCopyScript = new StringBuilder();
sbCopyScript.AppendLine("mkdir -p /etc/neon-registry-cache");
sbCopyScript.AppendLine("chmod 750 /etc/neon-registry-cache");
copyCommand.AddFile($"cache.crt", hive.HiveLogin.HiveCertificate.CertPem);
copyCommand.AddFile($"cache.key", hive.HiveLogin.HiveCertificate.KeyPem);
sbCopyScript.AppendLine($"cp cache.crt /etc/neon-registry-cache/cache.crt");
sbCopyScript.AppendLine($"cp cache.key /etc/neon-registry-cache/cache.key");
sbCopyScript.AppendLine($"chmod 640 /etc/neon-registry-cache/*");
copyCommand.AddFile("registry-cache-server-certs.sh", sbCopyScript.ToString(), isExecutable: true);
node.SudoCommand(copyCommand);
// Upload the cache certificates to every hive node at:
//
// /etc/docker/certs.d/<hostname>:{HiveHostPorts.RegistryCache}/ca.crt
//
// and then have Linux reload the trusted certificates.
node.InvokeIdempotentAction("setup/registrycache-cert",
() =>
{
node.Status = "upload: registry cache certs";
var uploadCommand = new CommandBundle("./registry-cache-client-certs.sh");
var sbUploadScript = new StringBuilder();
uploadCommand.AddFile($"hive-neon-registry-cache.crt", hive.HiveLogin.HiveCertificate.CertPem);
foreach (var manager in hive.Definition.SortedManagers)
{
var cacheHostName = hive.Definition.GetRegistryCacheHost(manager);
sbUploadScript.AppendLine($"mkdir -p /etc/docker/certs.d/{cacheHostName}:{HiveHostPorts.DockerRegistryCache}");
sbUploadScript.AppendLine($"cp hive-neon-registry-cache.crt /etc/docker/certs.d/{cacheHostName}:{HiveHostPorts.DockerRegistryCache}/ca.crt");
}
uploadCommand.AddFile("registry-cache-client-certs.sh", sbUploadScript.ToString(), isExecutable: true);
node.SudoCommand(uploadCommand);
});
// Start the registry cache containers if enabled for the hive.
if (hive.Definition.Docker.RegistryCache)
{
// Create the registry data volume.
node.Status = "create: registry cache volume";
node.SudoCommand(new CommandBundle("docker-volume-create \"neon-registry-cache\""));
// Start the registry cache using the required Docker public registry
// credentials, if any.
var publicRegistryCredentials = hive.Definition.Docker.Registries.SingleOrDefault(r => HiveHelper.IsDockerPublicRegistry(r.Registry));
publicRegistryCredentials = publicRegistryCredentials ?? new RegistryCredentials()
{
Registry = HiveConst.DockerPublicRegistry
};
publicRegistryCredentials.Username = publicRegistryCredentials.Username ?? string.Empty;
publicRegistryCredentials.Password = publicRegistryCredentials.Password ?? string.Empty;
node.Status = "start: neon-registry-cache";
var registry = publicRegistryCredentials.Registry;
if (string.IsNullOrEmpty(registry) || registry.Equals("docker.io", StringComparison.InvariantCultureIgnoreCase))
{
registry = "registry-1.docker.io";
}
ServiceHelper.StartContainer(node, "neon-registry-cache", hive.Definition.Image.RegistryCache, RunOptions.FaultOnError | hive.SecureRunOptions,
new CommandBundle(
"docker run",
"--name", "neon-registry-cache",
"--detach",
"--restart", "always",
"--publish", $"{HiveHostPorts.DockerRegistryCache}:5000",
"--volume", "/etc/neon-registry-cache:/etc/neon-registry-cache:ro", // Registry cache certificates folder
"--volume", "neon-registry-cache:/var/lib/neon-registry-cache",
"--env", $"HOSTNAME={node.Name}.{hive.Definition.Hostnames.RegistryCache}",
"--env", $"REGISTRY=https://{registry}",
"--env", $"USERNAME={publicRegistryCredentials.Username}",
"--env", $"PASSWORD={publicRegistryCredentials.Password}",
"--env", "LOG_LEVEL=info",
ServiceHelper.ImagePlaceholderArg));
}
});
node.Status = string.Empty;
}
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
// Split the command line on "--".
var split = commandLine.Split(SplitItem);
var leftCommandLine = split.Left;
var rightCommandLine = split.Right;
// Basic initialization.
if (leftCommandLine.HasHelpOption)
{
Console.WriteLine(usage);
Program.Exit(0);
}
// Initialize the hive.
var hiveLogin = Program.ConnectHive();
hive = new HiveProxy(hiveLogin);
vaultCredentials = hiveLogin.VaultCredentials;
if (rightCommandLine == null)
{
Console.WriteLine("*** ERROR: The [--] command separator is required.");
Console.WriteLine();
Console.WriteLine(usage);
Program.Exit(1);
}
// Determine which node we're going to target.
SshProxy <NodeDefinition> node;
string nodeName = leftCommandLine.GetOption("--node", null);
CommandBundle bundle;
CommandResponse response;
bool failed = false;
if (!string.IsNullOrEmpty(nodeName))
{
node = hive.GetNode(nodeName);
}
else
{
node = hive.GetReachableManager();
}
var command = rightCommandLine.Arguments.FirstOrDefault();;
switch (command)
{
case "init":
case "rekey":
case "server":
case "ssh":
Console.Error.WriteLine($"*** ERROR: [neon vault {command}] is not supported.");
Program.Exit(1);
break;
case "seal":
// Just run the command on the target node if one was specified.
if (nodeName != null)
{
ExecuteOnNode(node, rightCommandLine);
return;
}
// We need to seal the Vault instance on every manager node unless a
// specific node was requsted via [--node].
//
// Note also that it's not possible to seal a node that's in standby
// mode so we'll restart the Vault container instead.
Console.WriteLine();
if (!string.IsNullOrEmpty(nodeName))
{
response = node.SudoCommand($"vault-direct status");
if (response.ExitCode != 0)
{
Console.WriteLine($"[{node.Name}] is already sealed");
}
else
{
var vaultStatus = new VaultStatus(response.OutputText);
if (vaultStatus.HAMode == "standby")
{
Console.WriteLine($"[{node.Name}] restarting to seal standby node...");
response = node.SudoCommand($"systemctl restart vault");
if (response.ExitCode == 0)
{
Console.WriteLine($"[{node.Name}] sealed");
}
else
{
Console.WriteLine($"[{node.Name}] restart failed");
}
}
else
{
Console.WriteLine($"[{node.Name}] sealing...");
response = node.SudoCommand($"export VAULT_TOKEN={vaultCredentials.RootToken} && vault-direct operator seal", RunOptions.Redact);
if (response.ExitCode == 0)
{
Console.WriteLine($"[{node.Name}] sealed");
}
else
{
Console.WriteLine($"[{node.Name}] seal failed");
}
}
}
failed = response.ExitCode != 0;
}
else
{
foreach (var manager in hive.Managers)
{
try
{
response = manager.SudoCommand($"vault-direct status");
}
catch (SshOperationTimeoutException)
{
Console.WriteLine($"[{manager.Name}] ** unavailable **");
continue;
}
var vaultStatus = new VaultStatus(response.OutputText);
if (response.ExitCode != 0)
{
Console.WriteLine($"[{manager.Name}] is already sealed");
}
else
{
response = manager.SudoCommand($"vault-direct operator seal");
if (vaultStatus.HAMode == "standby")
{
Console.WriteLine($"[{manager.Name}] restarting to seal standby node...");
response = manager.SudoCommand($"systemctl restart vault");
if (response.ExitCode == 0)
{
Console.WriteLine($"[{manager.Name}] restart/seal [standby]");
}
else
{
Console.WriteLine($"[{manager.Name}] restart/seal failed [standby]");
}
}
else
{
Console.WriteLine($"[{manager.Name}] sealing...");
response = manager.SudoCommand($"export VAULT_TOKEN={vaultCredentials.RootToken} && vault-direct operator seal", RunOptions.Redact);
if (response.ExitCode == 0)
{
Console.WriteLine($"[{manager.Name}] sealed");
}
else
{
failed = true;
Console.WriteLine($"[{manager.Name}] seal failed");
}
}
}
}
if (!failed)
{
// Disable auto unseal until the operator explicitly unseals Vault again.
hive.Consul.Client.KV.PutBool($"{HiveConst.GlobalKey}/{HiveGlobals.UserDisableAutoUnseal}", true).Wait();
}
Program.Exit(failed ? 1 : 0);
}
break;
case "status":
// Just run the command on the target node if one was specified.
if (nodeName != null)
{
ExecuteOnNode(node, rightCommandLine);
return;
}
// We need to obtain the status from the Vault instance on every manager node unless a
// specific node was requsted via [--node].
Console.WriteLine();
if (!string.IsNullOrEmpty(nodeName))
{
response = node.SudoCommand("vault-direct status");
Console.WriteLine(response.AllText);
Program.Exit(response.ExitCode);
}
else
{
var allSealed = true;
foreach (var manager in hive.Managers)
{
try
{
response = manager.SudoCommand("vault-direct status");
}
catch (SshOperationTimeoutException)
{
Console.WriteLine($"[{manager.Name}] ** unavailable **");
continue;
}
var vaultStatus = new VaultStatus(response.OutputText);
if (response.ExitCode == 0)
{
allSealed = false;
var status = vaultStatus.HAMode;
if (status == "active")
{
status += " <-- LEADER";
}
Console.WriteLine($"[{manager.Name}] unsealed {status}");
}
else if (response.ExitCode == 2)
{
Console.WriteLine($"[{manager.Name}] sealed");
}
else
{
failed = true;
Console.WriteLine($"[{manager.Name}] error getting status");
}
}
if (allSealed)
{
Program.Exit(2);
}
else
{
Program.Exit(failed ? 1 : 0);
}
}
break;
case "unseal":
// Just run the command on the target node if one was specified.
if (nodeName != null)
{
ExecuteOnNode(node, rightCommandLine);
return;
}
// We need to unseal the Vault instance on every manager node unless a
// specific node was requsted via [--node].
Console.WriteLine();
if (!string.IsNullOrEmpty(nodeName))
{
// Verify that the instance isn't already unsealed.
response = node.SudoCommand($"vault-direct status");
if (response.ExitCode == 2)
{
Console.WriteLine($"[{node.Name}] unsealing...");
}
else if (response.ExitCode == 0)
{
Console.WriteLine($"[{node.Name}] is already unsealed");
break;
}
else
{
Console.WriteLine($"[{node.Name}] unseal failed");
Program.Exit(response.ExitCode);
}
// Note that we're passing the [-reset] option to ensure that
// any keys from previous attempts have been cleared.
node.SudoCommand($"vault-direct operator unseal -reset");
foreach (var key in vaultCredentials.UnsealKeys)
{
response = node.SudoCommand($"vault-direct operator unseal {key}", RunOptions.None);
if (response.ExitCode != 0)
{
Console.WriteLine($"[{node.Name}] unseal failed");
Program.Exit(1);
}
}
Console.WriteLine($"[{node.Name}] unsealed");
}
else
{
var commandFailed = false;
foreach (var manager in hive.Managers)
{
// Verify that the instance isn't already unsealed.
try
{
response = manager.SudoCommand($"vault-direct status");
}
catch (SshOperationTimeoutException)
{
Console.WriteLine($"[{manager.Name}] ** unavailable **");
continue;
}
if (response.ExitCode == 2)
{
Console.WriteLine($"[{manager.Name}] unsealing...");
}
else if (response.ExitCode == 0)
{
Console.WriteLine($"[{manager.Name}] is already unsealed");
continue;
}
else
{
Console.WriteLine($"[{manager.Name}] unseal failed");
continue;
}
// Note that we're passing the [-reset] option to ensure that
// any keys from previous attempts have been cleared.
manager.SudoCommand($"vault-direct operator unseal -reset");
foreach (var key in vaultCredentials.UnsealKeys)
{
response = manager.SudoCommand($"vault-direct operator unseal {key}", RunOptions.None);
if (response.ExitCode != 0)
{
failed = true;
commandFailed = true;
Console.WriteLine($"[{manager.Name}] unseal failed");
}
}
if (!failed)
{
Console.WriteLine($"[{manager.Name}] unsealed");
}
}
if (!commandFailed)
{
// Reenable auto unseal.
hive.Consul.Client.KV.PutBool($"{HiveConst.GlobalKey}/{HiveGlobals.UserDisableAutoUnseal}", false).Wait();
}
Program.Exit(commandFailed ? 1 : 0);
}
break;
case "write":
{
// We need handle any [key=@file] arguments specially by including them
// in a command bundle as data files.
var files = new List <CommandFile>();
var commandString = rightCommandLine.ToString();
foreach (var dataArg in rightCommandLine.Arguments.Skip(2))
{
var fields = dataArg.Split(new char[] { '=' }, 2);
if (fields.Length == 2 && fields[1].StartsWith("@"))
{
var fileName = fields[1].Substring(1);
var localFileName = $"{files.Count}.data";
files.Add(
new CommandFile()
{
Path = localFileName,
Data = File.ReadAllBytes(fileName)
});
commandString = commandString.Replace($"@{fileName}", $"@{localFileName}");
}
}
bundle = new CommandBundle($"export VAULT_TOKEN={vaultCredentials.RootToken} && {remoteVaultPath} {commandString}");
foreach (var file in files)
{
bundle.Add(file);
}
response = node.SudoCommand(bundle, RunOptions.IgnoreRemotePath | RunOptions.Redact);
Console.WriteLine(response.AllText);
Program.Exit(response.ExitCode);
}
break;
case "policy-write":
// The last command line item is either:
//
// * A "-", indicating that the content should come from standard input.
// * A file name prefixed by "@"
// * A string holding JSON or HCL
if (rightCommandLine.Items.Length < 2)
{
response = node.SudoCommand($"export VAULT_TOKEN={vaultCredentials.RootToken} && {remoteVaultPath} {rightCommandLine}", RunOptions.IgnoreRemotePath | RunOptions.Redact);
Console.WriteLine(response.AllText);
Program.Exit(response.ExitCode);
}
var lastItem = rightCommandLine.Items.Last();
var policyText = string.Empty;
if (lastItem == "-")
{
policyText = NeonHelper.ReadStandardInputText();
}
else if (lastItem.StartsWith("@"))
{
policyText = File.ReadAllText(lastItem.Substring(1), Encoding.UTF8);
}
else
{
policyText = lastItem;
}
// We're going to upload a text file holding the policy text and
// then run a script piping the policy file into the Vault command passed,
// with the last item replaced by a "-".
bundle = new CommandBundle("./set-vault-policy.sh.sh");
var sbScript = new StringBuilder();
sbScript.AppendLine($"export VAULT_TOKEN={vaultCredentials.RootToken}");
sbScript.Append($"cat policy | {remoteVaultPath}");
for (int i = 0; i < rightCommandLine.Items.Length - 1; i++)
{
sbScript.Append(' ');
sbScript.Append(rightCommandLine.Items[i]);
}
sbScript.AppendLine(" -");
bundle.AddFile("set-vault-policy.sh", sbScript.ToString(), isExecutable: true);
bundle.AddFile("policy", policyText);
response = node.SudoCommand(bundle, RunOptions.IgnoreRemotePath | RunOptions.Redact);
Console.WriteLine(response.AllText);
Program.Exit(response.ExitCode);
break;
default:
ExecuteOnNode(node, rightCommandLine);
break;
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
string hostname;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]");
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (context.HasErrors)
{
return;
}
var manager = hive.GetReachableManager();
var sbErrorNodes = new StringBuilder();
// Determine whether the registry service is already deployed and
// also retrieve the registry credentials from Vault if present.
// Note that the current registry hostname will be persisted to
// Consul at [neon/service/neon-registry/hostname] when a registry
// is deployed.
context.WriteLine(AnsibleVerbosity.Trace, $"Inspecting the [neon-registry] service.");
var currentService = hive.Docker.InspectService("neon-registry");
context.WriteLine(AnsibleVerbosity.Trace, $"Getting current registry hostname from Consul.");
var currentHostname = hive.Registry.GetLocalHostname();
var currentSecret = hive.Registry.GetLocalSecret();
var currentImage = currentService?.Spec.TaskTemplate.ContainerSpec.ImageWithoutSHA;
var currentCredentials = // Set blank properties for the change detection below.
new RegistryCredentials()
{
Registry = string.Empty,
Username = string.Empty,
Password = string.Empty
};
if (!string.IsNullOrEmpty(currentHostname))
{
context.WriteLine(AnsibleVerbosity.Trace, $"Reading existing registry credentials for [{currentHostname}].");
currentCredentials = hive.Registry.GetCredentials(currentHostname);
if (currentCredentials != null)
{
context.WriteLine(AnsibleVerbosity.Info, $"Registry credentials for [{currentHostname}] exist.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"Registry credentials for [{currentHostname}] do not exist.");
}
}
// Obtain the current registry TLS certificate (if any).
var currentCertificate = hive.Certificate.Get("neon-registry");
// Perform the operation.
switch (state)
{
case "absent":
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [hostname]");
if (!context.Arguments.TryGetValue <string>("hostname", out hostname))
{
throw new ArgumentException($"[hostname] module argument is required.");
}
if (currentService == null)
{
context.WriteLine(AnsibleVerbosity.Important, "[neon-registry] is not currently deployed.");
}
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Local registry will be removed when CHECK-MODE is disabled.");
return;
}
if (currentService == null)
{
return; // Nothing to do
}
context.Changed = true;
// Logout of the registry.
if (currentCredentials != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive out of the [{currentHostname}] registry.");
hive.Registry.Logout(currentHostname);
}
// Delete the [neon-registry] service and volume. Note that
// the volume should exist on all of the manager nodes.
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [neon-registry] service.");
manager.DockerCommand(RunOptions.None, "docker", "service", "rm", "neon-registry");
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [neon-registry] volumes.");
var volumeRemoveActions = new List <Action>();
var volumeRetryPolicy = new LinearRetryPolicy(typeof(TransientException), maxAttempts: 10, retryInterval: TimeSpan.FromSeconds(2));
foreach (var node in hive.Managers)
{
volumeRemoveActions.Add(
() =>
{
// $hack(jeff.lill):
//
// Docker service removal appears to be synchronous but the removal of the
// actual service task containers is not. We're going to detect this and
// throw a [TransientException] and then retry.
using (var clonedNode = node.Clone())
{
lock (context)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Removing [neon-registry] volume on [{clonedNode.Name}].");
}
volumeRetryPolicy.InvokeAsync(
async() =>
{
var response = clonedNode.DockerCommand(RunOptions.None, "docker", "volume", "rm", "neon-registry");
if (response.ExitCode != 0)
{
var message = $"Error removing [neon-registry] volume from [{clonedNode.Name}: {response.ErrorText}";
lock (syncLock)
{
context.WriteLine(AnsibleVerbosity.Info, message);
}
if (response.AllText.Contains("volume is in use"))
{
throw new TransientException(message);
}
}
else
{
lock (context)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Removed [neon-registry] volume on [{clonedNode.Name}].");
}
}
await Task.Delay(0);
}).Wait();
}
});
}
NeonHelper.WaitForParallel(volumeRemoveActions);
// Remove the traffic manager rule and certificate.
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [neon-registry] traffic manager rule.");
hive.PublicTraffic.RemoveRule("neon-registry");
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [neon-registry] traffic manager certificate.");
hive.Certificate.Remove("neon-registry");
// Remove any related Consul state.
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [neon-registry] Consul [hostname] and [secret].");
hive.Registry.SetLocalHostname(null);
hive.Registry.SetLocalSecret(null);
// Logout the hive from the registry.
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive out of the [{currentHostname}] registry.");
hive.Registry.Logout(currentHostname);
// Remove the hive DNS host entry.
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [{currentHostname}] registry DNS hosts entry.");
hive.Dns.Remove(hostname);
break;
case "present":
if (!hive.Definition.HiveFS.Enabled)
{
context.WriteErrorLine("The local registry service requires hive CephFS.");
return;
}
// Parse the [hostname], [certificate], [username] and [password] arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [hostname]");
if (!context.Arguments.TryGetValue <string>("hostname", out hostname))
{
throw new ArgumentException($"[hostname] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [certificate]");
if (!context.Arguments.TryGetValue <string>("certificate", out var certificatePem))
{
throw new ArgumentException($"[certificate] module argument is required.");
}
if (!TlsCertificate.TryParse(certificatePem, out var certificate))
{
throw new ArgumentException($"[certificate] is not a valid certificate.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [username]");
if (!context.Arguments.TryGetValue <string>("username", out var username))
{
throw new ArgumentException($"[username] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [password]");
if (!context.Arguments.TryGetValue <string>("password", out var password))
{
throw new ArgumentException($"[password] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [secret]");
if (!context.Arguments.TryGetValue <string>("secret", out var secret) || string.IsNullOrEmpty(secret))
{
throw new ArgumentException($"[secret] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [image]");
if (!context.Arguments.TryGetValue <string>("image", out var image))
{
image = HiveConst.NeonProdRegistry + "/neon-registry:latest";
}
// Detect service changes.
var hostnameChanged = hostname != currentCredentials?.Registry;
var usernameChanged = username != currentCredentials?.Username;
var passwordChanged = password != currentCredentials?.Password;
var secretChanged = secret != currentSecret;
var imageChanged = image != currentImage;
var certificateChanged = certificate?.CombinedPemNormalized != currentCertificate?.CombinedPemNormalized;
var updateRequired = hostnameChanged ||
usernameChanged ||
passwordChanged ||
secretChanged ||
imageChanged ||
certificateChanged;
if (hostnameChanged)
{
context.WriteLine(AnsibleVerbosity.Info, $"[hostname] changed from [{currentCredentials?.Registry}] --> [{hostname}]");
}
if (usernameChanged)
{
context.WriteLine(AnsibleVerbosity.Info, $"[username] changed from [{currentCredentials?.Username}] --> [{username}]");
}
if (usernameChanged)
{
context.WriteLine(AnsibleVerbosity.Info, $"[password] changed from [{currentCredentials?.Password}] --> [**REDACTED**]");
}
if (secretChanged)
{
context.WriteLine(AnsibleVerbosity.Info, $"[secret] changed from [{currentSecret}] --> [**REDACTED**]");
}
if (imageChanged)
{
context.WriteLine(AnsibleVerbosity.Info, $"[image] changed from [{currentImage}] --> [{image}]");
}
if (certificateChanged)
{
var currentCertRedacted = currentCertificate != null ? "**REDACTED**" : "**NONE**";
context.WriteLine(AnsibleVerbosity.Info, $"[certificate] changed from [{currentCertRedacted}] --> [**REDACTED**]");
}
// Handle CHECK-MODE.
if (context.CheckMode)
{
if (currentService == null)
{
context.WriteLine(AnsibleVerbosity.Important, $"Local registry will be deployed when CHECK-MODE is disabled.");
return;
}
if (updateRequired)
{
context.WriteLine(AnsibleVerbosity.Important, $"One or more of the arguments have changed so the registry will be updated when CHECK-MODE is disabled.");
return;
}
return;
}
// Create the hive DNS host entry we'll use to redirect traffic targeting the registry
// hostname to the hive managers. We need to do this because registry IP addresses
// are typically public, typically targeting the external firewall or load balancer
// interface.
//
// The problem is that hive nodes will generally be unable to connect to the
// local managers through the firewall/load balancer because most network routers
// block network traffic that originates from inside the hive, then leaves
// to hit the external router interface with the expectation of being routed
// back inside. I believe this is an anti-spoofing security measure.
var dnsRedirect = GetRegistryDnsEntry(hostname);
// Perform the operation.
if (currentService == null)
{
context.WriteLine(AnsibleVerbosity.Important, $"[neon-registry] service needs to be created.");
context.Changed = true;
// The registry service isn't running, so we'll do a full deployment.
context.WriteLine(AnsibleVerbosity.Trace, $"Setting certificate.");
hive.Certificate.Set("neon-registry", certificate);
context.WriteLine(AnsibleVerbosity.Trace, $"Updating Consul settings.");
hive.Registry.SetLocalHostname(hostname);
hive.Registry.SetLocalSecret(secret);
context.WriteLine(AnsibleVerbosity.Trace, $"Adding hive DNS host entry for [{hostname}].");
hive.Dns.Set(dnsRedirect, waitUntilPropagated: true);
context.WriteLine(AnsibleVerbosity.Trace, $"Writing traffic manager rule.");
hive.PublicTraffic.SetRule(GetRegistryTrafficManagerRule(hostname));
context.WriteLine(AnsibleVerbosity.Trace, $"Creating the [neon-registry] service.");
var createResponse = manager.DockerCommand(RunOptions.None,
"docker service create",
"--name", "neon-registry",
"--mode", "global",
"--constraint", "node.role==manager",
"--env", $"USERNAME={username}",
"--env", $"PASSWORD={password}",
"--env", $"SECRET={secret}",
"--env", $"LOG_LEVEL=info",
"--env", $"READ_ONLY=false",
"--mount", "type=volume,src=neon-registry,volume-driver=neon,dst=/var/lib/neon-registry",
"--network", "neon-public",
"--restart-delay", "10s",
image);
if (createResponse.ExitCode != 0)
{
context.WriteErrorLine($"[neon-registry] service create failed: {createResponse.ErrorText}");
return;
}
context.WriteLine(AnsibleVerbosity.Trace, $"Service created.");
context.WriteLine(AnsibleVerbosity.Trace, $"Wait for [neon-registry] service to stabilize (30s).");
Thread.Sleep(TimeSpan.FromSeconds(30));
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive into the [{hostname}] registry.");
hive.Registry.Login(hostname, username, password);
}
else if (updateRequired)
{
context.WriteLine(AnsibleVerbosity.Important, $"[neon-registry] service update is required.");
context.Changed = true;
// Update the service and related settings as required.
if (certificateChanged)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Updating certificate.");
hive.Certificate.Set("neon-registry", certificate);
}
if (hostnameChanged)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Updating traffic manager rule.");
hive.PublicTraffic.SetRule(GetRegistryTrafficManagerRule(hostname));
context.WriteLine(AnsibleVerbosity.Trace, $"Updating hive DNS host entry for [{hostname}] (60 seconds).");
hive.Dns.Set(dnsRedirect, waitUntilPropagated: true);
context.WriteLine(AnsibleVerbosity.Trace, $"Updating local hive hostname [{hostname}].");
hive.Registry.SetLocalHostname(hostname);
if (!string.IsNullOrEmpty(currentHostname))
{
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive out of the [{currentHostname}] registry.");
hive.Registry.Logout(currentHostname);
}
}
if (secretChanged)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Updating local hive secret.");
hive.Registry.SetLocalSecret(secret);
}
context.WriteLine(AnsibleVerbosity.Trace, $"Updating service.");
var updateResponse = manager.DockerCommand(RunOptions.None,
"docker service update",
"--env-add", $"USERNAME={username}",
"--env-add", $"PASSWORD={password}",
"--env-add", $"SECRET={secret}",
"--env-add", $"LOG_LEVEL=info",
"--env-add", $"READ_ONLY=false",
"--image", image,
"neon-registry");
if (updateResponse.ExitCode != 0)
{
context.WriteErrorLine($"[neon-registry] service update failed: {updateResponse.ErrorText}");
return;
}
context.WriteLine(AnsibleVerbosity.Trace, $"Service updated.");
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive into the [{hostname}] registry.");
hive.Registry.Login(hostname, username, password);
}
else
{
context.WriteLine(AnsibleVerbosity.Important, $"[neon-registry] service update is not required but we're logging all nodes into [{hostname}] to ensure hive consistency.");
hive.Registry.Login(hostname, username, password);
context.Changed = false;
}
break;
case "prune":
if (currentService == null)
{
context.WriteLine(AnsibleVerbosity.Important, "Registry service is not running.");
return;
}
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, "Registry will be pruned when CHECK-MODE is disabled.");
return;
}
context.Changed = true; // Always set this to TRUE for prune.
// We're going to upload a script to one of the managers that handles
// putting the [neon-registry] service into READ-ONLY mode, running
// the garbage collection container and then restoring [neon-registry]
// to READ/WRITE mode.
//
// The nice thing about this is that the operation will continue to
// completion on the manager node even if we lose the SSH connection.
var updateScript =
$@"#!/bin/bash
# Update [neon-registry] to READ-ONLY mode:
docker service update --env-rm READ_ONLY --env-add READ_ONLY=true neon-registry
# Prune the registry:
docker run \
--name neon-registry-prune \
--restart-condition=none \
--mount type=volume,src=neon-registry,volume-driver=neon,dst=/var/lib/neon-registry \
{HiveConst.NeonProdRegistry}/neon-registry garbage-collect
# Restore [neon-registry] to READ/WRITE mode:
docker service update --env-rm READ_ONLY --env-add READ_ONLY=false neon-registry
";
var bundle = new CommandBundle("./collect.sh");
bundle.AddFile("collect.sh", updateScript, isExecutable: true);
context.WriteLine(AnsibleVerbosity.Info, "Registry prune started.");
var pruneResponse = manager.SudoCommand(bundle, RunOptions.None);
if (pruneResponse.ExitCode != 0)
{
context.WriteErrorLine($"The prune operation failed. The registry may be running in READ-ONLY mode: {pruneResponse.ErrorText}");
return;
}
context.WriteLine(AnsibleVerbosity.Info, "Registry prune completed.");
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [prune].");
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
if (!context.Arguments.TryGetValue <string>("name", out var name))
{
throw new ArgumentException($"[name] module argument is required.");
}
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "deploy";
}
state = state.ToLowerInvariant();
var manager = hive.GetReachableManager();
var response = (CommandResponse)null;
switch (state)
{
case "deploy":
if (!context.Arguments.TryGetValue("stack", out var stackObject))
{
throw new ArgumentException($"[stack] module argument is required when [state=deploy].");
}
var stackJson = NeonHelper.JsonSerialize(stackObject);
var stackYaml = NeonHelper.JsonToYaml(stackJson);
var bundle = new CommandBundle($"docker stack deploy --compose-file ./compose.yaml {name}");
bundle.AddFile("compose.yaml", stackYaml);
response = manager.SudoCommand(bundle, RunOptions.None);
if (response.ExitCode != 0)
{
context.WriteErrorLine(response.ErrorText);
}
context.Changed = true;
break;
case "remove":
response = manager.SudoCommand("docker stack rm", RunOptions.None, name);
if (response.ExitCode != 0)
{
context.WriteErrorLine(response.ErrorText);
}
context.Changed = true;
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [rollback].");
}
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
var split = commandLine.Split("--");
var leftCommandLine = split.Left;
var rightCommandLine = split.Right;
// Basic initialization.
if (leftCommandLine.HasHelpOption)
{
Console.WriteLine(usage);
Program.Exit(0);
}
Program.ConnectHive();
var hive = HiveHelper.Hive;
// Process the nodes.
var nodeDefinitions = new List <NodeDefinition>();
var nodeOption = leftCommandLine.GetOption("--node", null);
if (!string.IsNullOrWhiteSpace(nodeOption))
{
if (nodeOption == "+")
{
foreach (var manager in hive.Definition.SortedManagers)
{
nodeDefinitions.Add(manager);
}
foreach (var worker in hive.Definition.SortedWorkers)
{
nodeDefinitions.Add(worker);
}
foreach (var pet in hive.Definition.SortedPets)
{
nodeDefinitions.Add(pet);
}
}
else
{
foreach (var name in nodeOption.Split(',', StringSplitOptions.RemoveEmptyEntries))
{
var trimmedName = name.Trim();
NodeDefinition node;
if (!hive.Definition.NodeDefinitions.TryGetValue(trimmedName, out node))
{
Console.Error.WriteLine($"*** ERROR: Node [{trimmedName}] is not present in the hive.");
Program.Exit(1);
}
nodeDefinitions.Add(node);
}
}
}
var groupName = leftCommandLine.GetOption("--group");
if (!string.IsNullOrEmpty(groupName))
{
var nodeGroups = hive.Definition.GetHostGroups();
if (!nodeGroups.TryGetValue(groupName, out var group))
{
Console.Error.WriteLine($"*** ERROR: Node group [{groupName}] is not defined for the hive.");
Program.Exit(1);
}
// Add the group nodes to the node definitions if they aren't
// already present.
foreach (var node in group)
{
if (nodeDefinitions.Count(n => n.Name.Equals(node.Name, StringComparison.InvariantCultureIgnoreCase)) == 0)
{
nodeDefinitions.Add(node);
}
}
}
if (nodeDefinitions.Count == 0)
{
// Default to a healthy manager.
nodeDefinitions.Add(hive.GetReachableManager().Metadata);
}
// Create the command bundle by appending the right command.
if (rightCommandLine == null)
{
Console.Error.WriteLine($"*** ERROR: [exec] command expectes: [-- COMMAND...]");
Program.Exit(1);
}
string command = rightCommandLine.Items.First();
var args = rightCommandLine.Items.Skip(1).ToArray();
var bundle = new CommandBundle(command, args.ToArray());
// Append any script, text, or data files to the bundle.
foreach (var scriptPath in leftCommandLine.GetOptionValues("--script"))
{
if (!File.Exists(scriptPath))
{
Console.Error.WriteLine($"*** ERROR: Script [{scriptPath}] does not exist.");
Program.Exit(1);
}
bundle.AddFile(Path.GetFileName(scriptPath), File.ReadAllText(scriptPath), isExecutable: true);
}
foreach (var textPath in leftCommandLine.GetOptionValues("--text"))
{
if (!File.Exists(textPath))
{
Console.Error.WriteLine($"*** ERROR: Text file [{textPath}] does not exist.");
Program.Exit(1);
}
bundle.AddFile(Path.GetFileName(textPath), File.ReadAllText(textPath));
}
foreach (var dataPath in leftCommandLine.GetOptionValues("--data"))
{
if (!File.Exists(dataPath))
{
Console.Error.WriteLine($"*** ERROR: Data file [{dataPath}] does not exist.");
Program.Exit(1);
}
bundle.AddFile(Path.GetFileName(dataPath), File.ReadAllBytes(dataPath));
}
// Perform the operation.
if (nodeDefinitions.Count == 1)
{
// Run the command on a single node and return the output and exit code.
var node = hive.GetNode(nodeDefinitions.First().Name);
var response = node.SudoCommand(bundle);
Console.WriteLine(response.OutputText);
Program.Exit(response.ExitCode);
}
else
{
// Run the command on multiple nodes and return an overall exit code.
var controller = new SetupController <NodeDefinition>(Program.SafeCommandLine, hive.Nodes.Where(n => nodeDefinitions.Exists(nd => nd.Name == n.Name)))
{
ShowStatus = !Program.Quiet,
MaxParallel = Program.MaxParallel
};
controller.SetDefaultRunOptions(RunOptions.FaultOnError);
controller.AddWaitUntilOnlineStep();
controller.AddStep($"run: {bundle.Command}",
(node, stepDelay) =>
{
Thread.Sleep(stepDelay);
node.Status = "running";
node.SudoCommand(bundle, RunOptions.FaultOnError | RunOptions.LogOutput);
if (Program.WaitSeconds > 0)
{
node.Status = $"stabilize ({Program.WaitSeconds}s)";
Thread.Sleep(TimeSpan.FromSeconds(Program.WaitSeconds));
}
});
if (!controller.Run())
{
Console.Error.WriteLine("*** ERROR: [exec] on one or more nodes failed.");
Program.Exit(1);
}
}
}
