public Credentials prompt(Host bookmark, string title, string reason, LoginOptions options)
{
Credentials credentials = new VaultCredentials();
credentials.setSaved(options.save());
AsyncDelegate d = delegate
{
View = ObjectFactory.GetInstance <IPasswordPromptView>();
View.Title = title;
View.Reason = reason;
View.OkButtonText = LocaleFactory.localizedString("Continue", "Credentials");
View.IconView = IconCache.Instance.IconForName(options.icon(), 64);
View.SavePassword = options.save();
View.ValidateInput += ValidateInputEventHandler;
if (DialogResult.Cancel == View.ShowDialog(_browser.View))
{
throw new LoginCanceledException();
}
credentials.setPassword(View.InputText);
credentials.setSaved(View.SavePassword);
};
_browser.Invoke(d);
return(credentials);
}
/// <summary>
/// Implements the service as a <see cref="Task"/>.
/// </summary>
/// <returns>The <see cref="Task"/>.</returns>
private static async Task RunAsync()
{
// Load the settings.
//
// Initialize the proxy manager settings to their default values
// if they don't already exist.
if (!await consul.KV.Exists(hivemqMaintainSecondsKey))
{
log.LogInfo($"Persisting setting [{hivemqMaintainSecondsKey}=60.0]");
await consul.KV.PutDouble(hivemqMaintainSecondsKey, 60);
}
if (!await consul.KV.Exists(logPurgeSecondsKey))
{
log.LogInfo($"Persisting setting [{logPurgeSecondsKey}=300.0]");
await consul.KV.PutDouble(logPurgeSecondsKey, 300);
}
if (!await consul.KV.Exists(managerTopologySecondsKey))
{
log.LogInfo($"Persisting setting [{managerTopologySecondsKey}=300.0]");
await consul.KV.PutDouble(managerTopologySecondsKey, 1800);
}
if (!await consul.KV.Exists(proxyUpdateSecondsKey))
{
log.LogInfo($"Persisting setting [{proxyUpdateSecondsKey}=60.0]");
await consul.KV.PutDouble(proxyUpdateSecondsKey, 60);
}
if (!await consul.KV.Exists(secretPurgeSecondsKey))
{
log.LogInfo($"Persisting setting [{secretPurgeSecondsKey}=300.0]");
await consul.KV.PutDouble(secretPurgeSecondsKey, 300);
}
if (!await consul.KV.Exists(swarmPollSecondsKey))
{
log.LogInfo($"Persisting setting [{swarmPollSecondsKey}=30.0]");
await consul.KV.PutDouble(swarmPollSecondsKey, 30.0);
}
if (!await consul.KV.Exists(vaultUnsealSecondsKey))
{
log.LogInfo($"Persisting setting [{vaultUnsealSecondsKey}=30.0]");
await consul.KV.PutDouble(vaultUnsealSecondsKey, 30.0);
}
hivemqMantainInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(hivemqMaintainSecondsKey));
logPurgerInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(logPurgeSecondsKey));
managerTopologyInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(managerTopologySecondsKey));
proxyUpdateInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(proxyUpdateSecondsKey));
secretPurgeInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(secretPurgeSecondsKey));
swarmPollInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(swarmPollSecondsKey));
vaultUnsealInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(vaultUnsealSecondsKey));
log.LogInfo(() => $"Using setting [{hivemqMaintainSecondsKey}={hivemqMantainInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{logPurgeSecondsKey}={logPurgerInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{managerTopologySecondsKey}={managerTopologyInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{proxyUpdateSecondsKey}={proxyUpdateInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{secretPurgeSecondsKey}={secretPurgeInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{swarmPollSecondsKey}={swarmPollInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{vaultUnsealSecondsKey}={vaultUnsealInterval.TotalSeconds}]");
// Parse the Vault credentials from the [neon-hive-manager-vaultkeys]
// secret, if it exists.
var vaultCredentialsJson = HiveHelper.GetSecret("neon-hive-manager-vaultkeys");
if (string.IsNullOrWhiteSpace(vaultCredentialsJson))
{
log.LogInfo(() => "Vault AUTO-UNSEAL is DISABLED because [neon-hive-manager-vaultkeys] Docker secret is not specified.");
}
else
{
try
{
vaultCredentials = NeonHelper.JsonDeserialize <VaultCredentials>(vaultCredentialsJson);
log.LogInfo(() => "Vault AUTO-UNSEAL is ENABLED.");
}
catch (Exception e)
{
log.LogError("Vault AUTO-UNSEAL is DISABLED because the [neon-hive-manager-vaultkeys] Docker secret could not be parsed.", e);
}
}
// We're going to need this later.
vaultUris = await GetVaultUrisAsync();
// Launch the sub-tasks. These will run until the service is terminated.
var tasks = new List <Task>();
// Start a task that handles HiveMQ related activities like ensuring that
// the [sysadmin] account has full permissions for all virtual hosts.
tasks.Add(HiveMQMaintainerAsync());
// Start a task that checks for Elasticsearch [logstash] and [metricbeat] indexes
// that are older than the number of retention days.
tasks.Add(LogPurgerAsync());
// Start a task that periodically checks for changes to the set of hive managers
// (e.g. if a manager is added or removed). This task will cause the service to exit
// so it can be restarted automatically by Docker to respond to the change.
tasks.Add(ManagerWatcherAsync());
// Start a task that checks for old [neon-secret-retriever-*] service instances
// as well as old persisted secrets and removes them.
tasks.Add(SecretPurgerAsync());
// Start a task that polls current hive state to update the hive definition in Consul, etc.
tasks.Add(SwarmPollerAsync());
// Start a task that periodically notifies the [neon-proxy-manager] service
// that it should proactively rebuild the proxy configurations.
tasks.Add(ProxyUpdaterAsync());
// We need to start a vault poller for the Vault instance running on each manager
// node. We're going to construct the direct Vault URIs by querying Docker for
// the current hive nodes and looking for the managers.
foreach (var uri in vaultUris)
{
tasks.Add(VaultUnsealerAsync(uri));
}
// Wait for all tasks to exit cleanly for a normal shutdown.
await NeonHelper.WaitAllAsync(tasks);
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
// Split the command line on "--".
var split = commandLine.Split(SplitItem);
var leftCommandLine = split.Left;
var rightCommandLine = split.Right;
// Basic initialization.
if (leftCommandLine.HasHelpOption)
{
Console.WriteLine(usage);
Program.Exit(0);
}
// Initialize the hive.
var hiveLogin = Program.ConnectHive();
hive = new HiveProxy(hiveLogin);
vaultCredentials = hiveLogin.VaultCredentials;
if (rightCommandLine == null)
{
Console.WriteLine("*** ERROR: The [--] command separator is required.");
Console.WriteLine();
Console.WriteLine(usage);
Program.Exit(1);
}
// Determine which node we're going to target.
SshProxy <NodeDefinition> node;
string nodeName = leftCommandLine.GetOption("--node", null);
CommandBundle bundle;
CommandResponse response;
bool failed = false;
if (!string.IsNullOrEmpty(nodeName))
{
node = hive.GetNode(nodeName);
}
else
{
node = hive.GetReachableManager();
}
var command = rightCommandLine.Arguments.FirstOrDefault();;
switch (command)
{
case "init":
case "rekey":
case "server":
case "ssh":
Console.Error.WriteLine($"*** ERROR: [neon vault {command}] is not supported.");
Program.Exit(1);
break;
case "seal":
// Just run the command on the target node if one was specified.
if (nodeName != null)
{
ExecuteOnNode(node, rightCommandLine);
return;
}
// We need to seal the Vault instance on every manager node unless a
// specific node was requsted via [--node].
//
// Note also that it's not possible to seal a node that's in standby
// mode so we'll restart the Vault container instead.
Console.WriteLine();
if (!string.IsNullOrEmpty(nodeName))
{
response = node.SudoCommand($"vault-direct status");
if (response.ExitCode != 0)
{
Console.WriteLine($"[{node.Name}] is already sealed");
}
else
{
var vaultStatus = new VaultStatus(response.OutputText);
if (vaultStatus.HAMode == "standby")
{
Console.WriteLine($"[{node.Name}] restarting to seal standby node...");
response = node.SudoCommand($"systemctl restart vault");
if (response.ExitCode == 0)
{
Console.WriteLine($"[{node.Name}] sealed");
}
else
{
Console.WriteLine($"[{node.Name}] restart failed");
}
}
else
{
Console.WriteLine($"[{node.Name}] sealing...");
response = node.SudoCommand($"export VAULT_TOKEN={vaultCredentials.RootToken} && vault-direct operator seal", RunOptions.Redact);
if (response.ExitCode == 0)
{
Console.WriteLine($"[{node.Name}] sealed");
}
else
{
Console.WriteLine($"[{node.Name}] seal failed");
}
}
}
failed = response.ExitCode != 0;
}
else
{
foreach (var manager in hive.Managers)
{
try
{
response = manager.SudoCommand($"vault-direct status");
}
catch (SshOperationTimeoutException)
{
Console.WriteLine($"[{manager.Name}] ** unavailable **");
continue;
}
var vaultStatus = new VaultStatus(response.OutputText);
if (response.ExitCode != 0)
{
Console.WriteLine($"[{manager.Name}] is already sealed");
}
else
{
response = manager.SudoCommand($"vault-direct operator seal");
if (vaultStatus.HAMode == "standby")
{
Console.WriteLine($"[{manager.Name}] restarting to seal standby node...");
response = manager.SudoCommand($"systemctl restart vault");
if (response.ExitCode == 0)
{
Console.WriteLine($"[{manager.Name}] restart/seal [standby]");
}
else
{
Console.WriteLine($"[{manager.Name}] restart/seal failed [standby]");
}
}
else
{
Console.WriteLine($"[{manager.Name}] sealing...");
response = manager.SudoCommand($"export VAULT_TOKEN={vaultCredentials.RootToken} && vault-direct operator seal", RunOptions.Redact);
if (response.ExitCode == 0)
{
Console.WriteLine($"[{manager.Name}] sealed");
}
else
{
failed = true;
Console.WriteLine($"[{manager.Name}] seal failed");
}
}
}
}
if (!failed)
{
// Disable auto unseal until the operator explicitly unseals Vault again.
hive.Consul.Client.KV.PutBool($"{HiveConst.GlobalKey}/{HiveGlobals.UserDisableAutoUnseal}", true).Wait();
}
Program.Exit(failed ? 1 : 0);
}
break;
case "status":
// Just run the command on the target node if one was specified.
if (nodeName != null)
{
ExecuteOnNode(node, rightCommandLine);
return;
}
// We need to obtain the status from the Vault instance on every manager node unless a
// specific node was requsted via [--node].
Console.WriteLine();
if (!string.IsNullOrEmpty(nodeName))
{
response = node.SudoCommand("vault-direct status");
Console.WriteLine(response.AllText);
Program.Exit(response.ExitCode);
}
else
{
var allSealed = true;
foreach (var manager in hive.Managers)
{
try
{
response = manager.SudoCommand("vault-direct status");
}
catch (SshOperationTimeoutException)
{
Console.WriteLine($"[{manager.Name}] ** unavailable **");
continue;
}
var vaultStatus = new VaultStatus(response.OutputText);
if (response.ExitCode == 0)
{
allSealed = false;
var status = vaultStatus.HAMode;
if (status == "active")
{
status += " <-- LEADER";
}
Console.WriteLine($"[{manager.Name}] unsealed {status}");
}
else if (response.ExitCode == 2)
{
Console.WriteLine($"[{manager.Name}] sealed");
}
else
{
failed = true;
Console.WriteLine($"[{manager.Name}] error getting status");
}
}
if (allSealed)
{
Program.Exit(2);
}
else
{
Program.Exit(failed ? 1 : 0);
}
}
break;
case "unseal":
// Just run the command on the target node if one was specified.
if (nodeName != null)
{
ExecuteOnNode(node, rightCommandLine);
return;
}
// We need to unseal the Vault instance on every manager node unless a
// specific node was requsted via [--node].
Console.WriteLine();
if (!string.IsNullOrEmpty(nodeName))
{
// Verify that the instance isn't already unsealed.
response = node.SudoCommand($"vault-direct status");
if (response.ExitCode == 2)
{
Console.WriteLine($"[{node.Name}] unsealing...");
}
else if (response.ExitCode == 0)
{
Console.WriteLine($"[{node.Name}] is already unsealed");
break;
}
else
{
Console.WriteLine($"[{node.Name}] unseal failed");
Program.Exit(response.ExitCode);
}
// Note that we're passing the [-reset] option to ensure that
// any keys from previous attempts have been cleared.
node.SudoCommand($"vault-direct operator unseal -reset");
foreach (var key in vaultCredentials.UnsealKeys)
{
response = node.SudoCommand($"vault-direct operator unseal {key}", RunOptions.None);
if (response.ExitCode != 0)
{
Console.WriteLine($"[{node.Name}] unseal failed");
Program.Exit(1);
}
}
Console.WriteLine($"[{node.Name}] unsealed");
}
else
{
var commandFailed = false;
foreach (var manager in hive.Managers)
{
// Verify that the instance isn't already unsealed.
try
{
response = manager.SudoCommand($"vault-direct status");
}
catch (SshOperationTimeoutException)
{
Console.WriteLine($"[{manager.Name}] ** unavailable **");
continue;
}
if (response.ExitCode == 2)
{
Console.WriteLine($"[{manager.Name}] unsealing...");
}
else if (response.ExitCode == 0)
{
Console.WriteLine($"[{manager.Name}] is already unsealed");
continue;
}
else
{
Console.WriteLine($"[{manager.Name}] unseal failed");
continue;
}
// Note that we're passing the [-reset] option to ensure that
// any keys from previous attempts have been cleared.
manager.SudoCommand($"vault-direct operator unseal -reset");
foreach (var key in vaultCredentials.UnsealKeys)
{
response = manager.SudoCommand($"vault-direct operator unseal {key}", RunOptions.None);
if (response.ExitCode != 0)
{
failed = true;
commandFailed = true;
Console.WriteLine($"[{manager.Name}] unseal failed");
}
}
if (!failed)
{
Console.WriteLine($"[{manager.Name}] unsealed");
}
}
if (!commandFailed)
{
// Reenable auto unseal.
hive.Consul.Client.KV.PutBool($"{HiveConst.GlobalKey}/{HiveGlobals.UserDisableAutoUnseal}", false).Wait();
}
Program.Exit(commandFailed ? 1 : 0);
}
break;
case "write":
{
// We need handle any [key=@file] arguments specially by including them
// in a command bundle as data files.
var files = new List <CommandFile>();
var commandString = rightCommandLine.ToString();
foreach (var dataArg in rightCommandLine.Arguments.Skip(2))
{
var fields = dataArg.Split(new char[] { '=' }, 2);
if (fields.Length == 2 && fields[1].StartsWith("@"))
{
var fileName = fields[1].Substring(1);
var localFileName = $"{files.Count}.data";
files.Add(
new CommandFile()
{
Path = localFileName,
Data = File.ReadAllBytes(fileName)
});
commandString = commandString.Replace($"@{fileName}", $"@{localFileName}");
}
}
bundle = new CommandBundle($"export VAULT_TOKEN={vaultCredentials.RootToken} && {remoteVaultPath} {commandString}");
foreach (var file in files)
{
bundle.Add(file);
}
response = node.SudoCommand(bundle, RunOptions.IgnoreRemotePath | RunOptions.Redact);
Console.WriteLine(response.AllText);
Program.Exit(response.ExitCode);
}
break;
case "policy-write":
// The last command line item is either:
//
// * A "-", indicating that the content should come from standard input.
// * A file name prefixed by "@"
// * A string holding JSON or HCL
if (rightCommandLine.Items.Length < 2)
{
response = node.SudoCommand($"export VAULT_TOKEN={vaultCredentials.RootToken} && {remoteVaultPath} {rightCommandLine}", RunOptions.IgnoreRemotePath | RunOptions.Redact);
Console.WriteLine(response.AllText);
Program.Exit(response.ExitCode);
}
var lastItem = rightCommandLine.Items.Last();
var policyText = string.Empty;
if (lastItem == "-")
{
policyText = NeonHelper.ReadStandardInputText();
}
else if (lastItem.StartsWith("@"))
{
policyText = File.ReadAllText(lastItem.Substring(1), Encoding.UTF8);
}
else
{
policyText = lastItem;
}
// We're going to upload a text file holding the policy text and
// then run a script piping the policy file into the Vault command passed,
// with the last item replaced by a "-".
bundle = new CommandBundle("./set-vault-policy.sh.sh");
var sbScript = new StringBuilder();
sbScript.AppendLine($"export VAULT_TOKEN={vaultCredentials.RootToken}");
sbScript.Append($"cat policy | {remoteVaultPath}");
for (int i = 0; i < rightCommandLine.Items.Length - 1; i++)
{
sbScript.Append(' ');
sbScript.Append(rightCommandLine.Items[i]);
}
sbScript.AppendLine(" -");
bundle.AddFile("set-vault-policy.sh", sbScript.ToString(), isExecutable: true);
bundle.AddFile("policy", policyText);
response = node.SudoCommand(bundle, RunOptions.IgnoreRemotePath | RunOptions.Redact);
Console.WriteLine(response.AllText);
Program.Exit(response.ExitCode);
break;
default:
ExecuteOnNode(node, rightCommandLine);
break;
}
}
