/// <summary>
/// Returns a Couchbase bucket connection using specified settings and a Docker secret.
/// </summary>
/// <param name="settings">The Couchbase settings.</param>
/// <param name="secretName">The Docker secret name.</param>
/// <returns>The connected <see cref="IBucket"/>.</returns>
public static IBucket ConnectBucket(this CouchbaseSettings settings, string secretName)
{
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(secretName));
var credentials = NeonHelper.JsonDeserialize <Credentials>(HiveHelper.GetSecret(secretName));
return(global::Couchbase.CouchbaseExtensions.OpenBucket(settings, credentials));
}
/// <summary>
/// Returns a RabbitMQ cluster connection using specified settings and credentials
/// loaded from a Docker secret. This works only for Docker services where the
/// Docker secret was mounted into the service containers.
/// </summary>
/// <param name="settings">The Couchbase settings.</param>
/// <param name="secretName">The local name of the Docker secret holding the credentials.</param>
/// <param name="dispatchConsumersAsync">Optionally enables <c>async</c> message consumers. This defaults to <c>false</c>.</param>
/// <returns>The RabbitMQ <see cref="IConnection"/>.</returns>
/// <remarks>
/// The credentials must be formatted as JSON as serialized by the <see cref="Credentials"/>
/// class.
/// </remarks>
public static IConnection ConnectUsingSecret(this HiveMQSettings settings, string secretName, bool dispatchConsumersAsync = false)
{
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(settings.VirtualHost));
Covenant.Requires <ArgumentNullException>(settings.AmqpHosts != null && settings.AmqpHosts.Count > 0);
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(secretName));
var credentials = NeonHelper.JsonDeserialize <Credentials>(HiveHelper.GetSecret(secretName), dispatchConsumersAsync);
return(new RabbitMQConnection(settings.ConnectRabbitMQ(credentials)));
}
/// <summary>
/// Application entry point.
/// </summary>
/// <param name="args">Command line arguments.</param>
public static async Task Main(string[] args)
{
LogManager.Default.SetLogLevel(Environment.GetEnvironmentVariable("LOG_LEVEL"));
log = LogManager.Default.GetLogger(typeof(Program));
// Create process terminator to handle termination signals.
terminator = new ProcessTerminator(log);
terminator.AddHandler(
() =>
{
// Cancel any operations in progress.
terminator.CancellationTokenSource.Cancel();
});
// Read the environment variables.
// $hack(jeff.lill:
//
// We're going to scan the Consul configuration key to determine whether this
// instance is managing the public or private proxy (or bridges) so we'll
// be completely compatible with existing deployments.
//
// In theory, we could have passed a new environment variable but that's not
// worth the trouble.
configKey = Environment.GetEnvironmentVariable("CONFIG_KEY");
if (string.IsNullOrEmpty(configKey))
{
log.LogError("[CONFIG_KEY] environment variable is required.");
Program.Exit(1, immediate: true);
}
isPublic = configKey.Contains("/public/");
var proxyName = isPublic ? "public" : "private";
serviceName = $"neon-proxy-{proxyName}:{GitVersion}";
log.LogInfo(() => $"Starting [{serviceName}]");
configHashKey = Environment.GetEnvironmentVariable("CONFIG_HASH_KEY");
if (string.IsNullOrEmpty(configHashKey))
{
log.LogError("[CONFIG_HASH_KEY] environment variable is required.");
Program.Exit(1, immediate: true);
}
vaultCredentialsName = Environment.GetEnvironmentVariable("VAULT_CREDENTIALS");
if (string.IsNullOrEmpty(vaultCredentialsName))
{
log.LogWarn("HTTPS routes are not supported because VAULT_CREDENTIALS is not specified or blank.");
}
var warnSeconds = Environment.GetEnvironmentVariable("WARN_SECONDS");
if (string.IsNullOrEmpty(warnSeconds) || !double.TryParse(warnSeconds, out var warnSecondsValue))
{
warnInterval = TimeSpan.FromSeconds(300);
}
else
{
warnInterval = TimeSpan.FromSeconds(warnSecondsValue);
}
var startSeconds = Environment.GetEnvironmentVariable("START_SECONDS");
if (string.IsNullOrEmpty(startSeconds) || !double.TryParse(startSeconds, out var startSecondsValue))
{
startDelay = TimeSpan.FromSeconds(10);
}
else
{
startDelay = TimeSpan.FromSeconds(startSecondsValue);
}
var maxHAProxyCountString = Environment.GetEnvironmentVariable("MAX_HAPROXY_COUNT");
if (!int.TryParse(maxHAProxyCountString, out maxHAProxyCount))
{
maxHAProxyCount = 10;
}
if (maxHAProxyCount < 0)
{
maxHAProxyCount = 0;
}
debugMode = "true".Equals(Environment.GetEnvironmentVariable("DEBUG"), StringComparison.InvariantCultureIgnoreCase);
log.LogInfo(() => $"LOG_LEVEL={LogManager.Default.LogLevel.ToString().ToUpper()}");
log.LogInfo(() => $"CONFIG_KEY={configKey}");
log.LogInfo(() => $"CONFIG_HASH_KEY={configHashKey}");
log.LogInfo(() => $"VAULT_CREDENTIALS={vaultCredentialsName}");
log.LogInfo(() => $"WARN_SECONDS={warnInterval}");
log.LogInfo(() => $"START_SECONDS={startDelay}");
log.LogInfo(() => $"MAX_HAPROXY_COUNT={maxHAProxyCount}");
log.LogInfo(() => $"DEBUG={debugMode}");
// Ensure that the required directories exist.
Directory.CreateDirectory(tmpfsFolder);
Directory.CreateDirectory(configFolder);
Directory.CreateDirectory(configUpdateFolder);
// Establish the hive connections.
if (NeonHelper.IsDevWorkstation)
{
throw new NotImplementedException("This service works only within a Linux container with HAProxy installed.");
//var vaultCredentialsSecret = "neon-proxy-manager-credentials";
//Environment.SetEnvironmentVariable("VAULT_CREDENTIALS", vaultCredentialsSecret);
//hive = HiveHelper.OpenHiveRemote(new DebugSecrets().VaultAppRole(vaultCredentialsSecret, $"neon-proxy-{proxyName}"));
}
else
{
hive = HiveHelper.OpenHive();
}
try
{
// Log into Vault using the Vault credentials persisted as a Docker
// secret, if one was specified. We won't open Vault otherwise.
if (!string.IsNullOrEmpty(vaultCredentialsName))
{
var vaultSecret = HiveHelper.GetSecret(vaultCredentialsName);
if (string.IsNullOrEmpty(vaultSecret))
{
log.LogCritical($"Cannot read Docker secret [{vaultCredentialsName}].");
Program.Exit(1, immediate: true);
}
var vaultCredentials = HiveCredentials.ParseJson(vaultSecret);
if (vaultCredentials == null)
{
log.LogCritical($"Cannot parse Docker secret [{vaultCredentialsName}].");
Program.Exit(1, immediate: true);
}
log.LogInfo(() => $"Connecting: Vault");
vault = HiveHelper.OpenVault(vaultCredentials);
}
else
{
vault = null;
// $hack(jeff.lill):
//
// This is a bit of backwards compatible hack. Instances started without the
// VAULT_CREDENTIALS environment variable are assumed to be proxy bridges.
isBridge = true;
}
// Open Consul and then start the service tasks.
log.LogInfo(() => $"Connecting: Consul");
using (consul = HiveHelper.OpenConsul())
{
log.LogInfo(() => $"Connecting: {HiveMQChannels.ProxyNotify} channel");
// Verify that the required Consul keys exist or loop to wait until they
// are created. This will allow the service wait for pending hive setup
// operations to be completed.
while (!await consul.KV.Exists(configKey))
{
log.LogWarn(() => $"Waiting for [{configKey}] key to be present in Consul.");
await Task.Delay(TimeSpan.FromSeconds(5));
}
while (!await consul.KV.Exists(configHashKey))
{
log.LogWarn(() => $"Waiting for [{configHashKey}] key to be present in Consul.");
await Task.Delay(TimeSpan.FromSeconds(5));
}
// Crank up the service tasks.
await NeonHelper.WaitAllAsync(
ErrorPollerAsync(),
HAProxShim());
}
}
catch (Exception e)
{
log.LogCritical(e);
Program.Exit(1);
return;
}
finally
{
HiveHelper.CloseHive();
terminator.ReadyToExit();
}
Program.Exit(0);
return;
}
/// <summary>
/// Implements the service as a <see cref="Task"/>.
/// </summary>
/// <returns>The <see cref="Task"/>.</returns>
private static async Task RunAsync()
{
// Load the settings.
//
// Initialize the proxy manager settings to their default values
// if they don't already exist.
if (!await consul.KV.Exists(hivemqMaintainSecondsKey))
{
log.LogInfo($"Persisting setting [{hivemqMaintainSecondsKey}=60.0]");
await consul.KV.PutDouble(hivemqMaintainSecondsKey, 60);
}
if (!await consul.KV.Exists(logPurgeSecondsKey))
{
log.LogInfo($"Persisting setting [{logPurgeSecondsKey}=300.0]");
await consul.KV.PutDouble(logPurgeSecondsKey, 300);
}
if (!await consul.KV.Exists(managerTopologySecondsKey))
{
log.LogInfo($"Persisting setting [{managerTopologySecondsKey}=300.0]");
await consul.KV.PutDouble(managerTopologySecondsKey, 1800);
}
if (!await consul.KV.Exists(proxyUpdateSecondsKey))
{
log.LogInfo($"Persisting setting [{proxyUpdateSecondsKey}=60.0]");
await consul.KV.PutDouble(proxyUpdateSecondsKey, 60);
}
if (!await consul.KV.Exists(secretPurgeSecondsKey))
{
log.LogInfo($"Persisting setting [{secretPurgeSecondsKey}=300.0]");
await consul.KV.PutDouble(secretPurgeSecondsKey, 300);
}
if (!await consul.KV.Exists(swarmPollSecondsKey))
{
log.LogInfo($"Persisting setting [{swarmPollSecondsKey}=30.0]");
await consul.KV.PutDouble(swarmPollSecondsKey, 30.0);
}
if (!await consul.KV.Exists(vaultUnsealSecondsKey))
{
log.LogInfo($"Persisting setting [{vaultUnsealSecondsKey}=30.0]");
await consul.KV.PutDouble(vaultUnsealSecondsKey, 30.0);
}
hivemqMantainInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(hivemqMaintainSecondsKey));
logPurgerInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(logPurgeSecondsKey));
managerTopologyInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(managerTopologySecondsKey));
proxyUpdateInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(proxyUpdateSecondsKey));
secretPurgeInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(secretPurgeSecondsKey));
swarmPollInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(swarmPollSecondsKey));
vaultUnsealInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(vaultUnsealSecondsKey));
log.LogInfo(() => $"Using setting [{hivemqMaintainSecondsKey}={hivemqMantainInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{logPurgeSecondsKey}={logPurgerInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{managerTopologySecondsKey}={managerTopologyInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{proxyUpdateSecondsKey}={proxyUpdateInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{secretPurgeSecondsKey}={secretPurgeInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{swarmPollSecondsKey}={swarmPollInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{vaultUnsealSecondsKey}={vaultUnsealInterval.TotalSeconds}]");
// Parse the Vault credentials from the [neon-hive-manager-vaultkeys]
// secret, if it exists.
var vaultCredentialsJson = HiveHelper.GetSecret("neon-hive-manager-vaultkeys");
if (string.IsNullOrWhiteSpace(vaultCredentialsJson))
{
log.LogInfo(() => "Vault AUTO-UNSEAL is DISABLED because [neon-hive-manager-vaultkeys] Docker secret is not specified.");
}
else
{
try
{
vaultCredentials = NeonHelper.JsonDeserialize <VaultCredentials>(vaultCredentialsJson);
log.LogInfo(() => "Vault AUTO-UNSEAL is ENABLED.");
}
catch (Exception e)
{
log.LogError("Vault AUTO-UNSEAL is DISABLED because the [neon-hive-manager-vaultkeys] Docker secret could not be parsed.", e);
}
}
// We're going to need this later.
vaultUris = await GetVaultUrisAsync();
// Launch the sub-tasks. These will run until the service is terminated.
var tasks = new List <Task>();
// Start a task that handles HiveMQ related activities like ensuring that
// the [sysadmin] account has full permissions for all virtual hosts.
tasks.Add(HiveMQMaintainerAsync());
// Start a task that checks for Elasticsearch [logstash] and [metricbeat] indexes
// that are older than the number of retention days.
tasks.Add(LogPurgerAsync());
// Start a task that periodically checks for changes to the set of hive managers
// (e.g. if a manager is added or removed). This task will cause the service to exit
// so it can be restarted automatically by Docker to respond to the change.
tasks.Add(ManagerWatcherAsync());
// Start a task that checks for old [neon-secret-retriever-*] service instances
// as well as old persisted secrets and removes them.
tasks.Add(SecretPurgerAsync());
// Start a task that polls current hive state to update the hive definition in Consul, etc.
tasks.Add(SwarmPollerAsync());
// Start a task that periodically notifies the [neon-proxy-manager] service
// that it should proactively rebuild the proxy configurations.
tasks.Add(ProxyUpdaterAsync());
// We need to start a vault poller for the Vault instance running on each manager
// node. We're going to construct the direct Vault URIs by querying Docker for
// the current hive nodes and looking for the managers.
foreach (var uri in vaultUris)
{
tasks.Add(VaultUnsealerAsync(uri));
}
// Wait for all tasks to exit cleanly for a normal shutdown.
await NeonHelper.WaitAllAsync(tasks);
}
/// <summary>
/// Application entry point.
/// </summary>
/// <param name="args">Command line arguments.</param>
public static async Task Main(string[] args)
{
LogManager.Default.SetLogLevel(Environment.GetEnvironmentVariable("LOG_LEVEL"));
log = LogManager.Default.GetLogger(typeof(Program));
log.LogInfo(() => $"Starting [{serviceName}]");
log.LogInfo(() => $"LOG_LEVEL={LogManager.Default.LogLevel.ToString().ToUpper()}");
// Create process terminator to handle process termination signals.
terminator = new ProcessTerminator(log);
terminator.AddHandler(
() =>
{
// Cancel any operations in progress.
exit = true;
terminator.CancellationTokenSource.Cancel();
// This gracefully closes the [proxyNotifyChannel] so HiveMQ will
// promptly remove the associated queue.
if (proxyNotifyChannel != null)
{
proxyNotifyChannel.Dispose();
proxyNotifyChannel = null;
}
try
{
NeonHelper.WaitFor(() => !processingConfigs, terminator.Timeout);
log.LogInfo(() => "Tasks stopped gracefully.");
}
catch (TimeoutException)
{
log.LogWarn(() => $"Tasks did not stop within [{terminator.Timeout}].");
}
});
// Establish the hive connections.
if (NeonHelper.IsDevWorkstation)
{
var vaultCredentialsSecret = "neon-proxy-manager-credentials";
Environment.SetEnvironmentVariable("VAULT_CREDENTIALS", vaultCredentialsSecret);
hive = HiveHelper.OpenHiveRemote(new DebugSecrets().VaultAppRole(vaultCredentialsSecret, "neon-proxy-manager"));
}
else
{
hive = HiveHelper.OpenHive();
}
// [neon-proxy-manager] requires access to the [IHostingManager] implementation for the
// current environment, so we'll need to initialize the hosting loader.
HostingLoader.Initialize();
try
{
// Log into Vault using a Docker secret.
var vaultCredentialsSecret = Environment.GetEnvironmentVariable("VAULT_CREDENTIALS");
if (string.IsNullOrEmpty(vaultCredentialsSecret))
{
log.LogCritical("[VAULT_CREDENTIALS] environment variable does not exist.");
Program.Exit(1, immediate: true);
}
var vaultSecret = HiveHelper.GetSecret(vaultCredentialsSecret);
if (string.IsNullOrEmpty(vaultSecret))
{
log.LogCritical($"Cannot read Docker secret [{vaultCredentialsSecret}].");
Program.Exit(1, immediate: true);
}
var vaultCredentials = HiveCredentials.ParseJson(vaultSecret);
if (vaultCredentials == null)
{
log.LogCritical($"Cannot parse Docker secret [{vaultCredentialsSecret}].");
Program.Exit(1, immediate: true);
}
// Open the hive data services and then start the main service task.
log.LogInfo(() => $"Connecting: Vault");
using (vault = HiveHelper.OpenVault(vaultCredentials))
{
log.LogInfo(() => $"Connecting: Consul");
using (consul = HiveHelper.OpenConsul())
{
log.LogInfo(() => $"Connecting: Docker");
using (docker = HiveHelper.OpenDocker())
{
log.LogInfo(() => $"Connecting: {HiveMQChannels.ProxyNotify} channel");
// NOTE:
//
// We're passing [useBootstrap=true] here so that the HiveMQ client will
// connect directly to the HiveMQ cluster nodes as opposed to routing
// traffic through the private traffic manager. This is necessary because
// the load balancers rely on HiveMQ to broadcast update notifications.
//
// One consequence of this is that this service will need to be restarted
// whenever HiveMQ instances are relocated to different hive hosts.
// We're going to monitor for changes to the HiveMQ bootstrap settings
// and gracefully terminate the process when this happens. We're then
// depending on Docker to restart the process so we'll be able to pick
// up the change.
hive.HiveMQ.Internal.HiveMQBootstrapChanged +=
(s, a) =>
{
log.LogInfo("HiveMQ bootstrap settings change detected. Terminating service with [exitcode=-1] expecting that Docker will restart it.");
// Use ExitCode=-1 so that we'll restart even if the service/container
// was not configured with [restart=always].
terminator.Exit(-1);
};
using (proxyNotifyChannel = hive.HiveMQ.Internal.GetProxyNotifyChannel(useBootstrap: true).Open())
{
// Read the service settings, initializing their default values
// if they don't already exist.
if (!await consul.KV.Exists(certWarnDaysKey))
{
log.LogInfo($"Persisting setting [{certWarnDaysKey}=30.0]");
await consul.KV.PutDouble(certWarnDaysKey, 30.0);
}
if (!await consul.KV.Exists(cacheRemoveSecondsKey))
{
log.LogInfo($"Persisting setting [{cacheRemoveSecondsKey}=300.0]");
await consul.KV.PutDouble(cacheRemoveSecondsKey, 300.0);
}
if (!await consul.KV.Exists(failsafeSecondsKey))
{
log.LogInfo($"Persisting setting [{failsafeSecondsKey}=120.0]");
await consul.KV.PutDouble(failsafeSecondsKey, 120);
}
certWarnTime = TimeSpan.FromDays(await consul.KV.GetDouble(certWarnDaysKey));
cacheRemoveDelay = TimeSpan.FromDays(await consul.KV.GetDouble(cacheRemoveSecondsKey));
failsafeInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(failsafeSecondsKey));
log.LogInfo(() => $"Using setting [{certWarnDaysKey}={certWarnTime.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{cacheRemoveSecondsKey}={cacheRemoveDelay.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{failsafeSecondsKey}={failsafeInterval.TotalSeconds}]");
// Run the service tasks.
var tasks = new List <Task>();
tasks.Add(ConfigGeneratorAsync());
tasks.Add(FailsafeBroadcasterAsync());
await NeonHelper.WaitAllAsync(tasks);
}
}
}
}
}
catch (Exception e)
{
log.LogCritical(e);
Program.Exit(1);
return;
}
finally
{
HiveHelper.CloseHive();
terminator.ReadyToExit();
}
Program.Exit(0);
return;
}
