/// <summary>
/// Manages the Varnish initial configuration from Consul and Vault settings and
/// then listens for <see cref="ProxyUpdateMessage"/> messages on the <see cref="HiveMQChannels.ProxyNotify"/>
/// broadcast by <b>neon-proxy-manager</b> signalling that the configuration has been
/// updated.
/// </summary>
/// <remarks>
/// <para>
/// This method will terminate the service with an error if the configuration could
/// not be retrieved or applied for the first attempt since this very likely indicates
/// a larger problem with the hive (e.g. Consul is down).
/// </para>
/// <para>
/// If Varnish was configured successfully on the first attempt, subsequent failures
/// will be logged as warnings but the service will continue running with the out-of-date
/// configuration to provide some resilience for running hive services.
/// </para>
/// </remarks>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async static Task VarnishShim()
{
// This call ensures that Varnish is started immediately.
await ConfigureVarnish();
// Register a handler for [ProxyUpdateMessage] messages that determines
// whether the message is meant for this service instance and handle it.
StartNotifyHandler();
// Register an event handler that will be fired when the HiveMQ bootstrap
// settings change. This will restart the [ProxyUpdateMessage] listener
// using the new settings.
hive.HiveMQ.Internal.HiveMQBootstrapChanged +=
(s, a) =>
{
StartNotifyHandler();
};
// Spin quietly while waiting for a cancellation indicating that
// the service is stopping.
var task = new AsyncPeriodicTask(
TimeSpan.FromMinutes(5),
onTaskAsync: async() => await Task.FromResult(false),
onTerminateAsync:
async() =>
{
log.LogInfo(() => "VARNISH-SHIM: Terminating");
if (proxyNotifyChannel != null)
{
proxyNotifyChannel.Dispose();
proxyNotifyChannel = null;
}
await Task.CompletedTask;
},
cancellationTokenSource: terminator.CancellationTokenSource);
await task.Run();
}
public void Dispose()
{
// TODO: Close opened channels?
// Assuming so for the time being, but don't know how to send stop messages yet
InputChannel.Dispose();
// InputTvRemoteChannel.Dispose();
TextChannel.Dispose();
MediaChannel.Dispose();
BroadcastChannel.Dispose();
_sessionMessageTransport.Dispose();
_messageTransport.Dispose();
}
protected virtual void Dispose(bool disposing)
{
if (!_disposed)
{
if (disposing)
{
// TODO: Close opened channels?
// Assuming so for the time being, but don't know how to send stop messages yet
InputChannel.Dispose();
// InputTvRemoteChannel.Dispose();
TextChannel.Dispose();
MediaChannel.Dispose();
BroadcastChannel.Dispose();
_sessionMessageTransport.Dispose();
_messageTransport.Dispose();
}
_disposed = true;
}
}
/// <summary>
/// Application entry point.
/// </summary>
/// <param name="args">Command line arguments.</param>
public static async Task Main(string[] args)
{
LogManager.Default.SetLogLevel(Environment.GetEnvironmentVariable("LOG_LEVEL"));
log = LogManager.Default.GetLogger(typeof(Program));
log.LogInfo(() => $"Starting [{serviceName}]");
log.LogInfo(() => $"LOG_LEVEL={LogManager.Default.LogLevel.ToString().ToUpper()}");
// Create process terminator to handle process termination signals.
terminator = new ProcessTerminator(log);
terminator.AddHandler(
() =>
{
// Cancel any operations in progress.
exit = true;
terminator.CancellationTokenSource.Cancel();
// This gracefully closes the [proxyNotifyChannel] so HiveMQ will
// promptly remove the associated queue.
if (proxyNotifyChannel != null)
{
proxyNotifyChannel.Dispose();
proxyNotifyChannel = null;
}
try
{
NeonHelper.WaitFor(() => !processingConfigs, terminator.Timeout);
log.LogInfo(() => "Tasks stopped gracefully.");
}
catch (TimeoutException)
{
log.LogWarn(() => $"Tasks did not stop within [{terminator.Timeout}].");
}
});
// Establish the hive connections.
if (NeonHelper.IsDevWorkstation)
{
var vaultCredentialsSecret = "neon-proxy-manager-credentials";
Environment.SetEnvironmentVariable("VAULT_CREDENTIALS", vaultCredentialsSecret);
hive = HiveHelper.OpenHiveRemote(new DebugSecrets().VaultAppRole(vaultCredentialsSecret, "neon-proxy-manager"));
}
else
{
hive = HiveHelper.OpenHive();
}
// [neon-proxy-manager] requires access to the [IHostingManager] implementation for the
// current environment, so we'll need to initialize the hosting loader.
HostingLoader.Initialize();
try
{
// Log into Vault using a Docker secret.
var vaultCredentialsSecret = Environment.GetEnvironmentVariable("VAULT_CREDENTIALS");
if (string.IsNullOrEmpty(vaultCredentialsSecret))
{
log.LogCritical("[VAULT_CREDENTIALS] environment variable does not exist.");
Program.Exit(1, immediate: true);
}
var vaultSecret = HiveHelper.GetSecret(vaultCredentialsSecret);
if (string.IsNullOrEmpty(vaultSecret))
{
log.LogCritical($"Cannot read Docker secret [{vaultCredentialsSecret}].");
Program.Exit(1, immediate: true);
}
var vaultCredentials = HiveCredentials.ParseJson(vaultSecret);
if (vaultCredentials == null)
{
log.LogCritical($"Cannot parse Docker secret [{vaultCredentialsSecret}].");
Program.Exit(1, immediate: true);
}
// Open the hive data services and then start the main service task.
log.LogInfo(() => $"Connecting: Vault");
using (vault = HiveHelper.OpenVault(vaultCredentials))
{
log.LogInfo(() => $"Connecting: Consul");
using (consul = HiveHelper.OpenConsul())
{
log.LogInfo(() => $"Connecting: Docker");
using (docker = HiveHelper.OpenDocker())
{
log.LogInfo(() => $"Connecting: {HiveMQChannels.ProxyNotify} channel");
// NOTE:
//
// We're passing [useBootstrap=true] here so that the HiveMQ client will
// connect directly to the HiveMQ cluster nodes as opposed to routing
// traffic through the private traffic manager. This is necessary because
// the load balancers rely on HiveMQ to broadcast update notifications.
//
// One consequence of this is that this service will need to be restarted
// whenever HiveMQ instances are relocated to different hive hosts.
// We're going to monitor for changes to the HiveMQ bootstrap settings
// and gracefully terminate the process when this happens. We're then
// depending on Docker to restart the process so we'll be able to pick
// up the change.
hive.HiveMQ.Internal.HiveMQBootstrapChanged +=
(s, a) =>
{
log.LogInfo("HiveMQ bootstrap settings change detected. Terminating service with [exitcode=-1] expecting that Docker will restart it.");
// Use ExitCode=-1 so that we'll restart even if the service/container
// was not configured with [restart=always].
terminator.Exit(-1);
};
using (proxyNotifyChannel = hive.HiveMQ.Internal.GetProxyNotifyChannel(useBootstrap: true).Open())
{
// Read the service settings, initializing their default values
// if they don't already exist.
if (!await consul.KV.Exists(certWarnDaysKey))
{
log.LogInfo($"Persisting setting [{certWarnDaysKey}=30.0]");
await consul.KV.PutDouble(certWarnDaysKey, 30.0);
}
if (!await consul.KV.Exists(cacheRemoveSecondsKey))
{
log.LogInfo($"Persisting setting [{cacheRemoveSecondsKey}=300.0]");
await consul.KV.PutDouble(cacheRemoveSecondsKey, 300.0);
}
if (!await consul.KV.Exists(failsafeSecondsKey))
{
log.LogInfo($"Persisting setting [{failsafeSecondsKey}=120.0]");
await consul.KV.PutDouble(failsafeSecondsKey, 120);
}
certWarnTime = TimeSpan.FromDays(await consul.KV.GetDouble(certWarnDaysKey));
cacheRemoveDelay = TimeSpan.FromDays(await consul.KV.GetDouble(cacheRemoveSecondsKey));
failsafeInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(failsafeSecondsKey));
log.LogInfo(() => $"Using setting [{certWarnDaysKey}={certWarnTime.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{cacheRemoveSecondsKey}={cacheRemoveDelay.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{failsafeSecondsKey}={failsafeInterval.TotalSeconds}]");
// Run the service tasks.
var tasks = new List <Task>();
tasks.Add(ConfigGeneratorAsync());
tasks.Add(FailsafeBroadcasterAsync());
await NeonHelper.WaitAllAsync(tasks);
}
}
}
}
}
catch (Exception e)
{
log.LogCritical(e);
Program.Exit(1);
return;
}
finally
{
HiveHelper.CloseHive();
terminator.ReadyToExit();
}
Program.Exit(0);
return;
}
