/// <summary>
/// Reports a missing environment variable.
/// </summary>
/// <param name="variable">The variable name.</param>
private void LogMissingVariable(string variable)
{
if (log != null)
{
log.LogError(() => $"[{variable}] environment variable does not exist.");
}
}
/// <summary>
/// Logs an <b>error</b> message.
/// </summary>
/// <param name="message">The message.</param>
public void Error(string message)
{
if (log != null)
{
log.LogError(message, Id);
}
}
/// <summary>
/// Logs an <b>error</b> message.
/// </summary>
/// <param name="message">The object that will be serialized into the message.</param>
public void Error(object message)
{
if (log != null)
{
log.LogError(message, Id);
}
}
/// <summary>
/// Handles received <see cref="ActivityInvokeRequest"/> messages.
/// </summary>
/// <param name="client">The receiving Cadence client.</param>
/// <param name="request">The request message.</param>
/// <returns>The reply message.</returns>
private static async Task <ActivityInvokeReply> OnActivityInvokeRequest(CadenceClient client, ActivityInvokeRequest request)
{
ActivityRegistration invokeInfo;
lock (syncLock)
{
if (!nameToRegistration.TryGetValue(GetActivityTypeKey(client, request.Activity), out invokeInfo))
{
throw new KeyNotFoundException($"Cannot resolve [activityTypeName = {request.Activity}] to a registered activity type and activity method.");
}
}
var activity = Create(client, invokeInfo, request.ContextId);
try
{
var result = await activity.OnInvokeAsync(client, request.Args);
if (activity.CompleteExternally)
{
return(new ActivityInvokeReply()
{
Pending = true
});
}
else
{
return(new ActivityInvokeReply()
{
Result = result,
});
}
}
catch (CadenceException e)
{
log.LogError(e);
return(new ActivityInvokeReply()
{
Error = e.ToCadenceError()
});
}
catch (TaskCanceledException e)
{
return(new ActivityInvokeReply()
{
Error = new CancelledException(e.Message).ToCadenceError()
});
}
catch (Exception e)
{
log.LogError(e);
return(new ActivityInvokeReply()
{
Error = new CadenceError(e)
});
}
}
//---------------------------------------------------------------------
// $todo(jefflill): At least support dependency injection when constructing the controller.
//
// https://github.com/nforgeio/neonKUBE/issues/1589
//
// For some reason, KubeOps does not seem to send RECONCILE events when no changes
// have been detected, even though we return a [ResourceControllerResult] with a
// delay. We're also not seeing any RECONCILE event when the operator starts and
// there are no resources. This used to work before we upgraded to KubeOps v7.0.0-preview2.
//
// NOTE: It's very possible that the old KubeOps behavior was invalid and the current
// behavior actually is correct.
//
// This completely breaks our logic where we expect to see an IDLE event after
// all of the existing resources have been discovered or when no resources were
// discovered.
//
// We're going to work around this with a pretty horrible hack for the time being:
//
// 1. We're going to use the [nextNoChangeReconcileUtc] field to track
// when the next IDLE event should be raised. This will default
// to the current time plus 1 minute when the resource manager is
// constructed. This gives KubeOps a chance to discover existing
// resources before we start raising IDLE events.
//
// 2. After RECONCILE events are handled by the operator's controller,
// we'll reset the [nextNoChangeReconcileUtc] property to be the current
// time plus the [reconciledNoChangeInterval].
//
// 3. The [NoChangeLoop()] method below loops watching for when [nextNoChangeReconcileUtc]
// indicates that an IDLE RECONCILE event should be raised. The loop
// will instantiate an instance of the controller, hardcoding the [IKubernetes]
// constructor parameter for now, rather than supporting real dependency
// injection. We'll then call [ReconcileAsync()] ourselves.
//
// The loop uses [mutex] to ensure that only controller event handler is
// called at a time, so this should be thread/task safe.
//
// 4. We're only going to do this for RECONCILE events right now: our
// operators aren't currently monitoring DELETED or STATUS-MODIFIED
// events and I suspect that KubeOps is probably doing the correct
// thing for these anyway.
//
// PROBLEM:
//
// This hack can result in a problem when KubeOps is not able to watch the resource
// for some reason. The problem is that if this continutes for the first 1 minute
// delay, then the loop below will tragger an IDLE RECONCILE event with no including
// no items, and then the operator could react by deleting any existing related physical
// resources, which would be REALLY BAD.
//
// To mitigate this, I'm going to special case the first IDLE reconcile to query the
// custom resources and only trigger the IDLE reconcile when the query succeeded and
// no items were returned. Otherwise KubeOps may be having trouble communicating with
// Kubernetes or when there are items, we should expect KubeOps to reconcile those for us.
/// <summary>
/// This loop handles raising of <see cref="IOperatorController{TEntity}.IdleAsync()"/>
/// events when there's been no changes to any of the monitored resources.
/// </summary>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task IdleLoopAsync()
{
await SyncContext.Clear;
var loopDelay = TimeSpan.FromSeconds(1);
while (!isDisposed && !stopIdleLoop)
{
await Task.Delay(loopDelay);
if (DateTime.UtcNow >= nextIdleReconcileUtc)
{
// Don't send an IDLE RECONCILE while we're when we're not the leader.
if (IsLeader)
{
// We're going to log and otherwise ignore any exceptions thrown by the
// operator's controller or from any members above called by the controller.
await mutex.ExecuteActionAsync(
async() =>
{
try
{
// $todo(jefflill):
//
// We're currently assuming that operator controllers all have a constructor
// that accepts a single [IKubernetes] parameter. We should change this to
// doing actual dependency injection when we have the time.
//
// https://github.com/nforgeio/neonKUBE/issues/1589
var controller = CreateController();
await controller.IdleAsync();
}
catch (OperationCanceledException)
{
// Exit the loop when the [mutex] is disposed which happens
// when the resource manager is disposed.
return;
}
catch (Exception e)
{
options.IdleErrorCounter?.Inc();
log.LogError(e);
}
});
}
nextIdleReconcileUtc = DateTime.UtcNow + options.IdleInterval;
}
}
}
/// <summary>
/// Implements the async timer loop.
/// </summary>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TimerLoopAsync()
{
await SyncContext.Clear;
while (!cts.IsCancellationRequested)
{
try
{
if (delayFirstTick)
{
await Task.Delay(interval, cts.Token);
}
try
{
await callback();
}
catch (Exception e)
{
log.LogError(e);
}
if (!delayFirstTick)
{
await Task.Delay(interval, cts.Token);
}
}
catch (OperationCanceledException)
{
return;
}
}
}
/// <summary>
/// Logs an error message retrieved via a message function.
/// </summary>
/// <param name="log">The log.</param>
/// <param name="messageFunc">The message function.</param>
/// <param name="activityId">The optional activity ID.</param>
/// <remarks>
/// This method is intended mostly to enable the efficient use of interpolated C# strings.
/// </remarks>
public static void LogError(this INeonLogger log, Func <string> messageFunc, string activityId = null)
{
if (log.IsLogErrorEnabled)
{
log.LogError(messageFunc(), activityId);
}
}
/// <summary>
/// Logs an error exception.
/// </summary>
/// <param name="log">The log.</param>
/// <param name="e">The exception.</param>
public static void LogError(this INeonLogger log, Exception e)
{
if (log.IsErrorEnabled)
{
log.LogError(null, e);
}
}
/// <summary>
/// Logs an error message retrieved via a message function.
/// </summary>
/// <param name="log">The log.</param>
/// <param name="messageFunc">The message function.</param>
/// <remarks>
/// This method is intended mostly to enable the efficient use of interpolated C# strings.
/// </remarks>
public static void LogError(this INeonLogger log, Func <object> messageFunc)
{
if (log.IsErrorEnabled)
{
log.LogError(messageFunc());
}
}
/// <summary>
/// Logs an error exception.
/// </summary>
/// <param name="log">The log.</param>
/// <param name="e">The exception.</param>
/// <param name="activityId">The optional activity ID.</param>
public static void LogError(this INeonLogger log, Exception e, string activityId = null)
{
if (log.IsLogErrorEnabled)
{
log.LogError(null, e, activityId);
}
}
/// <summary>
/// Implements the service as a <see cref="Task"/>.
/// </summary>
/// <returns>The <see cref="Task"/>.</returns>
private static async Task RunAsync()
{
try
{
var settings =
new CouchbaseSettings()
{
Servers = new List <Uri>()
{
new Uri("couchbase://10.0.0.90"),
new Uri("couchbase://10.0.0.91"),
new Uri("couchbase://10.0.0.92")
},
Bucket = "stoke"
};
var credentials =
new Credentials()
{
Username = Environment.GetEnvironmentVariable("TS_COUCHBASE_USERNAME"),
Password = Environment.GetEnvironmentVariable("TS_COUCHBASE_PASSWORD")
};
using (var bucket = settings.OpenBucket(credentials))
{
var retry = new ExponentialRetryPolicy(CouchbaseTransientDetector.IsTransient);
for (int i = 0; i < 500000; i++)
{
var key = bucket.GenKey();
await retry.InvokeAsync(async() => await bucket.InsertSafeAsync(key, new Document <Item>()
{
Id = key, Content = new Item()
{
Name = "Jeff", Age = 56
}
}));
var exists = await bucket.ExistsAsync(key);
var result2 = await bucket.GetAsync <Document <Item> >(key);
result2.EnsureSuccess();
}
}
}
catch (OperationCanceledException)
{
return;
}
catch (Exception e)
{
log.LogError(e);
}
finally
{
terminator.ReadyToExit();
}
}
/// <summary>
/// Track Exceptions in Google Analytics.
/// </summary>
/// <param name="method"></param>
/// <param name="exception"></param>
/// <param name="isFatal"></param>
/// <returns>The tracking <see cref="Task"/>.</returns>
public async Task TrackExceptionAsync(MethodBase method, Exception exception, bool?isFatal = false)
{
await SyncContext.Clear;
Logger.LogError(exception);
await Analytics.TrackEvent(method.Name, new
{
Category = "Exception",
Labels = new Dictionary <string, string>()
{
{ "Exception", $"{method.Name}::{exception.GetType().Name}" },
{ "IsFatal", $"{isFatal}" }
},
Message = exception.Message
});
}
/// <summary>
/// Called internally to execute the activity.
/// </summary>
/// <param name="client">The Cadence client.</param>
/// <param name="args">The encoded activity arguments.</param>
/// <returns>The activity results.</returns>
internal async Task <byte[]> OnInvokeAsync(CadenceClient client, byte[] args)
{
await SyncContext.Clear;
Covenant.Requires <ArgumentNullException>(client != null, nameof(client));
// Capture the activity information.
var reply = (ActivityGetInfoReply)(await Client.CallProxyAsync(
new ActivityGetInfoRequest()
{
ContextId = ContextId,
}));
reply.ThrowOnError();
ActivityTask = reply.Info.ToPublic();
// Invoke the activity.
if (IsLocal)
{
// This doesn't make sense for local activities.
ActivityTask.ActivityTypeName = null;
return(await InvokeAsync(client, args));
}
else
{
// Track the activity.
var activityKey = new ActivityKey(client, ContextId);
try
{
lock (syncLock)
{
idToActivity[activityKey] = this;
}
return(await InvokeAsync(client, args));
}
catch (Exception e)
{
logger.LogError(e);
throw;
}
finally
{
lock (syncLock)
{
idToActivity.Remove(activityKey);
}
}
}
}
/// <summary>
/// Application entry point.
/// </summary>
/// <param name="args">Command line arguments.</param>
public static async Task Main(string[] args)
{
LogManager.Default.SetLogLevel(Environment.GetEnvironmentVariable("LOG_LEVEL"));
log = LogManager.Default.GetLogger(typeof(Program));
log.LogInfo(() => $"Starting [{serviceName}]");
log.LogInfo(() => $"LOG_LEVEL={LogManager.Default.LogLevel.ToString().ToUpper()}");
// Create process terminator to handle termination signals.
terminator = new ProcessTerminator(log);
try
{
var commandLine = new CommandLine(args);
var command = commandLine.Arguments.ElementAtOrDefault(0);
if (command == null)
{
log.LogError("usage: vegomatic COMMAND ARGS...");
Program.Exit(1, immediate: true);
}
switch (command)
{
case "cephfs":
await new CephFS().ExecAsync(commandLine.Shift(1));
break;
case "issue-mntc":
await new IssueMntc().ExecAsync(commandLine.Shift(1));
break;
default:
case "test-server":
await new TestServer().ExecAsync(commandLine.Shift(1));
break;
}
}
catch (Exception e)
{
log.LogCritical(e);
Program.Exit(1);
return;
}
finally
{
HiveHelper.CloseHive();
terminator.ReadyToExit();
}
Program.Exit(0);
return;
}
/// <summary>
/// Add a subscription to the store.
/// </summary>
/// <param name="id"></param>
/// <param name="connection"></param>
/// <param name="subscribeMethod"></param>
/// <returns></returns>
public async Task AddSubscriptionAsync(string id, HubConnectionContext connection, Func <string, HubConnectionStore, Task <IAsyncSubscription> > subscribeMethod)
{
await _lock.WaitAsync();
logger?.LogDebug($"Subscribing to subject [Subject={id}].");
try
{
// Avoid adding subscription if connection is closing/closed
// We're in a lock and ConnectionAborted is triggered before OnDisconnectedAsync is called so this is guaranteed to be safe when adding while connection is closing and removing items
if (connection.ConnectionAborted.IsCancellationRequested)
{
return;
}
var subscription = subscriptions.GetOrAdd(id, _ => new HubConnectionStore());
subscription.Add(connection);
// Subscribe once
if (subscription.Count == 1)
{
var sAsync = await subscribeMethod(id, subscription);
sAsync.Start();
natsSubscriptions.GetOrAdd(id, _ => sAsync);
}
}
catch (Exception e)
{
logger?.LogError(e);
logger?.LogDebug($"Subscribing failed. [Subject={id}] [Connection={connection.ConnectionId}]");
}
finally
{
_lock.Release();
}
}
public static async Task Main(string[] args)
{
// Initialize the logger.
LogManager.Default.SetLogLevel("info");
logger = LogManager.Default.GetLogger(typeof(Program));
logger.LogInfo("Starting workflow service");
try
{
// Connect to Cadence
var settings = new CadenceSettings()
{
DefaultDomain = "my-domain",
CreateDomain = true,
Servers = new List <string>()
{
"cadence://localhost:7933"
}
};
using (var client = await CadenceClient.ConnectAsync(settings))
{
// Register your workflow and activity implementations to let
// Cadence know we're open for business.
await client.RegisterAssemblyAsync(System.Reflection.Assembly.GetExecutingAssembly());
await client.StartWorkerAsync("my-tasks");
// Spin forever, processing workflows and activities assigned by Cadence.
while (true)
{
await Task.Delay(TimeSpan.FromMinutes(5));
}
}
}
catch (Exception e)
{
logger.LogError(e);
}
finally
{
logger.LogInfo("Exiting workflow service");
}
}
public static async Task Main(string[] args)
{
// Initialize the logger.
LogManager.Default.SetLogLevel("info");
logger = LogManager.Default.GetLogger(typeof(Program));
logger.LogInfo("Starting workflow service");
try
{
// Connect to Temporal
var settings = new TemporalSettings()
{
Namespace = "my-namespace",
CreateNamespace = true,
HostPort = "localhost:7933"
};
using (var client = await TemporalClient.ConnectAsync(settings))
{
// Create a worker and register the workflow and activity
// implementations to let Temporal know we're open for business.
var worker = await client.NewWorkerAsync();
await worker.RegisterAssemblyAsync(System.Reflection.Assembly.GetExecutingAssembly());
await worker.StartAsync();
// Spin forever, processing workflows and activities assigned by Temporal.
while (true)
{
await Task.Delay(TimeSpan.FromMinutes(5));
}
}
}
catch (Exception e)
{
logger.LogError(e);
}
finally
{
logger.LogInfo("Exiting workflow service");
}
}
/// <summary>
/// Handles the timer dispatch.
/// </summary>
/// <param name="state"></param>
private void OnTimer(object state)
{
lock (syncLock)
{
// Ignore timer events if we're already processing one or if the
// timer has been disposed.
if (inCallback || timer == null)
{
return;
}
// Disable the timer and indicate that we're processing
// the callback
timer.Change(Timeout.Infinite, Timeout.Infinite);
inCallback = true;
}
try
{
callback(state);
}
catch (Exception e)
{
logger.LogError(e);
}
lock (syncLock)
{
inCallback = false;
// If the timer hasn't been disposed then reenable it.
if (timer != null)
{
timer.Change(period, period);
}
}
}
/// <summary>
/// Called internally to execute the activity.
/// </summary>
/// <param name="args">The encoded activity arguments.</param>
/// <returns>The activity results.</returns>
internal async Task <byte[]> OnInvokeAsync(byte[] args)
{
await SyncContext.Clear;
// Capture the activity context details.
var reply = (ActivityGetInfoReply)(await Client.CallProxyAsync(
new ActivityGetInfoRequest()
{
ContextId = ContextId,
}));
reply.ThrowOnError();
ActivityTask = reply.Info;
// Invoke the activity.
if (IsLocal)
{
// This doesn't make sense for local activities.
ActivityTask.ActivityType = null;
return(await InvokeAsync(Client, args));
}
else
{
try
{
return(await InvokeAsync(worker.Client, args));
}
catch (Exception e)
{
logger.LogError(e);
throw;
}
}
}
/// <summary>
/// Transforms the response before returning it to the client.
///
/// <para>
/// This method will add a <see cref="Cookie"/> to each response containing relevant information
/// about the current authentication flow. It also intercepts redirects from Dex and saves any relevant
/// tokens to a cache for reuse.
/// </para>
/// </summary>
/// <param name="httpContext"></param>
/// <param name="proxyResponse"></param>
/// <returns></returns>
public override async ValueTask <bool> TransformResponseAsync(HttpContext httpContext,
HttpResponseMessage proxyResponse)
{
await base.TransformResponseAsync(httpContext, proxyResponse);
Cookie cookie = null;
if (httpContext.Request.Cookies.TryGetValue(Service.SessionCookieName, out var requestCookieBase64))
{
try
{
logger.LogDebug($"Decrypting existing cookie.");
cookie = NeonHelper.JsonDeserialize <Cookie>(cipher.DecryptBytesFrom(requestCookieBase64));
}
catch (Exception e)
{
logger.LogError(e);
cookie = new Cookie();
}
}
else
{
logger.LogDebug($"Cookie not present.");
cookie = new Cookie();
}
// If we're being redirected, intercept request and save token to cookie.
if (httpContext.Response.Headers.Location.Count > 0 &&
Uri.IsWellFormedUriString(httpContext.Response.Headers.Location.Single(), UriKind.Absolute))
{
var location = new Uri(httpContext.Response.Headers.Location.Single());
var code = HttpUtility.ParseQueryString(location.Query).Get("code");
if (!string.IsNullOrEmpty(code))
{
if (cookie != null)
{
var redirect = cookie.RedirectUri;
var token = await dexClient.GetTokenAsync(cookie.ClientId, code, redirect, "authorization_code");
await cache.SetAsync(code, cipher.EncryptToBytes(NeonHelper.JsonSerializeToBytes(token)), cacheOptions);
logger.LogDebug(NeonHelper.JsonSerialize(token));
cookie.TokenResponse = token;
httpContext.Response.Cookies.Append(
Service.SessionCookieName,
cipher.EncryptToBase64(NeonHelper.JsonSerialize(cookie)),
new CookieOptions()
{
Path = "/",
Expires = DateTime.UtcNow.AddSeconds(token.ExpiresIn.Value).AddMinutes(-60),
Secure = true,
SameSite = SameSiteMode.Strict
});
return(true);
}
}
}
// Add query parameters to the cookie.
if (httpContext.Request.Query.TryGetValue("client_id", out var clientId))
{
logger.LogDebug($"Client ID: [{clientId}]");
cookie.ClientId = clientId;
}
if (httpContext.Request.Query.TryGetValue("state", out var state))
{
logger.LogDebug($"State: [{state}]");
cookie.State = state;
}
if (httpContext.Request.Query.TryGetValue("redirect_uri", out var redirectUri))
{
logger.LogDebug($"Redirect Uri: [{redirectUri}]");
cookie.RedirectUri = redirectUri;
}
if (httpContext.Request.Query.TryGetValue("scope", out var scope))
{
logger.LogDebug($"Scope: [{scope}]");
cookie.Scope = scope;
}
if (httpContext.Request.Query.TryGetValue("response_type", out var responseType))
{
logger.LogDebug($"Response Type: [{responseType}]");
cookie.ResponseType = responseType;
}
httpContext.Response.Cookies.Append(
Service.SessionCookieName,
cipher.EncryptToBase64(NeonHelper.JsonSerialize(cookie)),
new CookieOptions()
{
Path = "/",
Expires = DateTime.UtcNow.AddHours(24),
Secure = true,
SameSite = SameSiteMode.Strict
});
return(true);
}
/// <summary>
/// <para>
/// Entrypoint called as part of the request pipeline.
/// </para>
/// <para>
/// This method is responsible for intercepting token requests from clients.
/// If the client has a valid cookie with a token response in it, we save the
/// token to cache and redirect them back with a code referencing the token in
/// the cache.
/// </para>
/// </summary>
public async Task InvokeAsync(
HttpContext context,
Service NeonSsoSessionProxyService,
IDistributedCache cache,
AesCipher cipher,
DistributedCacheEntryOptions cacheOptions,
INeonLogger logger)
{
try
{
if (context.Request.Cookies.TryGetValue(Service.SessionCookieName, out var requestCookieBase64))
{
var requestCookie = NeonHelper.JsonDeserialize <Cookie>(cipher.DecryptBytesFrom(requestCookieBase64));
if (requestCookie.TokenResponse != null)
{
var code = NeonHelper.GetCryptoRandomPassword(10);
await cache.SetAsync(code, cipher.EncryptToBytes(NeonHelper.JsonSerializeToBytes(requestCookie.TokenResponse)), cacheOptions);
var query = new Dictionary <string, string>()
{
{ "code", code }
};
if (context.Request.Query.TryGetValue("state", out var state))
{
query["state"] = state;
}
if (context.Request.Query.TryGetValue("redirect_uri", out var redirectUri))
{
if (context.Request.Query.TryGetValue("client_id", out var clientId))
{
if (!NeonSsoSessionProxyService.Config.StaticClients.Where(client => client.Id == clientId).First().RedirectUris.Contains(redirectUri))
{
logger.LogError("Invalid redirect URI");
throw new HttpRequestException("Invalid redirect URI.");
}
context.Response.StatusCode = StatusCodes.Status302Found;
context.Response.Headers.Location = QueryHelpers.AddQueryString(redirectUri, query);
logger.LogDebug($"Client and Redirect URI confirmed. [ClientID={clientId}] [RedirectUri={redirectUri}]");
return;
}
else
{
logger.LogError("No Client ID specified.");
throw new HttpRequestException("Invalid Client ID.");
}
}
else
{
throw new HttpRequestException("No redirect_uri specified.");
}
}
}
}
catch (Exception e)
{
NeonSsoSessionProxyService.Log.LogError(e);
}
await _next(context);
}
/// <summary>
/// Called periodically to allow the operator to perform global events.
/// </summary>
/// <returns>The tracking <see cref="Task"/>.</returns>
public async Task IdleAsync()
{
log.LogInfo("[IDLE]");
// We're going to handle this by looking at each node task and checking
// to see whether the target node actually exists. Rather than listing
// the node first, which would be expensive for a large cluster we'll
// fetch and cache node information as we go along.
var nodeNameToExists = new Dictionary <string, bool>(StringComparer.InvariantCultureIgnoreCase);
var resources = (await k8s.ListClusterCustomObjectAsync <V1NeonNodeTask>()).Items;
foreach (var nodeTask in resources)
{
var deleteMessage = $"Deleting node task [{nodeTask.Name()}] because it is assigned to the non-existent cluster node [{nodeTask.Spec.Node}].";
if (nodeNameToExists.TryGetValue(nodeTask.Spec.Node, out var nodeExists))
{
// Target node status is known.
if (nodeExists)
{
continue;
}
else
{
log.LogInfo(deleteMessage);
try
{
await k8s.DeleteClusterCustomObjectAsync(nodeTask);
}
catch (Exception e)
{
log.LogError(e);
}
continue;
}
}
// Determine whether the node exists.
try
{
var node = await k8s.ReadNodeAsync(nodeTask.Spec.Node);
nodeExists = true;
nodeNameToExists.Add(nodeTask.Spec.Node, nodeExists);
}
catch (HttpOperationException e)
{
if (e.Response.StatusCode == HttpStatusCode.NotFound)
{
nodeExists = false;
nodeNameToExists.Add(nodeTask.Spec.Node, nodeExists);
}
else
{
log.LogError(e);
continue;
}
}
catch (Exception e)
{
log.LogError(e);
continue;
}
if (!nodeExists)
{
log.LogInfo(deleteMessage);
try
{
await k8s.DeleteClusterCustomObjectAsync(nodeTask);
}
catch (Exception e)
{
log.LogError(e);
}
}
}
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="neonLogger">The Neon base logger.</param>
public HiveEasyMQLogProvider(INeonLogger neonLogger)
{
Covenant.Requires <ArgumentNullException>(neonLogger != null);
this.neonLogger = neonLogger;
this.loggerFunc =
(logLevel, messageFunc, exception, formatParameters) =>
{
if (messageFunc == null)
{
return(true);
}
var message = LogMessageFormatter.FormatStructuredMessage(messageFunc(), formatParameters, out _);
switch (logLevel)
{
case EasyNetQ.Logging.LogLevel.Trace:
// NOTE: Neon logging doesn't have a TRACE level so we'll
// map these to DEBUG.
case EasyNetQ.Logging.LogLevel.Debug:
if (neonLogger.IsDebugEnabled)
{
if (exception == null)
{
neonLogger.LogDebug(message);
}
else
{
neonLogger.LogDebug(message, exception);
}
}
break;
case EasyNetQ.Logging.LogLevel.Error:
if (neonLogger.IsErrorEnabled)
{
if (exception == null)
{
neonLogger.LogError(message);
}
else
{
neonLogger.LogError(message, exception);
}
}
break;
case EasyNetQ.Logging.LogLevel.Fatal:
if (neonLogger.IsCriticalEnabled)
{
if (exception == null)
{
neonLogger.LogCritical(message);
}
else
{
neonLogger.LogCritical(message, exception);
}
}
break;
case EasyNetQ.Logging.LogLevel.Info:
if (neonLogger.IsInfoEnabled)
{
if (exception == null)
{
neonLogger.LogInfo(message);
}
else
{
neonLogger.LogInfo(message, exception);
}
}
break;
case EasyNetQ.Logging.LogLevel.Warn:
if (neonLogger.IsWarnEnabled)
{
if (exception == null)
{
neonLogger.LogWarn(message);
}
else
{
neonLogger.LogWarn(message, exception);
}
}
break;
}
return(true);
};
}
/// <summary>
/// Application entry point.
/// </summary>
/// <param name="args">Command line arguments.</param>
public static async Task Main(string[] args)
{
LogManager.Default.SetLogLevel(Environment.GetEnvironmentVariable("LOG_LEVEL"));
log = LogManager.Default.GetLogger(typeof(Program));
// Create process terminator to handle termination signals.
terminator = new ProcessTerminator(log);
terminator.AddHandler(
() =>
{
// Cancel any operations in progress.
terminator.CancellationTokenSource.Cancel();
});
// Read the environment variables.
// $hack(jeff.lill:
//
// We're going to scan the Consul configuration key to determine whether this
// instance is managing the public or private proxy (or bridges) so we'll
// be completely compatible with existing deployments.
//
// In theory, we could have passed a new environment variable but that's not
// worth the trouble.
configKey = Environment.GetEnvironmentVariable("CONFIG_KEY");
if (string.IsNullOrEmpty(configKey))
{
log.LogError("[CONFIG_KEY] environment variable is required.");
Program.Exit(1, immediate: true);
}
isPublic = configKey.Contains("/public/");
var proxyName = isPublic ? "public" : "private";
serviceName = $"neon-proxy-{proxyName}-cache:{GitVersion}";
log.LogInfo(() => $"Starting [{serviceName}]");
configHashKey = Environment.GetEnvironmentVariable("CONFIG_HASH_KEY");
if (string.IsNullOrEmpty(configHashKey))
{
log.LogError("[CONFIG_HASH_KEY] environment variable is required.");
Program.Exit(1, immediate: true);
}
var memoryLimitValue = Environment.GetEnvironmentVariable("MEMORY_LIMIT");
if (string.IsNullOrEmpty(memoryLimitValue))
{
memoryLimitValue = DefMemoryLimitString;
}
if (!NeonHelper.TryParseCount(memoryLimitValue, out var memoryLimitDouble))
{
memoryLimitDouble = DefMemoryLimit;
}
if (memoryLimitDouble < MinMemoryLimit)
{
log.LogWarn(() => $"[MEMORY_LIMIT={memoryLimitValue}] is to small. Using [{MinMemoryLimitString}] instead.");
memoryLimitDouble = MinMemoryLimit;
}
memoryLimit = (long)memoryLimitDouble;
var warnSeconds = Environment.GetEnvironmentVariable("WARN_SECONDS");
if (string.IsNullOrEmpty(warnSeconds) || !double.TryParse(warnSeconds, out var warnSecondsValue))
{
warnInterval = TimeSpan.FromSeconds(300);
}
else
{
warnInterval = TimeSpan.FromSeconds(warnSecondsValue);
}
debugMode = "true".Equals(Environment.GetEnvironmentVariable("DEBUG"), StringComparison.InvariantCultureIgnoreCase);
log.LogInfo(() => $"LOG_LEVEL={LogManager.Default.LogLevel.ToString().ToUpper()}");
log.LogInfo(() => $"CONFIG_KEY={configKey}");
log.LogInfo(() => $"CONFIG_HASH_KEY={configHashKey}");
log.LogInfo(() => $"MEMORY_LIMIT={memoryLimit}");
log.LogInfo(() => $"WARN_SECONDS={warnInterval}");
log.LogInfo(() => $"DEBUG={debugMode}");
// Ensure that the required directories exist.
Directory.CreateDirectory(tmpfsFolder);
Directory.CreateDirectory(configFolder);
Directory.CreateDirectory(configUpdateFolder);
// Establish the hive connections.
if (NeonHelper.IsDevWorkstation)
{
throw new NotImplementedException("This service works only within a Linux container with Varnish installed.");
//var vaultCredentialsSecret = "neon-proxy-manager-credentials";
//Environment.SetEnvironmentVariable("VAULT_CREDENTIALS", vaultCredentialsSecret);
//hive = HiveHelper.OpenHiveRemote(new DebugSecrets().VaultAppRole(vaultCredentialsSecret, $"neon-proxy-{proxyName}"));
}
else
{
hive = HiveHelper.OpenHive();
}
try
{
// Open Consul and then start the service tasks.
log.LogInfo(() => $"Connecting: Consul");
using (consul = HiveHelper.OpenConsul())
{
log.LogInfo(() => $"Connecting: {HiveMQChannels.ProxyNotify} channel");
// Verify that the required Consul keys exist or loop to wait until they
// are created. This will allow the service wait for pending hive setup
// operations to be completed.
while (!await consul.KV.Exists(configKey))
{
log.LogWarn(() => $"Waiting for [{configKey}] key to be present in Consul.");
await Task.Delay(TimeSpan.FromSeconds(5));
}
while (!await consul.KV.Exists(configHashKey))
{
log.LogWarn(() => $"Waiting for [{configHashKey}] key to be present in Consul.");
await Task.Delay(TimeSpan.FromSeconds(5));
}
// Crank up the service tasks.
log.LogInfo(() => $"Starting service tasks.");
await NeonHelper.WaitAllAsync(
CacheWarmer(),
ErrorPollerAsync(),
VarnishShim());
}
}
catch (Exception e)
{
log.LogCritical(e);
Program.Exit(1);
return;
}
finally
{
HiveHelper.CloseHive();
terminator.ReadyToExit();
}
Program.Exit(0);
return;
}
/// <summary>
/// Implements the service as a <see cref="Task"/>.
/// </summary>
/// <returns>The <see cref="Task"/>.</returns>
private static async Task RunAsync()
{
// Load the settings.
//
// Initialize the proxy manager settings to their default values
// if they don't already exist.
if (!await consul.KV.Exists(hivemqMaintainSecondsKey))
{
log.LogInfo($"Persisting setting [{hivemqMaintainSecondsKey}=60.0]");
await consul.KV.PutDouble(hivemqMaintainSecondsKey, 60);
}
if (!await consul.KV.Exists(logPurgeSecondsKey))
{
log.LogInfo($"Persisting setting [{logPurgeSecondsKey}=300.0]");
await consul.KV.PutDouble(logPurgeSecondsKey, 300);
}
if (!await consul.KV.Exists(managerTopologySecondsKey))
{
log.LogInfo($"Persisting setting [{managerTopologySecondsKey}=300.0]");
await consul.KV.PutDouble(managerTopologySecondsKey, 1800);
}
if (!await consul.KV.Exists(proxyUpdateSecondsKey))
{
log.LogInfo($"Persisting setting [{proxyUpdateSecondsKey}=60.0]");
await consul.KV.PutDouble(proxyUpdateSecondsKey, 60);
}
if (!await consul.KV.Exists(secretPurgeSecondsKey))
{
log.LogInfo($"Persisting setting [{secretPurgeSecondsKey}=300.0]");
await consul.KV.PutDouble(secretPurgeSecondsKey, 300);
}
if (!await consul.KV.Exists(swarmPollSecondsKey))
{
log.LogInfo($"Persisting setting [{swarmPollSecondsKey}=30.0]");
await consul.KV.PutDouble(swarmPollSecondsKey, 30.0);
}
if (!await consul.KV.Exists(vaultUnsealSecondsKey))
{
log.LogInfo($"Persisting setting [{vaultUnsealSecondsKey}=30.0]");
await consul.KV.PutDouble(vaultUnsealSecondsKey, 30.0);
}
hivemqMantainInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(hivemqMaintainSecondsKey));
logPurgerInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(logPurgeSecondsKey));
managerTopologyInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(managerTopologySecondsKey));
proxyUpdateInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(proxyUpdateSecondsKey));
secretPurgeInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(secretPurgeSecondsKey));
swarmPollInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(swarmPollSecondsKey));
vaultUnsealInterval = TimeSpan.FromSeconds(await consul.KV.GetDouble(vaultUnsealSecondsKey));
log.LogInfo(() => $"Using setting [{hivemqMaintainSecondsKey}={hivemqMantainInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{logPurgeSecondsKey}={logPurgerInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{managerTopologySecondsKey}={managerTopologyInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{proxyUpdateSecondsKey}={proxyUpdateInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{secretPurgeSecondsKey}={secretPurgeInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{swarmPollSecondsKey}={swarmPollInterval.TotalSeconds}]");
log.LogInfo(() => $"Using setting [{vaultUnsealSecondsKey}={vaultUnsealInterval.TotalSeconds}]");
// Parse the Vault credentials from the [neon-hive-manager-vaultkeys]
// secret, if it exists.
var vaultCredentialsJson = HiveHelper.GetSecret("neon-hive-manager-vaultkeys");
if (string.IsNullOrWhiteSpace(vaultCredentialsJson))
{
log.LogInfo(() => "Vault AUTO-UNSEAL is DISABLED because [neon-hive-manager-vaultkeys] Docker secret is not specified.");
}
else
{
try
{
vaultCredentials = NeonHelper.JsonDeserialize <VaultCredentials>(vaultCredentialsJson);
log.LogInfo(() => "Vault AUTO-UNSEAL is ENABLED.");
}
catch (Exception e)
{
log.LogError("Vault AUTO-UNSEAL is DISABLED because the [neon-hive-manager-vaultkeys] Docker secret could not be parsed.", e);
}
}
// We're going to need this later.
vaultUris = await GetVaultUrisAsync();
// Launch the sub-tasks. These will run until the service is terminated.
var tasks = new List <Task>();
// Start a task that handles HiveMQ related activities like ensuring that
// the [sysadmin] account has full permissions for all virtual hosts.
tasks.Add(HiveMQMaintainerAsync());
// Start a task that checks for Elasticsearch [logstash] and [metricbeat] indexes
// that are older than the number of retention days.
tasks.Add(LogPurgerAsync());
// Start a task that periodically checks for changes to the set of hive managers
// (e.g. if a manager is added or removed). This task will cause the service to exit
// so it can be restarted automatically by Docker to respond to the change.
tasks.Add(ManagerWatcherAsync());
// Start a task that checks for old [neon-secret-retriever-*] service instances
// as well as old persisted secrets and removes them.
tasks.Add(SecretPurgerAsync());
// Start a task that polls current hive state to update the hive definition in Consul, etc.
tasks.Add(SwarmPollerAsync());
// Start a task that periodically notifies the [neon-proxy-manager] service
// that it should proactively rebuild the proxy configurations.
tasks.Add(ProxyUpdaterAsync());
// We need to start a vault poller for the Vault instance running on each manager
// node. We're going to construct the direct Vault URIs by querying Docker for
// the current hive nodes and looking for the managers.
foreach (var uri in vaultUris)
{
tasks.Add(VaultUnsealerAsync(uri));
}
// Wait for all tasks to exit cleanly for a normal shutdown.
await NeonHelper.WaitAllAsync(tasks);
}
/// <summary>
/// Main program entry point.
/// </summary>
/// <param name="args">The command line arguments.</param>
public static void Main(string[] args)
{
LogManager.Default.SetLogLevel(Environment.GetEnvironmentVariable("LOG_LEVEL"));
log = LogManager.Default.GetLogger(typeof(Program));
log.LogInfo(() => $"Starting [{serviceName}]");
log.LogInfo(() => $"LOG_LEVEL={LogManager.Default.LogLevel.ToString().ToUpper()}");
// Create process terminator to handle termination signals.
terminator = new ProcessTerminator(log);
terminator.AddHandler(() => terminator.ReadyToExit());
// Establish the hive connections.
if (NeonHelper.IsDevWorkstation)
{
var vaultCredentialsSecret = "neon-proxy-manager-credentials";
Environment.SetEnvironmentVariable("VAULT_CREDENTIALS", vaultCredentialsSecret);
hive = HiveHelper.OpenHiveRemote(new DebugSecrets().VaultAppRole(vaultCredentialsSecret, "neon-proxy-manager"));
}
else
{
hive = HiveHelper.OpenHive();
}
// Parse the command line.
var commandLine = new CommandLine(args);
if (commandLine.Arguments.Count() != 2)
{
log.LogError($"*** ERROR: Invalid command line arguments: {commandLine}");
log.LogError($"*** Expected: MYSECRET MYCONSULKEY");
SleepForever();
}
var secretName = commandLine.Arguments[0];
var consulKey = commandLine.Arguments[1];
try
{
// Read the secret file.
var secretPath = ($"/run/secrets/{secretName}");
log.LogInfo($"Reading secret [{secretName}].");
if (!File.Exists(secretPath))
{
log.LogError($"The secret file [{secretPath}] does not exist.");
}
else
{
var secret = File.ReadAllBytes(secretPath);
log.LogInfo($"Writing secret to Consul [{consulKey}].");
HiveHelper.Consul.KV.PutBytes(consulKey, secret).Wait();
}
}
catch (Exception e)
{
log.LogError(e);
}
SleepForever();
}
/// <summary>
/// Implements the service as a <see cref="Task"/>.
/// </summary>
/// <returns>The <see cref="Task"/>.</returns>
private static async Task RunAsync()
{
var localMD5 = string.Empty;
var remoteMD5 = "[unknown]";
var verifyTimer = new PolledTimer(verifyInterval, autoReset: true);
var periodicTask =
new AsyncPeriodicTask(
pollInterval,
onTaskAsync:
async() =>
{
log.LogDebug(() => "Starting poll");
log.LogDebug(() => "Fetching DNS answers MD5 from Consul.");
remoteMD5 = await consul.KV.GetStringOrDefault(HiveConst.ConsulDnsHostsMd5Key, terminator.CancellationToken);
if (remoteMD5 == null)
{
remoteMD5 = "[unknown]";
}
var verify = verifyTimer.HasFired;
if (verify)
{
// Under normal circumstances, we should never see the reload signal file
// here because the [neon-dns-loader] service should have deleted it after
// handling the last change signal.
//
// This probably means that [neon-dns-loader] is not running or if this service
// is configured with POLL_INTERVAL being so short that [neon-dns-loader]
// hasn't had a chance to handle the previous signal.
if (File.Exists(reloadSignalPath))
{
log.LogWarn("[neon-dns-loader] service doesn't appear to be running because the reload signal file is present.");
}
}
if (!verify && localMD5 == remoteMD5)
{
log.LogDebug(() => "DNS answers are unchanged.");
}
else
{
if (localMD5 == remoteMD5)
{
log.LogDebug(() => "DNS answers have not changed but we're going to verify that we have the correct hosts anyway.");
}
else
{
log.LogDebug(() => "DNS answers have changed.");
}
log.LogDebug(() => "Fetching DNS answers.");
var hostsTxt = await consul.KV.GetStringOrDefault(HiveConst.ConsulDnsHostsKey, terminator.CancellationToken);
if (hostsTxt == null)
{
log.LogWarn(() => "DNS answers do not exist on Consul. Is [neon-dns-mon] functioning properly?");
}
else
{
var marker = "# -------- NEON-DNS --------";
// We have the host entries from Consul. We need to add these onto the
// end [/etc/powserdns/hosts], replacing any host entries written during
// a previous run.
//
// We're going to use the special marker line:
//
// # ---DYNAMIC-HOSTS---
//
// to separate the built-in hosts (above the line) from the dynamic hosts
// we're generating here (which will be below the line). Note that this
// line won't exist the first time this service runs, so we'll just add it.
//
// Note that it's possible that the PowerDNS Recursor might be reading this
// file while we're trying to write it. We're going to treat these as a
// transient errors and retry.
var retry = new LinearRetryPolicy(typeof(IOException), maxAttempts: 5, retryInterval: TimeSpan.FromSeconds(1));
await retry.InvokeAsync(
async() =>
{
using (var stream = new FileStream(powerDnsHostsPath, FileMode.Open, FileAccess.ReadWrite))
{
// Read a copy of the hosts file as bytes so we can compare
// the old version with the new one generated below for changes.
var orgHostBytes = stream.ReadToEnd();
stream.Position = 0;
// Generate the new hosts file.
var sbHosts = new StringBuilder();
// Read the hosts file up to but not including the special marker
// line (if it's present).
using (var reader = new StreamReader(stream, Encoding.UTF8, true, 32 * 1024, leaveOpen: true))
{
foreach (var line in reader.Lines())
{
if (line.StartsWith(marker))
{
break;
}
sbHosts.AppendLine(line);
}
}
// Strip any trailing whitespace from the hosts file so we'll
// be able to leave a nice blank line between the end of the
// original file and the special marker line.
var text = sbHosts.ToString().TrimEnd();
sbHosts.Clear();
sbHosts.AppendLine(text);
// Append the marker line, followed by dynamic host
// entries we downloaded from Consul.
sbHosts.AppendLine();
sbHosts.AppendLine(marker);
sbHosts.AppendLine();
sbHosts.Append(hostsTxt);
// Generate the new host file bytes, taking care to ensure that
// we're using Linux style line endings and then update the
// hosts file if anything changed.
var hostsText = NeonHelper.ToLinuxLineEndings(sbHosts.ToString());
var newHostBytes = Encoding.UTF8.GetBytes(hostsText);
if (NeonHelper.ArrayEquals(orgHostBytes, newHostBytes))
{
log.LogDebug(() => $"[{powerDnsHostsPath}] file is up-to-date.");
}
else
{
log.LogDebug(() => $"[{powerDnsHostsPath}] is being updated.");
stream.Position = 0;
stream.SetLength(0);
stream.Write(newHostBytes);
// Signal to the local [neon-dns-loader] systemd service that it needs
// to have PowerDNS Recursor reload the hosts file.
File.WriteAllText(reloadSignalPath, "reload now");
}
}
log.LogDebug(() => "Finished poll");
await Task.CompletedTask;
});
// We've successfully synchronized the local hosts file with
// the Consul DNS settings.
localMD5 = remoteMD5;
}
}
return(await Task.FromResult(false));
},
onExceptionAsync:
async e =>
{
log.LogError(e);
return(await Task.FromResult(false));
},
onTerminateAsync:
async() =>
{
log.LogInfo(() => "Terminating");
await Task.CompletedTask;
});
terminator.AddDisposable(periodicTask);
await periodicTask.Run();
}
/// <inheritdoc/>
public void LogError(object message, string activityId = null)
{
log.LogError(message, activityId);
capture.AppendLine($"[ERROR] {message}");
}
/// <summary>
/// Application entry point.
/// </summary>
/// <param name="args">Command line arguments.</param>
public static async Task Main(string[] args)
{
LogManager.Default.SetLogLevel(Environment.GetEnvironmentVariable("LOG_LEVEL"));
log = LogManager.Default.GetLogger(typeof(Program));
// Create process terminator to handle termination signals.
terminator = new ProcessTerminator(log);
terminator.AddHandler(
() =>
{
// Cancel any operations in progress.
terminator.CancellationTokenSource.Cancel();
});
// Read the environment variables.
// $hack(jeff.lill:
//
// We're going to scan the Consul configuration key to determine whether this
// instance is managing the public or private proxy (or bridges) so we'll
// be completely compatible with existing deployments.
//
// In theory, we could have passed a new environment variable but that's not
// worth the trouble.
configKey = Environment.GetEnvironmentVariable("CONFIG_KEY");
if (string.IsNullOrEmpty(configKey))
{
log.LogError("[CONFIG_KEY] environment variable is required.");
Program.Exit(1, immediate: true);
}
isPublic = configKey.Contains("/public/");
var proxyName = isPublic ? "public" : "private";
serviceName = $"neon-proxy-{proxyName}:{GitVersion}";
log.LogInfo(() => $"Starting [{serviceName}]");
configHashKey = Environment.GetEnvironmentVariable("CONFIG_HASH_KEY");
if (string.IsNullOrEmpty(configHashKey))
{
log.LogError("[CONFIG_HASH_KEY] environment variable is required.");
Program.Exit(1, immediate: true);
}
vaultCredentialsName = Environment.GetEnvironmentVariable("VAULT_CREDENTIALS");
if (string.IsNullOrEmpty(vaultCredentialsName))
{
log.LogWarn("HTTPS routes are not supported because VAULT_CREDENTIALS is not specified or blank.");
}
var warnSeconds = Environment.GetEnvironmentVariable("WARN_SECONDS");
if (string.IsNullOrEmpty(warnSeconds) || !double.TryParse(warnSeconds, out var warnSecondsValue))
{
warnInterval = TimeSpan.FromSeconds(300);
}
else
{
warnInterval = TimeSpan.FromSeconds(warnSecondsValue);
}
var startSeconds = Environment.GetEnvironmentVariable("START_SECONDS");
if (string.IsNullOrEmpty(startSeconds) || !double.TryParse(startSeconds, out var startSecondsValue))
{
startDelay = TimeSpan.FromSeconds(10);
}
else
{
startDelay = TimeSpan.FromSeconds(startSecondsValue);
}
var maxHAProxyCountString = Environment.GetEnvironmentVariable("MAX_HAPROXY_COUNT");
if (!int.TryParse(maxHAProxyCountString, out maxHAProxyCount))
{
maxHAProxyCount = 10;
}
if (maxHAProxyCount < 0)
{
maxHAProxyCount = 0;
}
debugMode = "true".Equals(Environment.GetEnvironmentVariable("DEBUG"), StringComparison.InvariantCultureIgnoreCase);
log.LogInfo(() => $"LOG_LEVEL={LogManager.Default.LogLevel.ToString().ToUpper()}");
log.LogInfo(() => $"CONFIG_KEY={configKey}");
log.LogInfo(() => $"CONFIG_HASH_KEY={configHashKey}");
log.LogInfo(() => $"VAULT_CREDENTIALS={vaultCredentialsName}");
log.LogInfo(() => $"WARN_SECONDS={warnInterval}");
log.LogInfo(() => $"START_SECONDS={startDelay}");
log.LogInfo(() => $"MAX_HAPROXY_COUNT={maxHAProxyCount}");
log.LogInfo(() => $"DEBUG={debugMode}");
// Ensure that the required directories exist.
Directory.CreateDirectory(tmpfsFolder);
Directory.CreateDirectory(configFolder);
Directory.CreateDirectory(configUpdateFolder);
// Establish the hive connections.
if (NeonHelper.IsDevWorkstation)
{
throw new NotImplementedException("This service works only within a Linux container with HAProxy installed.");
//var vaultCredentialsSecret = "neon-proxy-manager-credentials";
//Environment.SetEnvironmentVariable("VAULT_CREDENTIALS", vaultCredentialsSecret);
//hive = HiveHelper.OpenHiveRemote(new DebugSecrets().VaultAppRole(vaultCredentialsSecret, $"neon-proxy-{proxyName}"));
}
else
{
hive = HiveHelper.OpenHive();
}
try
{
// Log into Vault using the Vault credentials persisted as a Docker
// secret, if one was specified. We won't open Vault otherwise.
if (!string.IsNullOrEmpty(vaultCredentialsName))
{
var vaultSecret = HiveHelper.GetSecret(vaultCredentialsName);
if (string.IsNullOrEmpty(vaultSecret))
{
log.LogCritical($"Cannot read Docker secret [{vaultCredentialsName}].");
Program.Exit(1, immediate: true);
}
var vaultCredentials = HiveCredentials.ParseJson(vaultSecret);
if (vaultCredentials == null)
{
log.LogCritical($"Cannot parse Docker secret [{vaultCredentialsName}].");
Program.Exit(1, immediate: true);
}
log.LogInfo(() => $"Connecting: Vault");
vault = HiveHelper.OpenVault(vaultCredentials);
}
else
{
vault = null;
// $hack(jeff.lill):
//
// This is a bit of backwards compatible hack. Instances started without the
// VAULT_CREDENTIALS environment variable are assumed to be proxy bridges.
isBridge = true;
}
// Open Consul and then start the service tasks.
log.LogInfo(() => $"Connecting: Consul");
using (consul = HiveHelper.OpenConsul())
{
log.LogInfo(() => $"Connecting: {HiveMQChannels.ProxyNotify} channel");
// Verify that the required Consul keys exist or loop to wait until they
// are created. This will allow the service wait for pending hive setup
// operations to be completed.
while (!await consul.KV.Exists(configKey))
{
log.LogWarn(() => $"Waiting for [{configKey}] key to be present in Consul.");
await Task.Delay(TimeSpan.FromSeconds(5));
}
while (!await consul.KV.Exists(configHashKey))
{
log.LogWarn(() => $"Waiting for [{configHashKey}] key to be present in Consul.");
await Task.Delay(TimeSpan.FromSeconds(5));
}
// Crank up the service tasks.
await NeonHelper.WaitAllAsync(
ErrorPollerAsync(),
HAProxShim());
}
}
catch (Exception e)
{
log.LogCritical(e);
Program.Exit(1);
return;
}
finally
{
HiveHelper.CloseHive();
terminator.ReadyToExit();
}
Program.Exit(0);
return;
}
/// <summary>
/// A generic Watcher to watch Kubernetes resources, and respond with a custom (async) callback method.
/// </summary>
/// <param name="actionAsync">The async action called as watch events are received..</param>
/// <param name="namespaceParameter">That target Kubernetes namespace.</param>
/// <param name="fieldSelector">The optional field selector</param>
/// <param name="labelSelector">The optional label selector</param>
/// <param name="resourceVersion">The start resource version.</param>
/// <param name="resourceVersionMatch">The optional resourceVersionMatch setting.</param>
/// <param name="timeoutSeconds">Optional timeout override.</param>
/// <param name="cancellationToken">Optionally specifies a cancellation token.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
public async Task WatchAsync(
Func <WatchEvent <T>, Task> actionAsync,
string namespaceParameter = null,
string fieldSelector = null,
string labelSelector = null,
string resourceVersion = null,
string resourceVersionMatch = null,
int?timeoutSeconds = null,
CancellationToken cancellationToken = default)
{
await SyncContext.Clear;
// Validate the resource version we're being given.
if (!string.IsNullOrEmpty(resourceVersion))
{
await ValidateResourceVersionAsync(
fieldSelector : fieldSelector,
labelSelector : labelSelector,
resourceVersion : resourceVersion,
resourceVersionMatch : resourceVersionMatch,
timeoutSeconds : timeoutSeconds);
}
// Start the loop that handles the async action callbacks.
_ = EventHandlerAsync(actionAsync);
// This is where you'll actually listen for watch events from Kubernetes.
// When you receive an event, do this:
while (true)
{
this.resourceVersion = resourceVersion ?? "0";
while (!string.IsNullOrEmpty(this.resourceVersion))
{
try
{
Task <HttpOperationResponse <object> > listResponse;
if (string.IsNullOrEmpty(namespaceParameter))
{
listResponse = k8s.ListClusterCustomObjectWithHttpMessagesAsync <T>(
allowWatchBookmarks: true,
fieldSelector: fieldSelector,
labelSelector: labelSelector,
resourceVersion: this.resourceVersion,
resourceVersionMatch: resourceVersionMatch,
timeoutSeconds: timeoutSeconds,
watch: true,
cancellationToken: cancellationToken);
}
else
{
listResponse = k8s.ListNamespacedCustomObjectWithHttpMessagesAsync <T>(
namespaceParameter,
allowWatchBookmarks: true,
fieldSelector: fieldSelector,
labelSelector: labelSelector,
resourceVersion: this.resourceVersion,
resourceVersionMatch: resourceVersionMatch,
timeoutSeconds: timeoutSeconds,
cancellationToken: cancellationToken,
watch: true);
}
using (listResponse.Watch(
(WatchEventType type, T item) =>
{
lock (eventQueue)
{
eventQueue.Enqueue(new WatchEvent <T>()
{
Type = type, Value = item
});
eventReady.Set();
}
}))
{
while (true)
{
await Task.Delay(TimeSpan.FromHours(1));
}
}
}
catch (OperationCanceledException)
{
// This is the signal to quit.
return;
}
catch (KubernetesException kubernetesException)
{
logger?.LogError(kubernetesException);
// Deal with this non-recoverable condition "too old resource version"
if (string.Equals(kubernetesException.Status.Reason, "Expired", StringComparison.Ordinal))
{
// force control back to outer loop
this.resourceVersion = null;
}
}
}
}
}