/// <summary>
/// Returns the <see cref="V1OwnerReference"/> for the host node.
/// </summary>
/// <param name="k8s">The Kubernetes client to be used to query for the node information.</param>
/// <returns>
/// The <see cref="V1OwnerReference"/> for the node or <c>null</c> when this couldn't
/// be determined.
/// </returns>
public static async Task <V1OwnerReference> GetOwnerReferenceAsync(IKubernetes k8s)
{
Covenant.Requires <ArgumentNullException>(k8s != null, nameof(k8s));
if (NeonHelper.IsLinux)
{
using (await mutex.AcquireAsync())
{
// Return any cached information.
if (cachedOwnerReference != null)
{
return(cachedOwnerReference);
}
// Query Kubernetes for the node information based on the the node's hostname.
cachedNode = await k8s.ReadNodeAsync(Name);
cachedOwnerReference = new V1OwnerReference(apiVersion: cachedNode.ApiVersion, name: cachedNode.Name(), kind: cachedNode.Kind, uid: cachedNode.Uid());
return(cachedOwnerReference);
}
}
else
{
// Emulate without an owner reference.
return(null);
}
}
public async Task Dispose()
{
// Create a mutex, acquire it, and then create another task that will
// attempt to acquire it as well (and will fail because the mutex has
// already been acquired). Then dispose the mutex and verify that the
// waiting task saw the [ObjectDisposedException].
var mutex = new AsyncMutex();
var inTask = false;
var acquired = false;
var disposed = false;
await mutex.AcquireAsync();
var task = Task.Run(
async() =>
{
try
{
var acquireTask = mutex.AcquireAsync();
inTask = true;
await acquireTask;
acquired = true;
}
catch (ObjectDisposedException)
{
disposed = true;
}
});
// Wait for the task to have called [AcquireAsync()].
NeonHelper.WaitFor(() => inTask, defaultTimeout);
// Dispose the mutex, wait for the task to exit and then verify
// that it caught the [ObjectDisposedException].
mutex.Dispose();
task.Wait(defaultTimeout);
Assert.False(acquired);
Assert.True(disposed);
}
/// <summary>
/// Starts or restarts the handler listening for the [ProxyUpdateMessage] messages.
/// </summary>
private static void StartNotifyHandler()
{
lock (syncLock)
{
// Use the latest settings to reconnect to the [proxy-notify] channel.
if (proxyNotifyChannel != null)
{
proxyNotifyChannel.Dispose();
}
proxyNotifyChannel = hive.HiveMQ.Internal.GetProxyNotifyChannel(useBootstrap: true).Open();
// Register a handler for [ProxyUpdateMessage] messages that determines
// whether the message is meant for this service instance and handle it.
proxyNotifyChannel.ConsumeAsync <ProxyUpdateMessage>(
async message =>
{
// We cannot process updates in parallel so we'll use an
// AsyncMutex to prevent this.
using (await asyncLock.AcquireAsync())
{
var forThisInstance = false;
if (isPublic)
{
forThisInstance = message.PublicProxy && !isBridge ||
message.PublicBridge && isBridge;
}
else
{
forThisInstance = message.PrivateProxy && !isBridge ||
message.PrivateBridge && isBridge;
}
if (!forThisInstance)
{
log.LogInfo(() => $"HAPROXY-SHIM: Received but ignorning: {message}");
return;
}
log.LogInfo(() => $"HAPROXY-SHIM: Received: {message}");
var jitter = NeonHelper.RandTimespan(HiveConst.MaxJitter);
log.LogDebug(() => $"HAPROXY-SHIM: Jitter delay [{jitter}].");
await Task.Delay(jitter);
await ConfigureHAProxy();
}
});
}
}
public async Task Basic()
{
// Create a mutex and then several tasks that acquire the mutex for
// a period of time, verifying that each obtains exclusive
// access.
var taskCount = 12;
var refCount = 0;
var error = false;
var tasks = new List <Task>();
var stopwatch = new Stopwatch();
var testTime = defaultTimeout - TimeSpan.FromSeconds(2);
stopwatch.Start();
using (var mutex = new AsyncMutex())
{
for (int i = 0; i < taskCount; i++)
{
tasks.Add(Task.Run(
async() =>
{
while (stopwatch.Elapsed < testTime)
{
using (await mutex.AcquireAsync())
{
if (refCount > 0)
{
// This means that we don't have exclusive access indicating
// that the mutex must be broken.
error = true;
}
try
{
Interlocked.Increment(ref refCount);
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
finally
{
Interlocked.Decrement(ref refCount);
}
}
}
}));
await NeonHelper.WaitAllAsync(tasks, defaultTimeout);
}
Assert.False(error);
}
}
/// <summary>
/// Signals Cadence that the application is capable of executing workflows and/or activities for a specific
/// domain and task list.
/// </summary>
/// <param name="taskList">Specifies the task list implemented by the worker. This must not be empty.</param>
/// <param name="options">Optionally specifies additional worker options.</param>
/// <param name="domain">Optionally overrides the default <see cref="CadenceClient"/> domain.</param>
/// <returns>A <see cref="Worker"/> identifying the worker instance.</returns>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="taskList"/> is <c>null</c> or empty.</exception>
/// <exception cref="InvalidOperationException">
/// Thrown when an attempt is made to recreate a worker with the
/// same properties on a given client. See the note in the remarks.
/// </exception>
/// <remarks>
/// <note>
/// <see cref="CadenceClient"/> for more information on task lists.
/// </note>
/// <para>
/// Your workflow application will need to call this method so that Cadence will know
/// that it can schedule activities to run within the current process. You'll need
/// to specify the target Cadence domain and task list.
/// </para>
/// <para>
/// You may also specify an optional <see cref="WorkerOptions"/> parameter as well
/// as customize the name used to register the activity, which defaults to the
/// fully qualified name of the activity type.
/// </para>
/// <para>
/// This method returns a <see cref="Worker"/> which implements <see cref="IDisposable"/>.
/// It's a best practice to call <see cref="Dispose()"/> just before the a worker process
/// terminates, but this is optional. Advanced worker implementation that need to change
/// their configuration over time can also call <see cref="Dispose()"/> to stop workers
/// for specific domains and task lists.
/// </para>
/// <note>
/// The Cadence GOLANG client does not appear to support starting a worker with a given
/// set of parameters, stopping that workflow, and then restarting another worker
/// with the same parameters on the same client. This method detects this situation
/// and throws an <see cref="InvalidOperationException"/> when a restart is attempted.
/// </note>
/// </remarks>
public async Task <Worker> StartWorkerAsync(string taskList, WorkerOptions options = null, string domain = null)
{
await SyncContext.ClearAsync;
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(taskList), nameof(taskList), "Workers must be started with a non-empty workflow.");
EnsureNotDisposed();
options = options ?? new WorkerOptions();
domain = ResolveDomain(domain);
WorkerMode mode = options.Mode;
Worker worker;
try
{
using (await workerRegistrationMutex.AcquireAsync())
{
// Ensure that we haven't already registered a worker for the
// specified activity, domain, and task list. We'll just increment
// the reference count for the existing worker and return it
// in this case.
//
// I know that this is a linear search but the number of activity
// registrations per service will generally be very small and
// registrations will happen infrequently (typically just once
// per service, when it starts).
// $note(jefflill):
//
// If the worker exists but its RefCount==0, then we're going to
// throw an exception because Cadence doesn't support recreating
// a worker with the same parameters on the same client.
worker = workers.Values.SingleOrDefault(wf => wf.Mode == mode && wf.Domain == domain && wf.Tasklist == taskList);
if (worker != null)
{
if (worker.RefCount < 0)
{
throw new InvalidOperationException("A worker with these same parameters has already been started and stopped on this Cadence client. Cadence does not support recreating workers for a given client instance.");
}
Interlocked.Increment(ref worker.RefCount);
return(worker);
}
options = options ?? new WorkerOptions();
var reply = (NewWorkerReply)(await CallProxyAsync(
new NewWorkerRequest()
{
Domain = ResolveDomain(domain),
TaskList = taskList,
Options = options.ToInternal()
}));
reply.ThrowOnError();
worker = new Worker(this, mode, reply.WorkerId, domain, taskList);
workers.Add(reply.WorkerId, worker);
}
}
finally
{
switch (mode)
{
case WorkerMode.Activity:
activityWorkerStarted = true;
break;
case WorkerMode.Workflow:
workflowWorkerStarted = true;
break;
case WorkerMode.Both:
activityWorkerStarted = true;
workflowWorkerStarted = true;
break;
default:
throw new NotImplementedException();
}
}
// Fetch the stub for each registered workflow and activity type so that
// they'll be precompiled so compilation won't impact workflow and activity
// performance including potentially intruducing enough delay to cause
// decision tasks or activity heartbeats to fail (in very rare situations).
//
// Note that the compiled stubs are cached, so we don't need to worry
// about compiling stubs for types more than once causing a problem.
lock (registeredWorkflowTypes)
{
foreach (var workflowInterface in registeredWorkflowTypes)
{
// Workflows, we're going to compile both the external and child
// versions of the stubs.
StubManager.GetWorkflowStub(workflowInterface, isChild: false);
StubManager.GetWorkflowStub(workflowInterface, isChild: true);
}
}
lock (registeredActivityTypes)
{
foreach (var activityInterface in registeredActivityTypes)
{
StubManager.GetActivityStub(activityInterface);
}
}
return(worker);
}
/// <summary>
/// Handles requests to the emulated <b>cadence-proxy</b> root <b>"/"</b> endpoint path.
/// </summary>
/// <param name="context">The request context.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task OnEmulatedRootRequestAsync(HttpContext context)
{
var request = context.Request;
var response = context.Response;
var proxyMessage = ProxyMessage.Deserialize <ProxyMessage>(request.Body);
if (EmulatedLibraryClient == null && proxyMessage.Type != InternalMessageTypes.InitializeRequest)
{
response.StatusCode = StatusCodes.Status400BadRequest;
await response.WriteAsync($"Unexpected Message: Waiting for an [{nameof(InitializeRequest)}] message to specify the [cadence-client] network endpoint.");
return;
}
// Handle proxy reply messages by completing any pending [CallClientAsync()] operation.
var reply = proxyMessage as ProxyReply;
if (reply != null)
{
Operation operation;
using (await emulationMutex.AcquireAsync())
{
emulatedOperations.TryGetValue(reply.RequestId, out operation);
}
if (operation != null)
{
if (reply.Type != operation.Request.ReplyType)
{
response.StatusCode = StatusCodes.Status400BadRequest;
await response.WriteAsync($"[cadence-emulation] has a request [type={operation.Request.Type}, requestId={operation.RequestId}] pending but reply [type={reply.Type}] is not valid and will be ignored.");
}
else
{
operation.SetReply(reply);
response.StatusCode = StatusCodes.Status200OK;
}
}
else
{
log.LogWarn(() => $"[cadence-emulation] reply [type={reply.Type}, requestId={reply.RequestId}] does not map to a pending operation and will be ignored.");
response.StatusCode = StatusCodes.Status400BadRequest;
await response.WriteAsync($"[cadence-emulation] does not have a pending operation with [requestId={reply.RequestId}].");
}
return;
}
// Handle proxy request messages.
switch (proxyMessage.Type)
{
//-------------------------------------------------------------
// Client messages
case InternalMessageTypes.CancelRequest:
await OnEmulatedCancelRequestAsync((CancelRequest)proxyMessage);
break;
case InternalMessageTypes.DomainDescribeRequest:
await OnEmulatedDomainDescribeRequestAsync((DomainDescribeRequest)proxyMessage);
break;
case InternalMessageTypes.DomainRegisterRequest:
await OnEmulatedDomainRegisterRequestAsync((DomainRegisterRequest)proxyMessage);
break;
case InternalMessageTypes.DomainUpdateRequest:
await OnEmulatedDomainUpdateRequestAsync((DomainUpdateRequest)proxyMessage);
break;
case InternalMessageTypes.HeartbeatRequest:
await OnEmulatedHeartbeatRequestAsync((HeartbeatRequest)proxyMessage);
break;
case InternalMessageTypes.InitializeRequest:
await OnEmulatedInitializeRequestAsync((InitializeRequest)proxyMessage);
break;
case InternalMessageTypes.ConnectRequest:
await OnEmulatedConnectRequestAsync((ConnectRequest)proxyMessage);
break;
case InternalMessageTypes.TerminateRequest:
await OnEmulatedTerminateRequestAsync((TerminateRequest)proxyMessage);
break;
case InternalMessageTypes.NewWorkerRequest:
await OnEmulatedNewWorkerRequestAsync((NewWorkerRequest)proxyMessage);
break;
case InternalMessageTypes.StopWorkerRequest:
await OnEmulatedStopWorkerRequestAsync((StopWorkerRequest)proxyMessage);
break;
case InternalMessageTypes.PingRequest:
await OnEmulatedPingRequestAsync((PingRequest)proxyMessage);
break;
//-------------------------------------------------------------
// Workflow messages
case InternalMessageTypes.WorkflowExecuteRequest:
await OnEmulatedWorkflowExecuteRequestAsync((WorkflowExecuteRequest)proxyMessage);
break;
case InternalMessageTypes.WorkflowRegisterRequest:
await OnEmulatedWorkflowRegisterRequestAsync((WorkflowRegisterRequest)proxyMessage);
break;
case InternalMessageTypes.WorkflowSetCacheSizeRequest:
await OnEmulatedWorkflowSetCacheSizeRequestAsync((WorkflowSetCacheSizeRequest)proxyMessage);
break;
case InternalMessageTypes.WorkflowGetResultRequest:
await OnEmulatedWorkflowGetResultRequestAsync((WorkflowGetResultRequest)proxyMessage);
break;
//-------------------------------------------------------------
default:
response.StatusCode = StatusCodes.Status400BadRequest;
await response.WriteAsync($"EMULATION: Message [{proxyMessage.Type}] is not supported.");
break;
}
await Task.CompletedTask;
}
/// <summary>
/// Signals Cadence that the application is capable of executing activities for a specific
/// domain and task list.
/// </summary>
/// <param name="taskList">Optionally specifies the target task list (defaults to <b>"default"</b>).</param>
/// <param name="options">Optionally specifies additional worker options.</param>
/// <param name="domain">Optionally overrides the default <see cref="CadenceClient"/> domain.</param>
/// <returns>A <see cref="Worker"/> identifying the worker instance.</returns>
/// <exception cref="InvalidOperationException">
/// Thrown when an attempt is made to recreate a worker with the
/// same properties on a given client. See the note in the remarks.
/// </exception>
/// <remarks>
/// <para>
/// Your workflow application will need to call this method so that Cadence will know
/// that it can schedule activities to run within the current process. You'll need
/// to specify the target Cadence domain and task list.
/// </para>
/// <para>
/// You may also specify an optional <see cref="WorkerOptions"/> parameter as well
/// as customize the name used to register the activity, which defaults to the
/// fully qualified name of the activity type.
/// </para>
/// <para>
/// This method returns a <see cref="Worker"/> which implements <see cref="IDisposable"/>.
/// It's a best practice to call <see cref="Dispose()"/> just before the a worker process
/// terminates, but this is optional. Advanced worker implementation that need to change
/// their configuration over time can also call <see cref="Dispose()"/> to stop workers
/// for specific domains and task lists.
/// </para>
/// <note>
/// The Cadence GOLANG client does not appear to support starting a worker with a given
/// set of parameters, stopping that workflow, and then restarting another worker
/// with the same parameters on the same client. This method detects this situation
/// and throws an <see cref="InvalidOperationException"/> when these restart attempts
/// are made.
/// </note>
/// </remarks>
private async Task <Worker> StartWorkerAsync(string taskList = "default", WorkerOptions options = null, string domain = null)
{
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(taskList));
options = options ?? new WorkerOptions();
WorkerMode mode = options.Mode;
Worker worker;
try
{
using (await workerRegistrationMutex.AcquireAsync())
{
// Ensure that we haven't already registered a worker for the
// specified activity, domain, and task list. We'll just increment
// the reference count for the existing worker and return it
// in this case.
//
// I know that this is a linear search but the number of activity
// registrations per service will generally be very small and
// registrations will happen infrequently (typically just once
// per service, when it starts).
// $note(jeff.lill):
//
// If the worker exists but its refcount==0, then we're going to
// throw an exception because Cadence doesn't support recreating
// a worker with the same parameters on the same client.
worker = workers.Values.SingleOrDefault(wf => wf.Mode == mode && wf.Domain == domain && wf.Tasklist == taskList);
if (worker != null)
{
if (worker.RefCount == 0)
{
throw new InvalidOperationException("A worker with these same parameters has already been started and stopped on this Cadence client. Cadence does not support recreating workers for a given client instance.");
}
Interlocked.Increment(ref worker.RefCount);
return(worker);
}
options = options ?? new WorkerOptions();
var reply = (NewWorkerReply)(await CallProxyAsync(
new NewWorkerRequest()
{
Domain = ResolveDomain(domain),
TaskList = taskList,
Options = options.ToInternal()
}));
reply.ThrowOnError();
worker = new Worker(this, mode, reply.WorkerId, domain, taskList);
workers.Add(reply.WorkerId, worker);
}
}
finally
{
switch (mode)
{
case WorkerMode.Activity:
activityWorkerStarted = true;
break;
case WorkerMode.Workflow:
workflowWorkerStarted = true;
break;
case WorkerMode.Both:
activityWorkerStarted = true;
workflowWorkerStarted = true;
break;
default:
throw new NotImplementedException();
}
}
return(worker);
}