public async Task ExecuteAsync(CancellationToken cancellationToken)
{
await using var handle = await _distributedLockProvider.AcquireLockAsync(nameof (CreateSubscriptions), _elsaOptions.DistributedLockTimeout, cancellationToken);
if (handle == null)
{
throw new Exception("Could not acquire a lock within the maximum amount of time configured");
}
foreach (var messageType in _competingMessageTypes)
{
var bus = await _serviceBusFactory.GetServiceBusAsync(messageType.MessageType, messageType.QueueName, cancellationToken);
await bus.Subscribe(messageType.MessageType);
}
var containerName = _containerNameAccessor.GetContainerName();
foreach (var messageType in _pubSubMessageTypes)
{
var queueName = $"{containerName}:{messageType.QueueName ?? messageType.MessageType.Name}";
var bus = await _serviceBusFactory.GetServiceBusAsync(messageType.MessageType, queueName, cancellationToken);
await bus.Subscribe(messageType.MessageType);
}
}
private async Task <int> ResumeCorrelatedWorkflowsAsync(TriggerWorkflowsRequest request, CancellationToken cancellationToken)
{
var correlationId = request.CorrelationId !;
var lockKey = correlationId;
_logger.LogDebug("Acquiring lock on correlation ID {CorrelationId}", correlationId);
await using var handle = await _distributedLockProvider.AcquireLockAsync(lockKey, _elsaOptions.DistributedLockTimeout, cancellationToken);
if (handle == null)
{
throw new LockAcquisitionException($"Failed to acquire a lock on {lockKey}");
}
var correlatedWorkflowInstanceCount = !string.IsNullOrWhiteSpace(correlationId)
? await _workflowInstanceStore.CountAsync(new CorrelationIdSpecification <WorkflowInstance>(correlationId).WithStatus(WorkflowStatus.Suspended), cancellationToken)
: 0;
_logger.LogDebug("Found {CorrelatedWorkflowCount} correlated workflows,", correlatedWorkflowInstanceCount);
if (correlatedWorkflowInstanceCount > 0)
{
_logger.LogDebug("{WorkflowInstanceCount} existing workflows found with correlation ID '{CorrelationId}' will be queued for execution", correlatedWorkflowInstanceCount, correlationId);
var bookmarkResults = await _bookmarkFinder.FindBookmarksAsync(request.ActivityType, request.Bookmark, request.TenantId, cancellationToken).ToList();
await ResumeWorkflowsAsync(bookmarkResults, request.Input, cancellationToken);
}
return(correlatedWorkflowInstanceCount);
}
private async Task ScheduleJob(ITrigger trigger, CancellationToken cancellationToken)
{
var scheduler = await _schedulerProvider.GetSchedulerAsync(cancellationToken);
var sharedResource = $"{nameof(QuartzWorkflowInstanceScheduler)}:{trigger.Key}";
await using var handle = await _distributedLockProvider.AcquireLockAsync(sharedResource, _elsaOptions.DistributedLockTimeout, cancellationToken);
if (handle == null)
{
return;
}
try
{
var existingTrigger = await scheduler.GetTrigger(trigger.Key, cancellationToken);
// For workflow definitions we only schedule the job if one doesn't exist already because another node may have created it beforehand.
if (existingTrigger == null)
{
await scheduler.ScheduleJob(trigger, cancellationToken);
}
}
catch (SchedulerException e)
{
_logger.LogWarning(e, "Failed to schedule trigger {TriggerKey}", trigger.Key.ToString());
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
using var scope = _serviceScopeFactory.CreateScope();
var job = scope.ServiceProvider.GetRequiredService <CleanupJob>();
while (!stoppingToken.IsCancellationRequested)
{
await Task.Delay(_interval, stoppingToken);
await using var handle = await _distributedLockProvider.AcquireLockAsync(nameof (CleanupService), cancellationToken : stoppingToken);
if (handle == null)
{
continue;
}
try
{
await job.ExecuteAsync(stoppingToken);
}
catch (Exception e)
{
_logger.LogError(e, "Failed to perform cleanup this time around. Next cleanup attempt will happen in {Interval}", _interval);
}
}
}
public async Task <RunWorkflowResult> ExecuteAsync(string workflowInstanceId, string?activityId, WorkflowInput?input = default, CancellationToken cancellationToken = default)
{
var workflowInstanceLockKey = $"workflow-instance:{workflowInstanceId}";
await using var workflowInstanceLockHandle = await _distributedLockProvider.AcquireLockAsync(workflowInstanceLockKey, _elsaOptions.DistributedLockTimeout, cancellationToken);
if (workflowInstanceLockHandle == null)
{
throw new LockAcquisitionException("Could not acquire a lock within the configured amount of time");
}
var workflowInstance = await _workflowInstanceStore.FindByIdAsync(workflowInstanceId, cancellationToken);
if (workflowInstance == null)
{
_logger.LogWarning("Could not run workflow instance with ID {WorkflowInstanceId} because it does not exist", workflowInstanceId);
return(new RunWorkflowResult(workflowInstance, activityId, false));
}
var correlationId = workflowInstance.CorrelationId;
if (!string.IsNullOrWhiteSpace(correlationId))
{
// We need to lock on correlation ID to prevent a race condition with WorkflowLaunchpad that is used to find workflows by correlation ID to execute.
// The race condition is: when a workflow instance is done executing, the BookmarkIndexer will collect bookmarks.
// But if in the meantime an event comes in that triggers correlated workflows, the bookmarks may not have been created yet.
await using var correlationLockHandle = await _distributedLockProvider.AcquireLockAsync(correlationId, _elsaOptions.DistributedLockTimeout, cancellationToken);
if (correlationLockHandle == null)
{
throw new LockAcquisitionException($"Could not acquire a lock on correlation {correlationId} within the configured amount of time");
}
var result = await _workflowInstanceExecutor.ExecuteAsync(workflowInstance, activityId, input, cancellationToken);
return(result);
}
else
{
var result = await _workflowInstanceExecutor.ExecuteAsync(workflowInstance, activityId, input, cancellationToken);
return(result);
}
}
public async Task ExecuteAsync(CancellationToken stoppingToken)
{
await using var handle = await _distributedLockProvider.AcquireLockAsync(nameof (StartJobs), null, stoppingToken);
if (handle == null)
{
return;
}
await _retryPolicy.ExecuteAsync(async() => await ExecuteInternalAsync(stoppingToken));
}
private async Task <IDistributedSynchronizationHandle> AcquireLockAsync(string resource, CancellationToken cancellationToken)
{
var handle = await _distributedLockProvider.AcquireLockAsync(resource, _elsaOptions.DistributedLockTimeout, cancellationToken);
if (handle == null)
{
throw new LockAcquisitionException($"Failed to acquire a lock on {resource}");
}
return(handle);
}
public async Task <DeleteWorkflowInstanceResult> DeleteAsync(string workflowInstanceId, CancellationToken cancellationToken = default)
{
var workflowInstanceLockKey = $"workflow-instance:{workflowInstanceId}";
await using var workflowInstanceLockHandle = await _distributedLockProvider.AcquireLockAsync(workflowInstanceLockKey, _elsaOptions.DistributedLockTimeout, cancellationToken);
if (workflowInstanceLockHandle == null)
{
throw new LockAcquisitionException("Could not acquire a lock within the configured amount of time");
}
return(await _workflowInstanceDeleter.DeleteAsync(workflowInstanceId, cancellationToken));
}
public async Task ExecuteAsync(CancellationToken stoppingToken)
{
await using var handle = await _distributedLockProvider.AcquireLockAsync(nameof (StartJobs), null, stoppingToken);
if (handle == null)
{
return;
}
await ScheduleTimerEventWorkflowsAsync(stoppingToken);
await ScheduleCronEventWorkflowsAsync(stoppingToken);
await ScheduleStartAtWorkflowsAsync(stoppingToken);
}
public async Task ExecuteAsync(CancellationToken cancellationToken = default)
{
var lockKey = GetType().Name;
await using var handle = await _distributedLockProvider.AcquireLockAsync(lockKey, Duration.FromSeconds(10), cancellationToken);
if (handle == null)
{
return;
}
var instances = await _workflowInstanceStore.FindManyAsync(new WorkflowStatusSpecification(WorkflowStatus.Running), cancellationToken : cancellationToken).ToList();
if (instances.Any())
{
_logger.LogInformation("Found {WorkflowInstanceCount} workflows with status 'Running'. Resuming each one of them", instances.Count);
}
else
{
_logger.LogInformation("Found no workflows with status 'Running'. Nothing to resume");
}
foreach (var instance in instances)
{
_logger.LogInformation("Resuming {WorkflowInstanceId}", instance.Id);
var scheduledActivities = instance.ScheduledActivities;
if (instance.CurrentActivity == null && !scheduledActivities.Any())
{
if (instance.BlockingActivities.Any())
{
_logger.LogWarning(
"Workflow '{WorkflowInstanceId}' was in the Running state, but has no scheduled activities not has a currently executing one. However, it does have blocking activities, so switching to Suspended status",
instance.Id);
instance.WorkflowStatus = WorkflowStatus.Suspended;
await _workflowInstanceStore.SaveAsync(instance, cancellationToken);
continue;
}
_logger.LogWarning("Workflow '{WorkflowInstanceId}' was in the Running state, but has no scheduled activities nor has a currently executing one", instance.Id);
continue;
}
var scheduledActivity = instance.CurrentActivity ?? instance.ScheduledActivities.Peek();
await _workflowInstanceDispatcher.DispatchAsync(new ExecuteWorkflowInstanceRequest(instance.Id, scheduledActivity.ActivityId, scheduledActivity.Input), cancellationToken);
}
}
/// <inheritdoc/>
public async Task <bool> CanIdentifyAsync(int shardId, CancellationToken token)
{
token.ThrowIfCancellationRequested();
try
{
await cache.AcquireLockAsync(GetCacheKey(shardId), token);
}
catch (OperationCanceledException)
{
return(false);
}
await cache.ExpiresAsync(GetCacheKey(shardId), TimeSpan.FromSeconds(5));
return(true);
}
public async Task ExecuteAsync(CancellationToken cancellationToken = default)
{
await using var handle = await _distributedLockProvider.AcquireLockAsync(nameof (PurgeSubscriptions), _elsaOptions.DistributedLockTimeout, cancellationToken);
if (handle == null)
{
throw new Exception("Could not acquire a lock within the maximum amount of time configured");
}
var messageTypes = _elsaOptions.PubSubMessageTypes;
var convention = new DefaultAzureServiceBusTopicNameConvention();
var topics = messageTypes.Select(x => convention.GetTopic(x.MessageType)).Distinct();
var client = _managementClient;
foreach (var topic in topics)
{
var subscriptions = client.GetSubscriptionsAsync(topic);
await foreach (var subscription in subscriptions)
{
try
{
await client.GetQueueAsync(subscription.SubscriptionName);
_logger.LogInformation("Subscription {Name} still active", subscription.SubscriptionName);
}
catch (ServiceBusException e) when(e.Reason ==
ServiceBusFailureReason.MessagingEntityNotFound)
{
_logger.LogInformation("Found stale subscription {Name}, deleting...", subscription.SubscriptionName);
try
{
await client.DeleteSubscriptionAsync(topic, subscription.SubscriptionName);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete subscription, ignoring...");
}
}
}
}
}
public async Task <RunWorkflowResult> ExecuteAsync(string workflowInstanceId, string?activityId, object?input = default, CancellationToken cancellationToken = default)
{
var key = $"workflow-instance:{workflowInstanceId}";
_logger.LogDebug("Acquiring lock on {LockKey}", key);
await using var handle = await _distributedLockProvider.AcquireLockAsync(key, _elsaOptions.DistributedLockTimeout, cancellationToken);
if (handle == null)
{
throw new LockAcquisitionException("Could not acquire a lock within the configured amount of time");
}
_logger.LogDebug("Lock acquired on {LockKey}", key);
var result = await _workflowInstanceExecutor.ExecuteAsync(workflowInstanceId, activityId, input, cancellationToken);
await handle.DisposeAsync();
_logger.LogDebug("Released lock on {LockKey}", key);
return(result);
}
public async Task <WorkflowInstance> RunWorkflowAsync(
IWorkflowBlueprint workflowBlueprint,
WorkflowInstance workflowInstance,
string?activityId = default,
object?input = default,
CancellationToken cancellationToken = default)
{
var key = $"locking-workflow-runner:{workflowInstance.Id}";
await using var handle = await _distributedLockProvider.AcquireLockAsync(key, _elsaOptions.DistributedLockTimeout, cancellationToken);
if (handle == null)
{
throw new LockAcquisitionException("Could not acquire a lock within the configured amount of time");
}
workflowInstance = await _workflowRunner.RunWorkflowAsync(workflowBlueprint, workflowInstance, activityId, input, cancellationToken);
await handle.DisposeAsync();
return(workflowInstance);
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
await using var handle = await _distributedLockProvider.AcquireLockAsync(nameof (CreateSubscriptions), _elsaOptions.DistributedLockTimeout, cancellationToken);
if (handle == null)
{
throw new Exception("Could not acquire a lock within the maximum amount of time configured");
}
var competingMessageTypeGroups = _competingMessageTypes.GroupBy(x => x.QueueName);
foreach (var messageTypeGroup in competingMessageTypeGroups)
{
var queueName = messageTypeGroup.Key;
var messageTypes = messageTypeGroup.Select(x => x.MessageType).ToList();
var bus = _serviceBusFactory.ConfigureServiceBus(messageTypes, queueName);
foreach (var messageType in messageTypes)
{
await bus.Subscribe(messageType);
}
}
var containerName = _containerNameAccessor.GetContainerName();
var pubSubMessageTypeGroups = _pubSubMessageTypes.GroupBy(x => x.QueueName);
foreach (var messageTypeGroup in pubSubMessageTypeGroups)
{
var queueName = $"{containerName}:{messageTypeGroup.Key}";
var messageTypes = messageTypeGroup.Select(x => x.MessageType).ToList();
var bus = _serviceBusFactory.ConfigureServiceBus(messageTypes, queueName);
foreach (var messageType in messageTypes)
{
await bus.Subscribe(messageType);
}
}
}
public async Task <StartableWorkflow?> CollectStartableWorkflowAsync(IWorkflowBlueprint workflowBlueprint, string?activityId, string?correlationId = default, string?contextId = default, string?tenantId = default, CancellationToken cancellationToken = default)
{
var workflowDefinitionId = workflowBlueprint.Id;
if (!ValidatePreconditions(workflowDefinitionId, workflowBlueprint))
{
return(null);
}
var correlationLockHandle = default(IDistributedSynchronizationHandle?);
// If we are creating a correlated workflow, make sure to acquire a lock on it to prevent duplicate workflow instances from being created.
if (!string.IsNullOrWhiteSpace(correlationId))
{
_logger.LogDebug("Acquiring lock on correlation ID {CorrelationId}", correlationId);
correlationLockHandle = await _distributedLockProvider.AcquireLockAsync(correlationId, _elsaOptions.DistributedLockTimeout, cancellationToken);
if (correlationLockHandle == null)
{
throw new LockAcquisitionException($"Failed to acquire a lock on correlation ID {correlationId}");
}
}
try
{
// Acquire a lock on the workflow definition so that we can ensure singleton-workflows never execute more than one instance.
var lockKey = $"execute-workflow-definition:tenant:{tenantId}:workflow-definition:{workflowDefinitionId}";
await using var handle = await _distributedLockProvider.AcquireLockAsync(lockKey, _elsaOptions.DistributedLockTimeout, cancellationToken);
if (handle == null)
{
throw new LockAcquisitionException($"Failed to acquire a lock on {lockKey}");
}
if (!workflowBlueprint !.IsSingleton || await GetWorkflowIsAlreadyExecutingAsync(tenantId, workflowDefinitionId) == false)
{
var startActivities = _getsStartActivities.GetStartActivities(workflowBlueprint).Select(x => x.Id).ToHashSet();
var startActivityId = activityId == null?startActivities.FirstOrDefault() : startActivities.Contains(activityId) ? activityId : default;
if (startActivityId == null)
{
_logger.LogWarning("Cannot start workflow {WorkflowDefinitionId} with version {WorkflowDefinitionVersion} because it has no starting activities", workflowBlueprint.Id, workflowBlueprint.Version);
return(null);
}
var workflowInstance = await _workflowFactory.InstantiateAsync(
workflowBlueprint,
correlationId,
contextId,
cancellationToken);
await _workflowInstanceStore.SaveAsync(workflowInstance, cancellationToken);
return(new StartableWorkflow(workflowBlueprint, workflowInstance, startActivityId));
}
}
finally
{
if (correlationLockHandle != null)
{
await correlationLockHandle.DisposeAsync();
}
}
return(null);
}