private async Task <bool> SeedSubscription(Event evt, EventSubscription sub, List <string> toQueue, CancellationToken cancellationToken)
{
foreach (var eventId in await _eventRepository.GetEvents(sub.EventName, sub.EventKey, sub.SubscribeAsOf))
{
if (eventId == evt.Id)
{
continue;
}
var siblingEvent = await _eventRepository.GetEvent(eventId);
if ((!siblingEvent.IsProcessed) && (siblingEvent.EventTime < evt.EventTime))
{
await QueueProvider.QueueWork(eventId, QueueType.Event);
return(false);
}
if (!siblingEvent.IsProcessed)
{
toQueue.Add(siblingEvent.Id);
}
}
if (!await _lockProvider.AcquireLock(sub.WorkflowId, cancellationToken))
{
Logger.LogInformation("Workflow locked {0}", sub.WorkflowId);
return(false);
}
try
{
var workflow = await _workflowRepository.GetWorkflowInstance(sub.WorkflowId);
var pointers = workflow.ExecutionPointers.Where(p => p.EventName == sub.EventName && p.EventKey == sub.EventKey && !p.EventPublished && p.EndTime == null);
foreach (var p in pointers)
{
p.EventData = evt.EventData;
p.EventPublished = true;
p.Active = true;
}
workflow.NextExecution = 0;
await _workflowRepository.PersistWorkflow(workflow);
await _subscriptionRepository.TerminateSubscription(sub.Id);
return(true);
}
catch (Exception ex)
{
Logger.LogError(ex, ex.Message);
return(false);
}
finally
{
await _lockProvider.ReleaseLock(sub.WorkflowId);
await QueueProvider.QueueWork(sub.WorkflowId, QueueType.Workflow);
}
}
public async Task <bool> ResumeWorkflow(string workflowId)
{
if (await LockProvider.AcquireLock(workflowId, new CancellationToken()))
{
bool requeue = false;
try
{
var wf = await PersistenceStore.GetWorkflowInstance(workflowId);
if (wf.Status == WorkflowStatus.Suspended)
{
wf.Status = WorkflowStatus.Runnable;
await PersistenceStore.PersistWorkflow(wf);
requeue = true;
return(true);
}
return(false);
}
finally
{
await LockProvider.ReleaseLock(workflowId);
if (requeue)
{
await QueueProvider.QueueWork(workflowId, QueueType.Workflow);
}
}
}
return(false);
}
public async Task PublishEvent(string eventName, string eventKey, object eventData, DateTime?effectiveDate = null)
{
if (_shutdown)
{
throw new Exception("Host is not running");
}
Logger.LogDebug("Creating event {0} {1}", eventName, eventKey);
Event evt = new Event();
if (effectiveDate.HasValue)
{
evt.EventTime = effectiveDate.Value.ToUniversalTime();
}
else
{
evt.EventTime = DateTime.Now.ToUniversalTime();
}
evt.EventData = eventData;
evt.EventKey = eventKey;
evt.EventName = eventName;
evt.IsProcessed = false;
string eventId = await PersistenceStore.CreateEvent(evt);
await QueueProvider.QueueWork(eventId, QueueType.Event);
}
protected override async Task ProcessItem(string itemId, CancellationToken cancellationToken)
{
if (!await _lockProvider.AcquireLock(itemId, cancellationToken))
{
Logger.LogInformation("Workflow locked {0}", itemId);
return;
}
WorkflowInstance workflow = null;
WorkflowExecutorResult result = null;
try
{
cancellationToken.ThrowIfCancellationRequested();
workflow = await _persistenceStore.GetWorkflowInstance(itemId, cancellationToken);
if (workflow.Status == WorkflowStatus.Runnable)
{
try
{
result = await _executor.Execute(workflow, cancellationToken);
}
finally
{
await _persistenceStore.PersistWorkflow(workflow, cancellationToken);
await QueueProvider.QueueWork(itemId, QueueType.Index);
_greylist.Remove($"wf:{itemId}");
}
}
}
finally
{
await _lockProvider.ReleaseLock(itemId);
if ((workflow != null) && (result != null))
{
foreach (var sub in result.Subscriptions)
{
await SubscribeEvent(sub, _persistenceStore, cancellationToken);
}
await _persistenceStore.PersistErrors(result.Errors, cancellationToken);
var readAheadTicks = _datetimeProvider.UtcNow.Add(Options.PollInterval).Ticks;
if ((workflow.Status == WorkflowStatus.Runnable) && workflow.NextExecution.HasValue && workflow.NextExecution.Value < readAheadTicks)
{
new Task(() => FutureQueue(workflow, cancellationToken)).Start();
}
}
}
}
protected override async Task ProcessItem(string itemId, CancellationToken cancellationToken)
{
try
{
var workflow = await FetchWorkflow(itemId);
WorkflowActivity.Enrich(workflow, "index");
await _searchIndex.IndexWorkflow(workflow);
lock (_errorCounts)
{
_errorCounts.Remove(itemId);
}
}
catch (Exception e)
{
Logger.LogWarning(default(EventId), $"Error indexing workfow - {itemId} - {e.Message}");
var errCount = 0;
lock (_errorCounts)
{
if (!_errorCounts.ContainsKey(itemId))
{
_errorCounts.Add(itemId, 0);
}
_errorCounts[itemId]++;
errCount = _errorCounts[itemId];
}
if (errCount < 5)
{
await QueueProvider.QueueWork(itemId, Queue);
return;
}
if (errCount < 20)
{
await Task.Delay(TimeSpan.FromSeconds(10));
await QueueProvider.QueueWork(itemId, Queue);
return;
}
lock (_errorCounts)
{
_errorCounts.Remove(itemId);
}
Logger.LogError(default(EventId), e, $"Unable to index workfow - {itemId} - {e.Message}");
}
}
private async Task SubscribeEvent(EventSubscription subscription, IPersistenceProvider persistenceStore, CancellationToken cancellationToken)
{
//TODO: move to own class
Logger.LogDebug("Subscribing to event {0} {1} for workflow {2} step {3}", subscription.EventName, subscription.EventKey, subscription.WorkflowId, subscription.StepId);
await persistenceStore.CreateEventSubscription(subscription, cancellationToken);
if (subscription.EventName != Event.EventTypeActivity)
{
var events = await persistenceStore.GetEvents(subscription.EventName, subscription.EventKey, subscription.SubscribeAsOf, cancellationToken);
foreach (var evt in events)
{
var eventKey = $"evt:{evt}";
bool acquiredLock = false;
try
{
acquiredLock = await _lockProvider.AcquireLock(eventKey, cancellationToken);
int attempt = 0;
while (!acquiredLock && attempt < 10)
{
await Task.Delay(Options.IdleTime, cancellationToken);
acquiredLock = await _lockProvider.AcquireLock(eventKey, cancellationToken);
attempt++;
}
if (!acquiredLock)
{
Logger.LogWarning($"Failed to lock {evt}");
}
else
{
_greylist.Remove(eventKey);
await persistenceStore.MarkEventUnprocessed(evt, cancellationToken);
await QueueProvider.QueueWork(evt, QueueType.Event);
}
}
finally
{
if (acquiredLock)
{
await _lockProvider.ReleaseLock(eventKey);
}
}
}
}
}
private async Task SubscribeEvent(EventSubscription subscription, IPersistenceProvider persistenceStore)
{
//TODO: move to own class
Logger.LogDebug("Subscribing to event {0} {1} for workflow {2} step {3}", subscription.EventName, subscription.EventKey, subscription.WorkflowId, subscription.StepId);
await persistenceStore.CreateEventSubscription(subscription);
var events = await persistenceStore.GetEvents(subscription.EventName, subscription.EventKey, subscription.SubscribeAsOf);
foreach (var evt in events)
{
await persistenceStore.MarkEventUnprocessed(evt);
await QueueProvider.QueueWork(evt, QueueType.Event);
}
}
public async Task <string> StartWorkflow <TData>(string workflowId, int?version, TData data = null)
where TData : class
{
if (_shutdown)
{
throw new InvalidOperationException("Host is not running");
}
var def = Registry.GetDefinition(workflowId, version);
if (def == null)
{
throw new WorkflowNotRegisteredException(workflowId, version);
}
var wf = new WorkflowInstance
{
WorkflowDefinitionId = workflowId,
Version = def.Version,
Data = data,
Description = def.Description,
NextExecution = 0,
CreateTime = DateTime.Now.ToUniversalTime(),
Status = WorkflowStatus.Runnable
};
if ((def.DataType != null) && (data == null))
{
wf.Data = def.DataType.GetConstructor(new Type[] { }).Invoke(null);
}
wf.ExecutionPointers.Add(new ExecutionPointer
{
Id = Guid.NewGuid().ToString(),
StepId = 0,
Active = true,
StepName = def.Steps.First(x => x.Id == 0).Name
});
string id = await PersistenceStore.CreateNewWorkflow(wf);
await QueueProvider.QueueWork(id, QueueType.Workflow);
return(id);
}
protected override async Task ProcessItem(string itemId, CancellationToken cancellationToken)
{
if (!await _lockProvider.AcquireLock($"evt:{itemId}", cancellationToken))
{
Logger.LogInformation($"Event locked {itemId}");
return;
}
try
{
cancellationToken.ThrowIfCancellationRequested();
var evt = await _eventRepository.GetEvent(itemId);
if (evt.EventTime <= _datetimeProvider.UtcNow)
{
var subs = await _subscriptionRepository.GetSubscriptions(evt.EventName, evt.EventKey, evt.EventTime);
var toQueue = new List <string>();
var complete = true;
foreach (var sub in subs.ToList())
{
complete = complete && await SeedSubscription(evt, sub, toQueue, cancellationToken);
}
if (complete)
{
await _eventRepository.MarkEventProcessed(itemId);
}
foreach (var eventId in toQueue)
{
await QueueProvider.QueueWork(eventId, QueueType.Event);
}
}
}
finally
{
await _lockProvider.ReleaseLock($"evt:{itemId}");
}
}
private async Task <bool> SeedSubscription(Event evt, EventSubscription sub, CancellationToken cancellationToken)
{
if (await _lockProvider.AcquireLock(sub.WorkflowId, cancellationToken))
{
try
{
var workflow = await _persistenceStore.GetWorkflowInstance(sub.WorkflowId);
var pointers = workflow.ExecutionPointers.Where(p => p.EventName == sub.EventName && p.EventKey == sub.EventKey && !p.EventPublished && p.EndTime == null);
foreach (var p in pointers)
{
p.EventData = evt.EventData;
p.EventPublished = true;
p.Active = true;
}
workflow.NextExecution = 0;
await _persistenceStore.PersistWorkflow(workflow);
await _persistenceStore.TerminateSubscription(sub.Id);
return(true);
}
catch (Exception ex)
{
Logger.LogError(ex.Message, ex);
return(false);
}
finally
{
await _lockProvider.ReleaseLock(sub.WorkflowId);
await QueueProvider.QueueWork(sub.WorkflowId, QueueType.Workflow);
}
}
else
{
Logger.LogInformation("Workflow locked {0}", sub.WorkflowId);
return(false);
}
}
private async void FutureQueue(WorkflowInstance workflow, CancellationToken cancellationToken)
{
try
{
if (!workflow.NextExecution.HasValue)
{
return;
}
var target = (workflow.NextExecution.Value - _datetimeProvider.Now.ToUniversalTime().Ticks);
if (target > 0)
{
await Task.Delay(TimeSpan.FromTicks(target), cancellationToken);
}
await QueueProvider.QueueWork(workflow.Id, QueueType.Workflow);
}
catch (Exception ex)
{
Logger.LogError(ex, ex.Message);
}
}
protected override async Task ProcessItem(string itemId, CancellationToken cancellationToken)
{
if (!await _lockProvider.AcquireLock($"evt:{itemId}", cancellationToken))
{
Logger.LogInformation($"Event locked {itemId}");
return;
}
try
{
cancellationToken.ThrowIfCancellationRequested();
var evt = await _eventRepository.GetEvent(itemId, cancellationToken);
if (evt.IsProcessed)
{
_greylist.Add($"evt:{evt.Id}");
return;
}
if (evt.EventTime <= _datetimeProvider.UtcNow)
{
IEnumerable <EventSubscription> subs = null;
if (evt.EventData is ActivityResult)
{
var activity = await _subscriptionRepository.GetSubscription((evt.EventData as ActivityResult).SubscriptionId, cancellationToken);
if (activity == null)
{
Logger.LogWarning($"Activity already processed - {(evt.EventData as ActivityResult).SubscriptionId}");
await _eventRepository.MarkEventProcessed(itemId, cancellationToken);
return;
}
subs = new List <EventSubscription> {
activity
};
}
else
{
subs = await _subscriptionRepository.GetSubscriptions(evt.EventName, evt.EventKey, evt.EventTime, cancellationToken);
}
var toQueue = new HashSet <string>();
var complete = true;
foreach (var sub in subs.ToList())
{
complete = complete && await SeedSubscription(evt, sub, toQueue, cancellationToken);
}
if (complete)
{
await _eventRepository.MarkEventProcessed(itemId, cancellationToken);
}
else
{
_greylist.Remove($"evt:{evt.Id}");
}
foreach (var eventId in toQueue)
{
await QueueProvider.QueueWork(eventId, QueueType.Event);
}
}
}
finally
{
await _lockProvider.ReleaseLock($"evt:{itemId}");
}
}
protected override async Task ProcessItem(string itemId, CancellationToken cancellationToken)
{
if (!await _lockProvider.AcquireLock(itemId, cancellationToken))
{
Logger.LogInformation("Workflow locked {0}", itemId);
return;
}
WorkflowInstance workflow = null;
WorkflowExecutorResult result = null;
var persistenceStore = _persistenceStorePool.Get();
try
{
try
{
cancellationToken.ThrowIfCancellationRequested();
workflow = await persistenceStore.GetWorkflowInstance(itemId);
if (workflow?.Status == WorkflowStatus.Runnable)
{
var executor = _executorPool.Get();
try
{
if (!Options.ExecuteStepsUntilEventWaits)
{
try
{
result = await executor.Execute(workflow);
}
finally
{
await persistenceStore.PersistWorkflow(workflow);
}
}
else
{
result = await executor.ExecuteUntilEventWait(workflow, async (wfResult) =>
{
//every step should be saved
await persistenceStore.PersistWorkflow(workflow);
//check cancellation request every step
cancellationToken.ThrowIfCancellationRequested();
});
}
}
finally
{
_executorPool.Return(executor);
await QueueProvider.QueueWork(itemId, QueueType.Index);
}
}
}
finally
{
await _lockProvider.ReleaseLock(itemId);
if ((workflow != null) && (result != null))
{
foreach (var sub in result.Subscriptions)
{
await SubscribeEvent(sub, persistenceStore);
}
await persistenceStore.PersistErrors(result.Errors);
var readAheadTicks = _datetimeProvider.Now.Add(Options.PollInterval).ToUniversalTime().Ticks;
if ((workflow.Status == WorkflowStatus.Runnable) && workflow.NextExecution.HasValue && workflow.NextExecution.Value < readAheadTicks)
{
new Task(() => FutureQueue(workflow, cancellationToken)).Start();
}
}
}
}
finally
{
_persistenceStorePool.Return(persistenceStore);
}
}
