private async Task <Execution> SaveExecutionAsync( OrchestrationDbContext dbContext, OrchestrationRuntimeState runtimeState, Execution existingExecution = null) { Execution execution; if (existingExecution == null) { execution = _executionMapper.CreateExecution(runtimeState); await dbContext.Executions.AddAsync(execution); } else { execution = existingExecution; dbContext.Executions.Attach(execution); _executionMapper.UpdateExecution(execution, runtimeState); } var initialSequenceNumber = runtimeState.Events.Count - runtimeState.NewEvents.Count; var newEvents = runtimeState.NewEvents .Select((e, i) => new Event { Id = Guid.NewGuid(), InstanceId = runtimeState.OrchestrationInstance.InstanceId, ExecutionId = runtimeState.OrchestrationInstance.ExecutionId, SequenceNumber = initialSequenceNumber + i, Content = _options.DataConverter.Serialize(e) }).ToArray(); await dbContext.Events.AddRangeAsync(newEvents); return(execution); }
public async Task <IList <TaskMessage> > FetchNewMessagesAsync( OrchestrationDbContext dbContext, CancellationToken cancellationToken = default) { var dbWorkItems = await dbContext.OrchestrationMessages .Where(w => w.AvailableAt <= DateTime.UtcNow && w.Queue == Instance.LastQueueName) .Where(w => w.Instance.InstanceId == Instance.InstanceId && w.Instance.LockId == Instance.LockId) .Where(w => !Messages.Contains(w)) .OrderBy(w => w.AvailableAt) .ThenBy(w => w.SequenceNumber) .AsNoTracking() .ToArrayAsync(cancellationToken); var isExecutable = RuntimeState.ExecutionStartedEvent == null || RuntimeState.OrchestrationStatus == OrchestrationStatus.Pending || RuntimeState.OrchestrationStatus == OrchestrationStatus.Running; var messagesToDiscard = dbWorkItems .Where(m => !isExecutable || (m.ExecutionId != null && m.ExecutionId != Instance.LastExecutionId)) .ToArray(); if (messagesToDiscard.Length > 0) { foreach (var message in messagesToDiscard) { dbContext.OrchestrationMessages.Attach(message); dbContext.OrchestrationMessages.Remove(message); } dbWorkItems = dbWorkItems .Except(messagesToDiscard) .ToArray(); } Messages.AddRange(dbWorkItems); var deserializedMessages = dbWorkItems .Select(w => _options.DataConverter.Deserialize <TaskMessage>(w.Message)) .ToArray(); return(deserializedMessages); }
private async Task <Instance> LockNextInstance(OrchestrationDbContext dbContext, INameVersionInfo[] orchestrations) { if (orchestrations == null) { return(await _dbContextExtensions.TryLockNextInstanceAsync(dbContext, _options.OrchestrationLockTimeout)); } var queues = orchestrations .Select(QueueMapper.ToQueueName) .ToArray(); var instance = await _dbContextExtensions.TryLockNextInstanceAsync(dbContext, queues, _options.OrchestrationLockTimeout); if (instance != null) { return(instance); } return(null); }
private async Task <ActivityMessage> LockActivityMessage(OrchestrationDbContext dbContext, INameVersionInfo[] activities) { var lockId = Guid.NewGuid().ToString(); var lockUntilUtc = DateTime.UtcNow.Add(_options.OrchestrationLockTimeout); if (activities == null) { return(await _dbContextExtensions.TryLockNextActivityMessageAsync(dbContext, _options.OrchestrationLockTimeout)); } var queues = activities .Select(QueueMapper.ToQueueName) .ToArray(); var activityMessage = await _dbContextExtensions.TryLockNextActivityMessageAsync(dbContext, queues, _options.OrchestrationLockTimeout); if (activityMessage != null) { return(activityMessage); } return(null); }
private async Task RewindInstanceAsync(OrchestrationDbContext dbContext, string instanceId, string reason) { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // REWIND ALGORITHM: // 1. Finds failed execution of specified orchestration instance to rewind // 2. Finds failure entities to clear and over-writes them (as well as corresponding trigger events) // 3. Identifies sub-orchestration failure(s) from parent instance and calls RewindHistoryAsync recursively on failed sub-orchestration child instance(s) // 4. Resets orchestration status of rewound instance in instance store table to prepare it to be restarted // 5. Restart that doesn't have failed suborchestrations with a generic event //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// var lastExecution = dbContext.Instances .Where(i => i.InstanceId == instanceId) .Select(i => i.LastExecution) .FirstOrDefault(); var events = await dbContext.Events .Where(e => e.ExecutionId == lastExecution.ExecutionId) .ToArrayAsync(); var historyEvents = events .ToDictionary(e => _options.DataConverter.Deserialize <HistoryEvent>(e.Content)); bool hasFailedSubOrchestrations = false; foreach (var historyEvent in historyEvents.Keys) { if (historyEvent is TaskFailedEvent taskFailedEvent) { var taskScheduledEvent = historyEvents.Keys.OfType <TaskScheduledEvent>() .FirstOrDefault(e => e.EventId == taskFailedEvent.TaskScheduledId); var rewoundTaskScheduledData = _options.RewindDataConverter.Serialize(new { taskScheduledEvent.EventType, taskScheduledEvent.Name, taskScheduledEvent.Version, taskScheduledEvent.Input }); historyEvents[taskScheduledEvent].Content = _options.DataConverter.Serialize( new GenericEvent(taskScheduledEvent.EventId, $"Rewound: {rewoundTaskScheduledData}") { Timestamp = taskScheduledEvent.Timestamp } ); var rewoundTaskFailedData = _options.RewindDataConverter.Serialize(new { taskFailedEvent.EventType, taskFailedEvent.Reason, taskFailedEvent.TaskScheduledId }); historyEvents[taskFailedEvent].Content = _options.DataConverter.Serialize( new GenericEvent(taskFailedEvent.EventId, $"Rewound: {rewoundTaskFailedData}") { Timestamp = taskFailedEvent.Timestamp } ); } else if (historyEvent is SubOrchestrationInstanceFailedEvent soFailedEvent) { hasFailedSubOrchestrations = true; var soCreatedEvent = historyEvents.Keys.OfType <SubOrchestrationInstanceCreatedEvent>() .FirstOrDefault(e => e.EventId == soFailedEvent.TaskScheduledId); var rewoundSoCreatedData = _options.RewindDataConverter.Serialize(new { soCreatedEvent.EventType, soCreatedEvent.Name, soCreatedEvent.Version, soCreatedEvent.Input }); historyEvents[soCreatedEvent].Content = _options.DataConverter.Serialize( new GenericEvent(soCreatedEvent.EventId, $"Rewound: {rewoundSoCreatedData}") { Timestamp = soCreatedEvent.Timestamp } ); var rewoundSoFailedData = _options.RewindDataConverter.Serialize(new { soFailedEvent.EventType, soFailedEvent.Reason, soFailedEvent.TaskScheduledId }); historyEvents[soFailedEvent].Content = _options.DataConverter.Serialize( new GenericEvent(soFailedEvent.EventId, $"Rewound: {rewoundSoFailedData}") { Timestamp = soFailedEvent.Timestamp } ); // recursive call to clear out failure events on child instances await RewindInstanceAsync(dbContext, soCreatedEvent.InstanceId, reason); } else if (historyEvent is ExecutionCompletedEvent executionCompletedEvent && executionCompletedEvent.OrchestrationStatus == OrchestrationStatus.Failed) { var rewoundExecutionCompletedData = _options.RewindDataConverter.Serialize(new { executionCompletedEvent.EventType, executionCompletedEvent.Result, executionCompletedEvent.OrchestrationStatus }); historyEvents[executionCompletedEvent].Content = _options.DataConverter.Serialize( new GenericEvent(executionCompletedEvent.EventId, $"Rewound: {rewoundExecutionCompletedData}") { Timestamp = executionCompletedEvent.Timestamp } ); } } // Reset execution status lastExecution.Status = OrchestrationStatus.Running; lastExecution.LastUpdatedTime = DateTime.UtcNow; if (!hasFailedSubOrchestrations) { var orchestrationInstance = new OrchestrationInstance { InstanceId = instanceId }; var taskMessage = new TaskMessage { OrchestrationInstance = orchestrationInstance, Event = new GenericEvent(-1, reason) }; var knownQueues = new Dictionary <string, string> { [lastExecution.InstanceId] = QueueMapper.ToQueueName(lastExecution.Name, lastExecution.Version) }; var orchestrationMessage = await _orchestrationMessageMapper.CreateOrchestrationMessageAsync(taskMessage, 0, dbContext, knownQueues); await dbContext.OrchestrationMessages.AddAsync(orchestrationMessage); } }
public abstract Task Migrate(OrchestrationDbContext dbContext);
public abstract Task <int> PurgeInstanceHistoryAsync(OrchestrationDbContext dbContext, string instanceId);
public abstract Task PurgeOrchestrationHistoryAsync(OrchestrationDbContext dbContext, DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType);
public abstract Task <ActivityMessage> TryLockNextActivityMessageAsync( OrchestrationDbContext dbContext, string[] queues, TimeSpan lockTimeout);
public abstract Task <Instance> TryLockNextInstanceAsync( OrchestrationDbContext dbContext, string[] queues, TimeSpan lockTimeout);