private async Task <Execution> SaveExecutionAsync(
OrchestrationDbContext dbContext,
OrchestrationRuntimeState runtimeState,
Execution existingExecution = null)
{
Execution execution;
if (existingExecution == null)
{
execution = _executionMapper.CreateExecution(runtimeState);
await dbContext.Executions.AddAsync(execution);
}
else
{
execution = existingExecution;
dbContext.Executions.Attach(execution);
_executionMapper.UpdateExecution(execution, runtimeState);
}
var initialSequenceNumber = runtimeState.Events.Count - runtimeState.NewEvents.Count;
var newEvents = runtimeState.NewEvents
.Select((e, i) => new Event
{
Id = Guid.NewGuid(),
InstanceId = runtimeState.OrchestrationInstance.InstanceId,
ExecutionId = runtimeState.OrchestrationInstance.ExecutionId,
SequenceNumber = initialSequenceNumber + i,
Content = _options.DataConverter.Serialize(e)
}).ToArray();
await dbContext.Events.AddRangeAsync(newEvents);
return(execution);
}
public async Task <IList <TaskMessage> > FetchNewMessagesAsync(
OrchestrationDbContext dbContext,
CancellationToken cancellationToken = default)
{
var dbWorkItems = await dbContext.OrchestrationMessages
.Where(w => w.AvailableAt <= DateTime.UtcNow &&
w.Queue == Instance.LastQueueName)
.Where(w => w.Instance.InstanceId == Instance.InstanceId &&
w.Instance.LockId == Instance.LockId)
.Where(w => !Messages.Contains(w))
.OrderBy(w => w.AvailableAt)
.ThenBy(w => w.SequenceNumber)
.AsNoTracking()
.ToArrayAsync(cancellationToken);
var isExecutable = RuntimeState.ExecutionStartedEvent == null ||
RuntimeState.OrchestrationStatus == OrchestrationStatus.Pending ||
RuntimeState.OrchestrationStatus == OrchestrationStatus.Running;
var messagesToDiscard = dbWorkItems
.Where(m => !isExecutable || (m.ExecutionId != null && m.ExecutionId != Instance.LastExecutionId))
.ToArray();
if (messagesToDiscard.Length > 0)
{
foreach (var message in messagesToDiscard)
{
dbContext.OrchestrationMessages.Attach(message);
dbContext.OrchestrationMessages.Remove(message);
}
dbWorkItems = dbWorkItems
.Except(messagesToDiscard)
.ToArray();
}
Messages.AddRange(dbWorkItems);
var deserializedMessages = dbWorkItems
.Select(w => _options.DataConverter.Deserialize <TaskMessage>(w.Message))
.ToArray();
return(deserializedMessages);
}
private async Task <Instance> LockNextInstance(OrchestrationDbContext dbContext, INameVersionInfo[] orchestrations)
{
if (orchestrations == null)
{
return(await _dbContextExtensions.TryLockNextInstanceAsync(dbContext, _options.OrchestrationLockTimeout));
}
var queues = orchestrations
.Select(QueueMapper.ToQueueName)
.ToArray();
var instance = await _dbContextExtensions.TryLockNextInstanceAsync(dbContext, queues, _options.OrchestrationLockTimeout);
if (instance != null)
{
return(instance);
}
return(null);
}
private async Task <ActivityMessage> LockActivityMessage(OrchestrationDbContext dbContext, INameVersionInfo[] activities)
{
var lockId = Guid.NewGuid().ToString();
var lockUntilUtc = DateTime.UtcNow.Add(_options.OrchestrationLockTimeout);
if (activities == null)
{
return(await _dbContextExtensions.TryLockNextActivityMessageAsync(dbContext, _options.OrchestrationLockTimeout));
}
var queues = activities
.Select(QueueMapper.ToQueueName)
.ToArray();
var activityMessage = await _dbContextExtensions.TryLockNextActivityMessageAsync(dbContext, queues, _options.OrchestrationLockTimeout);
if (activityMessage != null)
{
return(activityMessage);
}
return(null);
}
private async Task RewindInstanceAsync(OrchestrationDbContext dbContext, string instanceId, string reason)
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// REWIND ALGORITHM:
// 1. Finds failed execution of specified orchestration instance to rewind
// 2. Finds failure entities to clear and over-writes them (as well as corresponding trigger events)
// 3. Identifies sub-orchestration failure(s) from parent instance and calls RewindHistoryAsync recursively on failed sub-orchestration child instance(s)
// 4. Resets orchestration status of rewound instance in instance store table to prepare it to be restarted
// 5. Restart that doesn't have failed suborchestrations with a generic event
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
var lastExecution = dbContext.Instances
.Where(i => i.InstanceId == instanceId)
.Select(i => i.LastExecution)
.FirstOrDefault();
var events = await dbContext.Events
.Where(e => e.ExecutionId == lastExecution.ExecutionId)
.ToArrayAsync();
var historyEvents = events
.ToDictionary(e => _options.DataConverter.Deserialize <HistoryEvent>(e.Content));
bool hasFailedSubOrchestrations = false;
foreach (var historyEvent in historyEvents.Keys)
{
if (historyEvent is TaskFailedEvent taskFailedEvent)
{
var taskScheduledEvent = historyEvents.Keys.OfType <TaskScheduledEvent>()
.FirstOrDefault(e => e.EventId == taskFailedEvent.TaskScheduledId);
var rewoundTaskScheduledData = _options.RewindDataConverter.Serialize(new
{
taskScheduledEvent.EventType,
taskScheduledEvent.Name,
taskScheduledEvent.Version,
taskScheduledEvent.Input
});
historyEvents[taskScheduledEvent].Content = _options.DataConverter.Serialize(
new GenericEvent(taskScheduledEvent.EventId, $"Rewound: {rewoundTaskScheduledData}")
{
Timestamp = taskScheduledEvent.Timestamp
}
);
var rewoundTaskFailedData = _options.RewindDataConverter.Serialize(new
{
taskFailedEvent.EventType,
taskFailedEvent.Reason,
taskFailedEvent.TaskScheduledId
});
historyEvents[taskFailedEvent].Content = _options.DataConverter.Serialize(
new GenericEvent(taskFailedEvent.EventId, $"Rewound: {rewoundTaskFailedData}")
{
Timestamp = taskFailedEvent.Timestamp
}
);
}
else if (historyEvent is SubOrchestrationInstanceFailedEvent soFailedEvent)
{
hasFailedSubOrchestrations = true;
var soCreatedEvent = historyEvents.Keys.OfType <SubOrchestrationInstanceCreatedEvent>()
.FirstOrDefault(e => e.EventId == soFailedEvent.TaskScheduledId);
var rewoundSoCreatedData = _options.RewindDataConverter.Serialize(new
{
soCreatedEvent.EventType,
soCreatedEvent.Name,
soCreatedEvent.Version,
soCreatedEvent.Input
});
historyEvents[soCreatedEvent].Content = _options.DataConverter.Serialize(
new GenericEvent(soCreatedEvent.EventId, $"Rewound: {rewoundSoCreatedData}")
{
Timestamp = soCreatedEvent.Timestamp
}
);
var rewoundSoFailedData = _options.RewindDataConverter.Serialize(new
{
soFailedEvent.EventType,
soFailedEvent.Reason,
soFailedEvent.TaskScheduledId
});
historyEvents[soFailedEvent].Content = _options.DataConverter.Serialize(
new GenericEvent(soFailedEvent.EventId, $"Rewound: {rewoundSoFailedData}")
{
Timestamp = soFailedEvent.Timestamp
}
);
// recursive call to clear out failure events on child instances
await RewindInstanceAsync(dbContext, soCreatedEvent.InstanceId, reason);
}
else if (historyEvent is ExecutionCompletedEvent executionCompletedEvent &&
executionCompletedEvent.OrchestrationStatus == OrchestrationStatus.Failed)
{
var rewoundExecutionCompletedData = _options.RewindDataConverter.Serialize(new
{
executionCompletedEvent.EventType,
executionCompletedEvent.Result,
executionCompletedEvent.OrchestrationStatus
});
historyEvents[executionCompletedEvent].Content = _options.DataConverter.Serialize(
new GenericEvent(executionCompletedEvent.EventId, $"Rewound: {rewoundExecutionCompletedData}")
{
Timestamp = executionCompletedEvent.Timestamp
}
);
}
}
// Reset execution status
lastExecution.Status = OrchestrationStatus.Running;
lastExecution.LastUpdatedTime = DateTime.UtcNow;
if (!hasFailedSubOrchestrations)
{
var orchestrationInstance = new OrchestrationInstance
{
InstanceId = instanceId
};
var taskMessage = new TaskMessage
{
OrchestrationInstance = orchestrationInstance,
Event = new GenericEvent(-1, reason)
};
var knownQueues = new Dictionary <string, string>
{
[lastExecution.InstanceId] = QueueMapper.ToQueueName(lastExecution.Name, lastExecution.Version)
};
var orchestrationMessage = await _orchestrationMessageMapper.CreateOrchestrationMessageAsync(taskMessage, 0, dbContext, knownQueues);
await dbContext.OrchestrationMessages.AddAsync(orchestrationMessage);
}
}
public abstract Task Migrate(OrchestrationDbContext dbContext);
public abstract Task <int> PurgeInstanceHistoryAsync(OrchestrationDbContext dbContext, string instanceId);
public abstract Task PurgeOrchestrationHistoryAsync(OrchestrationDbContext dbContext, DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType);
public abstract Task <ActivityMessage> TryLockNextActivityMessageAsync(
OrchestrationDbContext dbContext,
string[] queues,
TimeSpan lockTimeout);
public abstract Task <Instance> TryLockNextInstanceAsync(
OrchestrationDbContext dbContext,
string[] queues,
TimeSpan lockTimeout);
