public async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses)
{
using (var dbContext = _dbContextFactory())
{
var executionStartedEvent = creationMessage.Event as ExecutionStartedEvent;
var instanceId = creationMessage.OrchestrationInstance.InstanceId;
var executionId = creationMessage.OrchestrationInstance.ExecutionId;
var instance = await dbContext.Instances
.Include(i => i.LastExecution)
.FirstOrDefaultAsync(i => i.InstanceId == instanceId);
if (instance != null)
{
if (dedupeStatuses != null && dedupeStatuses.Contains(instance.LastExecution.Status))
{
return;
}
if (!IsFinalInstanceStatus(instance.LastExecution.Status))
{
throw new Exception("Orchestration has an active execution");
}
}
var runtimeState = new OrchestrationRuntimeState(new[] { executionStartedEvent });
if (instance == null)
{
instance = _instanceMapper.CreateInstance(executionStartedEvent);
await dbContext.Instances.AddAsync(instance);
}
else
{
_instanceMapper.UpdateInstance(instance, runtimeState);
}
var execution = _executionMapper.CreateExecution(runtimeState);
await dbContext.Executions.AddAsync(execution);
var knownQueues = new Dictionary <string, string>
{
[instance.InstanceId] = QueueMapper.ToQueueName(runtimeState.Name, runtimeState.Version)
};
var orchestrationWorkItem = await _orchestrationMessageMapper.CreateOrchestrationMessageAsync(
creationMessage,
0,
dbContext,
knownQueues
);
await dbContext.OrchestrationMessages.AddAsync(orchestrationWorkItem);
await dbContext.SaveChangesAsync();
}
}
public async Task CompleteTaskOrchestrationWorkItemAsync(
TaskOrchestrationWorkItem workItem,
OrchestrationRuntimeState newOrchestrationRuntimeState,
IList <TaskMessage> outboundMessages,
IList <TaskMessage> orchestratorMessages,
IList <TaskMessage> timerMessages,
TaskMessage continuedAsNewMessage,
OrchestrationState orchestrationState)
{
using (var dbContext = _dbContextFactory())
{
var session = workItem.Session as EFCoreOrchestrationSession;
// Create child orchestrations
foreach (var executionStartedEvent in orchestratorMessages.Select(m => m.Event).OfType <ExecutionStartedEvent>())
{
var childInstance = _instanceMapper.CreateInstance(executionStartedEvent);
await dbContext.Instances.AddAsync(childInstance);
var childRuntimeState = new OrchestrationRuntimeState(new[] { executionStartedEvent });
var childExecution = _executionMapper.CreateExecution(childRuntimeState);
await dbContext.Executions.AddAsync(childExecution);
}
var orchestrationQueueName = QueueMapper.ToQueueName(orchestrationState.Name, orchestrationState.Version);
var knownQueues = new Dictionary <string, string>
{
[orchestrationState.OrchestrationInstance.InstanceId] = orchestrationQueueName
};
if (orchestrationState.ParentInstance != null)
{
knownQueues[orchestrationState.ParentInstance.OrchestrationInstance.InstanceId] = QueueMapper.ToQueueName(orchestrationState.ParentInstance.Name, orchestrationState.ParentInstance.Version);
}
// Write messages
var activityMessages = outboundMessages
.Select(m => _activityMessageMapper.CreateActivityMessage(m, orchestrationQueueName))
.ToArray();
var orchestatorMessages = await orchestratorMessages
.Select((m, i) => _orchestrationMessageMapper.CreateOrchestrationMessageAsync(m, i, dbContext, knownQueues))
.WhenAllSerial();
var timerOrchestrationMessages = await timerMessages
.Select((m, i) => _orchestrationMessageMapper.CreateOrchestrationMessageAsync(m, i, dbContext, knownQueues))
.WhenAllSerial();
var continuedAsNewOrchestrationMessage = continuedAsNewMessage != null
? await _orchestrationMessageMapper.CreateOrchestrationMessageAsync(continuedAsNewMessage, 0, dbContext, knownQueues)
: null;
await dbContext.ActivityMessages.AddRangeAsync(activityMessages);
await dbContext.OrchestrationMessages.AddRangeAsync(orchestatorMessages);
await dbContext.OrchestrationMessages.AddRangeAsync(timerOrchestrationMessages);
if (continuedAsNewOrchestrationMessage != null)
{
await dbContext.OrchestrationMessages.AddAsync(continuedAsNewOrchestrationMessage);
}
// Remove executed messages
dbContext.AttachRange(session.Messages);
dbContext.OrchestrationMessages.RemoveRange(session.Messages);
// Update instance
var instance = session.Instance;
dbContext.Instances.Attach(instance);
_instanceMapper.UpdateInstance(instance, newOrchestrationRuntimeState);
// Update current execution
EnrichNewEventsInput(workItem.OrchestrationRuntimeState, outboundMessages, orchestratorMessages);
session.Execution = await SaveExecutionAsync(dbContext, workItem.OrchestrationRuntimeState, session.Execution);
// Update new execution
if (newOrchestrationRuntimeState != workItem.OrchestrationRuntimeState)
{
EnrichNewEventsInput(newOrchestrationRuntimeState, outboundMessages, orchestratorMessages);
session.Execution = await SaveExecutionAsync(dbContext, newOrchestrationRuntimeState);
}
await dbContext.SaveChangesAsync();
session.RuntimeState = newOrchestrationRuntimeState;
session.ClearMessages();
}
}
private async Task RewindInstanceAsync(OrchestrationDbContext dbContext, string instanceId, string reason)
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// REWIND ALGORITHM:
// 1. Finds failed execution of specified orchestration instance to rewind
// 2. Finds failure entities to clear and over-writes them (as well as corresponding trigger events)
// 3. Identifies sub-orchestration failure(s) from parent instance and calls RewindHistoryAsync recursively on failed sub-orchestration child instance(s)
// 4. Resets orchestration status of rewound instance in instance store table to prepare it to be restarted
// 5. Restart that doesn't have failed suborchestrations with a generic event
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
var lastExecution = dbContext.Instances
.Where(i => i.InstanceId == instanceId)
.Select(i => i.LastExecution)
.FirstOrDefault();
var events = await dbContext.Events
.Where(e => e.ExecutionId == lastExecution.ExecutionId)
.ToArrayAsync();
var historyEvents = events
.ToDictionary(e => _options.DataConverter.Deserialize <HistoryEvent>(e.Content));
bool hasFailedSubOrchestrations = false;
foreach (var historyEvent in historyEvents.Keys)
{
if (historyEvent is TaskFailedEvent taskFailedEvent)
{
var taskScheduledEvent = historyEvents.Keys.OfType <TaskScheduledEvent>()
.FirstOrDefault(e => e.EventId == taskFailedEvent.TaskScheduledId);
var rewoundTaskScheduledData = _options.RewindDataConverter.Serialize(new
{
taskScheduledEvent.EventType,
taskScheduledEvent.Name,
taskScheduledEvent.Version,
taskScheduledEvent.Input
});
historyEvents[taskScheduledEvent].Content = _options.DataConverter.Serialize(
new GenericEvent(taskScheduledEvent.EventId, $"Rewound: {rewoundTaskScheduledData}")
{
Timestamp = taskScheduledEvent.Timestamp
}
);
var rewoundTaskFailedData = _options.RewindDataConverter.Serialize(new
{
taskFailedEvent.EventType,
taskFailedEvent.Reason,
taskFailedEvent.TaskScheduledId
});
historyEvents[taskFailedEvent].Content = _options.DataConverter.Serialize(
new GenericEvent(taskFailedEvent.EventId, $"Rewound: {rewoundTaskFailedData}")
{
Timestamp = taskFailedEvent.Timestamp
}
);
}
else if (historyEvent is SubOrchestrationInstanceFailedEvent soFailedEvent)
{
hasFailedSubOrchestrations = true;
var soCreatedEvent = historyEvents.Keys.OfType <SubOrchestrationInstanceCreatedEvent>()
.FirstOrDefault(e => e.EventId == soFailedEvent.TaskScheduledId);
var rewoundSoCreatedData = _options.RewindDataConverter.Serialize(new
{
soCreatedEvent.EventType,
soCreatedEvent.Name,
soCreatedEvent.Version,
soCreatedEvent.Input
});
historyEvents[soCreatedEvent].Content = _options.DataConverter.Serialize(
new GenericEvent(soCreatedEvent.EventId, $"Rewound: {rewoundSoCreatedData}")
{
Timestamp = soCreatedEvent.Timestamp
}
);
var rewoundSoFailedData = _options.RewindDataConverter.Serialize(new
{
soFailedEvent.EventType,
soFailedEvent.Reason,
soFailedEvent.TaskScheduledId
});
historyEvents[soFailedEvent].Content = _options.DataConverter.Serialize(
new GenericEvent(soFailedEvent.EventId, $"Rewound: {rewoundSoFailedData}")
{
Timestamp = soFailedEvent.Timestamp
}
);
// recursive call to clear out failure events on child instances
await RewindInstanceAsync(dbContext, soCreatedEvent.InstanceId, reason);
}
else if (historyEvent is ExecutionCompletedEvent executionCompletedEvent &&
executionCompletedEvent.OrchestrationStatus == OrchestrationStatus.Failed)
{
var rewoundExecutionCompletedData = _options.RewindDataConverter.Serialize(new
{
executionCompletedEvent.EventType,
executionCompletedEvent.Result,
executionCompletedEvent.OrchestrationStatus
});
historyEvents[executionCompletedEvent].Content = _options.DataConverter.Serialize(
new GenericEvent(executionCompletedEvent.EventId, $"Rewound: {rewoundExecutionCompletedData}")
{
Timestamp = executionCompletedEvent.Timestamp
}
);
}
}
// Reset execution status
lastExecution.Status = OrchestrationStatus.Running;
lastExecution.LastUpdatedTime = DateTime.UtcNow;
if (!hasFailedSubOrchestrations)
{
var orchestrationInstance = new OrchestrationInstance
{
InstanceId = instanceId
};
var taskMessage = new TaskMessage
{
OrchestrationInstance = orchestrationInstance,
Event = new GenericEvent(-1, reason)
};
var knownQueues = new Dictionary <string, string>
{
[lastExecution.InstanceId] = QueueMapper.ToQueueName(lastExecution.Name, lastExecution.Version)
};
var orchestrationMessage = await _orchestrationMessageMapper.CreateOrchestrationMessageAsync(taskMessage, 0, dbContext, knownQueues);
await dbContext.OrchestrationMessages.AddAsync(orchestrationMessage);
}
}