TaskMessage?ProcessWorkflowCompletedTaskDecision(
OrchestrationCompleteOrchestratorAction completeOrchestratorAction,
OrchestrationRuntimeState runtimeState,
bool includeDetails,
out bool continuedAsNew)
{
ExecutionCompletedEvent executionCompletedEvent;
continuedAsNew = (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew);
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
{
executionCompletedEvent = new ContinueAsNewEvent(completeOrchestratorAction.Id,
completeOrchestratorAction.Result);
}
else
{
executionCompletedEvent = new ExecutionCompletedEvent(completeOrchestratorAction.Id,
completeOrchestratorAction.Result,
completeOrchestratorAction.OrchestrationStatus,
completeOrchestratorAction.FailureDetails);
}
runtimeState.AddEvent(executionCompletedEvent);
this.logHelper.OrchestrationCompleted(runtimeState, completeOrchestratorAction);
TraceHelper.TraceInstance(
runtimeState.OrchestrationStatus == OrchestrationStatus.Failed ? TraceEventType.Warning : TraceEventType.Information,
"TaskOrchestrationDispatcher-InstanceCompleted",
runtimeState.OrchestrationInstance,
"Instance Id '{0}' completed in state {1} with result: {2}",
runtimeState.OrchestrationInstance,
runtimeState.OrchestrationStatus,
completeOrchestratorAction.Result);
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-InstanceCompletionEvents",
runtimeState.OrchestrationInstance,
() => Utils.EscapeJson(JsonDataConverter.Default.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true)));
// Check to see if we need to start a new execution
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
{
var taskMessage = new TaskMessage();
var startedEvent = new ExecutionStartedEvent(-1, completeOrchestratorAction.Result)
{
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = runtimeState.OrchestrationInstance !.InstanceId,
ExecutionId = Guid.NewGuid().ToString("N")
},
/// <summary>
/// Method to process a new work item
/// </summary>
/// <param name="workItem">The work item to process</param>
protected async Task <bool> OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
var messagesToSend = new List <TaskMessage>();
var timerMessages = new List <TaskMessage>();
var orchestratorMessages = new List <TaskMessage>();
var isCompleted = false;
var continuedAsNew = false;
var isInterrupted = false;
// correlation
CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext);
ExecutionStartedEvent continueAsNewExecutionStarted = null;
TaskMessage continuedAsNewMessage = null;
IList <HistoryEvent> carryOverEvents = null;
string carryOverStatus = null;
OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState;
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
OrchestrationRuntimeState originalOrchestrationRuntimeState = runtimeState;
OrchestrationState instanceState = null;
if (!ReconcileMessagesWithState(workItem))
{
// TODO : mark an orchestration as faulted if there is data corruption
TraceHelper.TraceSession(
TraceEventType.Error,
"TaskOrchestrationDispatcher-DeletedOrchestration",
runtimeState.OrchestrationInstance?.InstanceId,
"Received result for a deleted orchestration");
isCompleted = true;
}
else
{
do
{
continuedAsNew = false;
continuedAsNewMessage = null;
TraceHelper.TraceInstance(
TraceEventType.Verbose,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin",
runtimeState.OrchestrationInstance,
"Executing user orchestration: {0}",
DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true));
if (workItem.Cursor == null)
{
workItem.Cursor = await ExecuteOrchestrationAsync(runtimeState);
}
else
{
await ResumeOrchestrationAsync(workItem.Cursor);
}
IReadOnlyList <OrchestratorAction> decisions = workItem.Cursor.LatestDecisions.ToList();
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-End",
runtimeState.OrchestrationInstance,
"Executed user orchestration. Received {0} orchestrator actions: {1}",
decisions.Count,
string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType)));
foreach (OrchestratorAction decision in decisions)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ProcessOrchestratorAction",
runtimeState.OrchestrationInstance,
"Processing orchestrator action of type {0}",
decision.OrchestratorActionType);
switch (decision.OrchestratorActionType)
{
case OrchestratorActionType.ScheduleOrchestrator:
messagesToSend.Add(
ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState,
IncludeParameters));
break;
case OrchestratorActionType.CreateTimer:
var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision;
timerMessages.Add(ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState));
break;
case OrchestratorActionType.CreateSubOrchestration:
var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision;
orchestratorMessages.Add(
ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction,
runtimeState, IncludeParameters));
break;
case OrchestratorActionType.SendEvent:
var sendEventAction = (SendEventOrchestratorAction)decision;
orchestratorMessages.Add(
ProcessSendEventDecision(sendEventAction, runtimeState));
break;
case OrchestratorActionType.OrchestrationComplete:
OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision;
TaskMessage workflowInstanceCompletedMessage =
ProcessWorkflowCompletedTaskDecision(completeDecision, runtimeState, IncludeDetails, out continuedAsNew);
if (workflowInstanceCompletedMessage != null)
{
// Send complete message to parent workflow or to itself to start a new execution
// Store the event so we can rebuild the state
carryOverEvents = null;
if (continuedAsNew)
{
continuedAsNewMessage = workflowInstanceCompletedMessage;
continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent;
if (completeDecision.CarryoverEvents.Any())
{
carryOverEvents = completeDecision.CarryoverEvents.ToList();
completeDecision.CarryoverEvents.Clear();
}
}
else
{
orchestratorMessages.Add(workflowInstanceCompletedMessage);
}
}
isCompleted = !continuedAsNew;
break;
default:
throw TraceHelper.TraceExceptionInstance(
TraceEventType.Error,
"TaskOrchestrationDispatcher-UnsupportedDecisionType",
runtimeState.OrchestrationInstance,
new NotSupportedException("decision type not supported"));
}
// Underlying orchestration service provider may have a limit of messages per call, to avoid the situation
// we keep on asking the provider if message count is ok and stop processing new decisions if not.
//
// We also put in a fake timer to force next orchestration task for remaining messages
int totalMessages = messagesToSend.Count + orchestratorMessages.Count + timerMessages.Count;
if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState))
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-MaxMessageCountReached",
runtimeState.OrchestrationInstance,
"MaxMessageCount reached. Adding timer to process remaining events in next attempt.");
if (isCompleted || continuedAsNew)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted",
runtimeState.OrchestrationInstance,
"Orchestration already completed. Skip adding timer for splitting messages.");
break;
}
var dummyTimer = new CreateTimerOrchestratorAction
{
Id = FrameworkConstants.FakeTimerIdToSplitDecision,
FireAt = DateTime.UtcNow
};
timerMessages.Add(ProcessCreateTimerDecision(dummyTimer, runtimeState));
isInterrupted = true;
break;
}
}
// finish up processing of the work item
if (!continuedAsNew && runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted)
{
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
}
if (isCompleted)
{
TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state");
if (runtimeState.ExecutionStartedEvent != null)
{
instanceState = Utils.BuildOrchestrationState(runtimeState);
}
}
else
{
if (continuedAsNew)
{
TraceHelper.TraceSession(
TraceEventType.Information,
"TaskOrchestrationDispatcher-UpdatingStateForContinuation",
workItem.InstanceId,
"Updating state for continuation");
runtimeState = new OrchestrationRuntimeState();
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
runtimeState.AddEvent(continueAsNewExecutionStarted);
runtimeState.Status = workItem.OrchestrationRuntimeState.Status ?? carryOverStatus;
carryOverStatus = workItem.OrchestrationRuntimeState.Status;
if (carryOverEvents != null)
{
foreach (var historyEvent in carryOverEvents)
{
runtimeState.AddEvent(historyEvent);
}
}
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
workItem.OrchestrationRuntimeState = runtimeState;
TaskOrchestration newOrchestration = this.objectManager.GetObject(
runtimeState.Name,
continueAsNewExecutionStarted.Version);
workItem.Cursor = new OrchestrationExecutionCursor(
runtimeState,
newOrchestration,
new TaskOrchestrationExecutor(
runtimeState,
newOrchestration,
orchestrationService.EventBehaviourForContinueAsNew),
latestDecisions: null);
if (workItem.LockedUntilUtc < DateTime.UtcNow.AddMinutes(1))
{
await orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem);
}
}
instanceState = Utils.BuildOrchestrationState(runtimeState);
}
} while (continuedAsNew);
}
workItem.OrchestrationRuntimeState = originalOrchestrationRuntimeState;
runtimeState.Status = runtimeState.Status ?? carryOverStatus;
await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync(
workItem,
runtimeState,
continuedAsNew?null : messagesToSend,
orchestratorMessages,
continuedAsNew?null : timerMessages,
continuedAsNewMessage,
instanceState);
workItem.OrchestrationRuntimeState = runtimeState;
return(isCompleted || continuedAsNew || isInterrupted);
}
/// <summary>
/// Method to process a new work item
/// </summary>
/// <param name="workItem">The work item to process</param>
protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
var messagesToSend = new List <TaskMessage>();
var timerMessages = new List <TaskMessage>();
var subOrchestrationMessages = new List <TaskMessage>();
bool isCompleted = false;
bool continuedAsNew = false;
ExecutionStartedEvent continueAsNewExecutionStarted = null;
TaskMessage continuedAsNewMessage = null;
OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState;
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
if (!ReconcileMessagesWithState(workItem))
{
// TODO : mark an orchestration as faulted if there is data corruption
TraceHelper.TraceSession(
TraceEventType.Error,
"TaskOrchestrationDispatcher-DeletedOrchestration",
runtimeState.OrchestrationInstance?.InstanceId,
"Received result for a deleted orchestration");
isCompleted = true;
}
else
{
TraceHelper.TraceInstance(
TraceEventType.Verbose,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin",
runtimeState.OrchestrationInstance,
"Executing user orchestration: {0}",
DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true));
IList <OrchestratorAction> decisions = (await ExecuteOrchestrationAsync(runtimeState)).ToList();
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-End",
runtimeState.OrchestrationInstance,
"Executed user orchestration. Received {0} orchestrator actions: {1}",
decisions.Count(),
string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType)));
foreach (OrchestratorAction decision in decisions)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ProcessOrchestratorAction",
runtimeState.OrchestrationInstance,
"Processing orchestrator action of type {0}",
decision.OrchestratorActionType);
switch (decision.OrchestratorActionType)
{
case OrchestratorActionType.ScheduleOrchestrator:
messagesToSend.Add(
ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState,
IncludeParameters));
break;
case OrchestratorActionType.CreateTimer:
var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision;
timerMessages.Add(ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState));
break;
case OrchestratorActionType.CreateSubOrchestration:
var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision;
subOrchestrationMessages.Add(
ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction,
runtimeState, IncludeParameters));
break;
case OrchestratorActionType.OrchestrationComplete:
TaskMessage workflowInstanceCompletedMessage =
ProcessWorkflowCompletedTaskDecision((OrchestrationCompleteOrchestratorAction)decision, runtimeState, IncludeDetails, out continuedAsNew);
if (workflowInstanceCompletedMessage != null)
{
// Send complete message to parent workflow or to itself to start a new execution
// Store the event so we can rebuild the state
if (continuedAsNew)
{
continuedAsNewMessage = workflowInstanceCompletedMessage;
continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent;
}
else
{
subOrchestrationMessages.Add(workflowInstanceCompletedMessage);
}
}
isCompleted = !continuedAsNew;
break;
default:
throw TraceHelper.TraceExceptionInstance(
TraceEventType.Error,
"TaskOrchestrationDispatcher-UnsupportedDecisionType",
runtimeState.OrchestrationInstance,
new NotSupportedException("decision type not supported"));
}
// Underlying orchestration service provider may have a limit of messages per call, to avoid the situation
// we keep on asking the provider if message count is ok and stop processing new decisions if not.
//
// We also put in a fake timer to force next orchestration task for remaining messages
int totalMessages = messagesToSend.Count + subOrchestrationMessages.Count + timerMessages.Count;
if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState))
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-MaxMessageCountReached",
runtimeState.OrchestrationInstance,
"MaxMessageCount reached. Adding timer to process remaining events in next attempt.");
if (isCompleted || continuedAsNew)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted",
runtimeState.OrchestrationInstance,
"Orchestration already completed. Skip adding timer for splitting messages.");
break;
}
var dummyTimer = new CreateTimerOrchestratorAction
{
Id = FrameworkConstants.FakeTimerIdToSplitDecision,
FireAt = DateTime.UtcNow
};
timerMessages.Add(ProcessCreateTimerDecision(dummyTimer, runtimeState));
break;
}
}
}
// finish up processing of the work item
if (!continuedAsNew)
{
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
}
OrchestrationRuntimeState newOrchestrationRuntimeState = workItem.OrchestrationRuntimeState;
OrchestrationState instanceState = null;
if (isCompleted)
{
TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state");
if (newOrchestrationRuntimeState.ExecutionStartedEvent != null)
{
instanceState = Utils.BuildOrchestrationState(newOrchestrationRuntimeState);
}
newOrchestrationRuntimeState = null;
}
else
{
instanceState = Utils.BuildOrchestrationState(newOrchestrationRuntimeState);
if (continuedAsNew)
{
TraceHelper.TraceSession(
TraceEventType.Information,
"TaskOrchestrationDispatcher-UpdatingStateForContinuation",
workItem.InstanceId,
"Updating state for continuation");
newOrchestrationRuntimeState = new OrchestrationRuntimeState();
newOrchestrationRuntimeState.AddEvent(new OrchestratorStartedEvent(-1));
newOrchestrationRuntimeState.AddEvent(continueAsNewExecutionStarted);
newOrchestrationRuntimeState.AddEvent(new OrchestratorCompletedEvent(-1));
}
}
await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync(
workItem,
newOrchestrationRuntimeState,
continuedAsNew?null : messagesToSend,
subOrchestrationMessages,
continuedAsNew?null : timerMessages,
continuedAsNewMessage,
instanceState);
}
TaskMessage ProcessWorkflowCompletedTaskDecision(
OrchestrationCompleteOrchestratorAction completeOrchestratorAction,
OrchestrationRuntimeState runtimeState,
bool includeDetails,
out bool continuedAsNew)
{
ExecutionCompletedEvent executionCompletedEvent;
continuedAsNew = (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew);
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
{
executionCompletedEvent = new ContinueAsNewEvent(completeOrchestratorAction.Id,
completeOrchestratorAction.Result);
}
else
{
executionCompletedEvent = new ExecutionCompletedEvent(completeOrchestratorAction.Id,
completeOrchestratorAction.Result,
completeOrchestratorAction.OrchestrationStatus);
}
runtimeState.AddEvent(executionCompletedEvent);
this.logHelper.OrchestrationCompleted(runtimeState, completeOrchestratorAction);
TraceHelper.TraceInstance(
runtimeState.OrchestrationStatus == OrchestrationStatus.Failed ? TraceEventType.Warning : TraceEventType.Information,
"TaskOrchestrationDispatcher-InstanceCompleted",
runtimeState.OrchestrationInstance,
"Instance Id '{0}' completed in state {1} with result: {2}",
runtimeState.OrchestrationInstance,
runtimeState.OrchestrationStatus,
completeOrchestratorAction.Result);
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-InstanceCompletionEvents",
runtimeState.OrchestrationInstance,
() => Utils.EscapeJson(DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true)));
// Check to see if we need to start a new execution
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
{
var taskMessage = new TaskMessage();
var startedEvent = new ExecutionStartedEvent(-1, completeOrchestratorAction.Result)
{
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = runtimeState.OrchestrationInstance.InstanceId,
ExecutionId = Guid.NewGuid().ToString("N")
},
Tags = runtimeState.Tags,
ParentInstance = runtimeState.ParentInstance,
Name = runtimeState.Name,
Version = completeOrchestratorAction.NewVersion ?? runtimeState.Version
};
taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance;
taskMessage.Event = startedEvent;
return(taskMessage);
}
// If this is a Sub Orchestration, and not tagged as fire-and-forget,
// then notify the parent by sending a complete message
if (runtimeState.ParentInstance != null &&
!OrchestrationTags.IsTaggedAsFireAndForget(runtimeState.Tags))
{
var taskMessage = new TaskMessage();
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Completed)
{
var subOrchestrationCompletedEvent =
new SubOrchestrationInstanceCompletedEvent(-1, runtimeState.ParentInstance.TaskScheduleId,
completeOrchestratorAction.Result);
taskMessage.Event = subOrchestrationCompletedEvent;
}
else if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Failed ||
completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Terminated)
{
var subOrchestrationFailedEvent =
new SubOrchestrationInstanceFailedEvent(-1, runtimeState.ParentInstance.TaskScheduleId,
completeOrchestratorAction.Result,
includeDetails ? completeOrchestratorAction.Details : null);
taskMessage.Event = subOrchestrationFailedEvent;
}
if (taskMessage.Event != null)
{
taskMessage.OrchestrationInstance = runtimeState.ParentInstance.OrchestrationInstance;
return(taskMessage);
}
}
return(null);
}