public async Task Release(TaskOrchestrationWorkItem workItem)
{
var request = new TaskOrchestrationRequest
{
ReleaseRequest = new ReleaseTaskOrchestrationWorkItemRequest()
};
await _stream.RequestStream.WriteAsync(request);
var cts = new CancellationTokenSource(_releaseResponseTimeout);
if (!await _stream.ResponseStream.MoveNext(cts.Token))
{
throw new Exception("Session aborted");
}
if (_stream.ResponseStream.Current.MessageCase != TaskOrchestrationResponse.MessageOneofCase.ReleaseResponse)
{
throw new Exception("Unexpected response");
}
await _stream.RequestStream.CompleteAsync();
// Last read to close stream
await _stream.ResponseStream.MoveNext(cts.Token);
_stream.Dispose();
}
public async Task Renew(TaskOrchestrationWorkItem workItem)
{
var request = new TaskOrchestrationRequest
{
RenewRequest = new RenewTaskOrchestrationWorkItemLockRequest()
};
await _stream.RequestStream.WriteAsync(request);
var cts = new CancellationTokenSource(_renewResponseTimeout);
if (!await _stream.ResponseStream.MoveNext(cts.Token))
{
throw new Exception("Session aborted");
}
if (_stream.ResponseStream.Current.MessageCase != TaskOrchestrationResponse.MessageOneofCase.RenewResponse)
{
throw new Exception("Unexpected response");
}
var renewResponse = _stream.ResponseStream.Current.RenewResponse;
workItem.LockedUntilUtc = renewResponse.LockedUntilUtc.ToDateTime();
}
public async Task <IList <TaskMessage> > FetchNewOrchestrationMessagesAsync(
TaskOrchestrationWorkItem workItem)
{
var request = new TaskOrchestrationRequest
{
FetchRequest = new FetchNewOrchestrationMessagesRequest()
};
await _stream.RequestStream.WriteAsync(request);
var cts = new CancellationTokenSource(_fetchResponseTimeout);
if (!await _stream.ResponseStream.MoveNext(cts.Token))
{
throw new Exception("Session aborted");
}
if (_stream.ResponseStream.Current.MessageCase != TaskOrchestrationResponse.MessageOneofCase.FetchResponse)
{
throw new Exception("Unexpected response");
}
var fetchResponse = _stream.ResponseStream.Current.FetchResponse;
if (fetchResponse.NewMessages == null)
{
return(null);
}
return(fetchResponse.NewMessages.Messages
.Select(x => _options.DataConverter.Deserialize <TaskMessage>(x))
.ToArray());
}
Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem)
{
OrchestrationExecutionCursor cursor = workItem.Cursor;
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance);
dispatchContext.SetProperty(cursor.TaskOrchestration);
dispatchContext.SetProperty(cursor.RuntimeState);
dispatchContext.SetProperty(workItem);
cursor.LatestDecisions = Enumerable.Empty <OrchestratorAction>();
return(this.dispatchPipeline.RunAsync(dispatchContext, _ => {
cursor.LatestDecisions = cursor.OrchestrationExecutor.ExecuteNewEvents();
return CompletedTask;
}));
}
public async Task Complete(
TaskOrchestrationWorkItem workItem,
OrchestrationRuntimeState newOrchestrationRuntimeState,
IList <TaskMessage> outboundMessages,
IList <TaskMessage> orchestratorMessages,
IList <TaskMessage> timerMessages,
TaskMessage continuedAsNewMessage,
OrchestrationState orchestrationState)
{
var request = new TaskOrchestrationRequest
{
CompleteRequest = new CompleteTaskOrchestrationWorkItemRequest
{
NewEvents = { workItem.OrchestrationRuntimeState.NewEvents.Select(_options.DataConverter.Serialize) },
NewStatus = workItem.OrchestrationRuntimeState.Status,
NewOrchestrationEvents = { newOrchestrationRuntimeState == workItem.OrchestrationRuntimeState
? Enumerable.Empty <string>()
: newOrchestrationRuntimeState.NewEvents.Select(_options.DataConverter.Serialize) },
NewOrchestrationStatus = newOrchestrationRuntimeState == workItem.OrchestrationRuntimeState
? null
: newOrchestrationRuntimeState.Status,
OutboundMessages = { outboundMessages.Select(_options.DataConverter.Serialize) },
OrchestratorMessages = { orchestratorMessages.Select(_options.DataConverter.Serialize) },
TimerMessages = { timerMessages.Select(_options.DataConverter.Serialize) },
ContinuedAsNewMessage = continuedAsNewMessage == null
? string.Empty
: _options.DataConverter.Serialize(continuedAsNewMessage),
OrchestrationState = _options.DataConverter.Serialize(orchestrationState)
}
};
await _stream.RequestStream.WriteAsync(request);
var cts = new CancellationTokenSource(_completeResponseTimeout);
if (!await _stream.ResponseStream.MoveNext(cts.Token))
{
throw new Exception("Session aborted");
}
if (_stream.ResponseStream.Current.MessageCase != TaskOrchestrationResponse.MessageOneofCase.CompleteResponse)
{
throw new Exception("Unexpected response");
}
}
public async Task CompleteTaskOrchestrationWorkItemAsync(
TaskOrchestrationWorkItem workItem,
OrchestrationRuntimeState newOrchestrationRuntimeState,
IList <TaskMessage> outboundMessages,
IList <TaskMessage> orchestratorMessages,
IList <TaskMessage> timerMessages,
TaskMessage continuedAsNewMessage,
OrchestrationState orchestrationState)
{
await(workItem.Session as GrpcClientOrchestrationSession).Complete(
workItem,
newOrchestrationRuntimeState,
outboundMessages,
orchestratorMessages,
timerMessages,
continuedAsNewMessage,
orchestrationState);
}
public async Task Abandon(TaskOrchestrationWorkItem workItem)
{
var request = new TaskOrchestrationRequest
{
AbandonRequest = new AbandonTaskOrchestrationWorkItemLockRequest()
};
await _stream.RequestStream.WriteAsync(request);
var cts = new CancellationTokenSource(_abandonResponseTimeout);
if (!await _stream.ResponseStream.MoveNext(cts.Token))
{
throw new Exception("Session aborted");
}
if (_stream.ResponseStream.Current.MessageCase != TaskOrchestrationResponse.MessageOneofCase.AbandonResponse)
{
throw new Exception("Unexpected response");
}
}
async Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem)
{
OrchestrationExecutionCursor cursor = workItem.Cursor;
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance);
dispatchContext.SetProperty(cursor.TaskOrchestration);
dispatchContext.SetProperty(cursor.RuntimeState);
dispatchContext.SetProperty(workItem);
cursor.LatestDecisions = Enumerable.Empty <OrchestratorAction>();
await this.dispatchPipeline.RunAsync(dispatchContext, _ =>
{
OrchestratorExecutionResult result = cursor.OrchestrationExecutor.ExecuteNewEvents();
dispatchContext.SetProperty(result);
return(CompletedTask);
});
var result = dispatchContext.GetProperty <OrchestratorExecutionResult>();
cursor.LatestDecisions = result?.Actions ?? Enumerable.Empty <OrchestratorAction>();
cursor.RuntimeState.Status = result?.CustomStatus;
}
async Task <OrchestrationExecutionCursor> ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem)
{
TaskOrchestration taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version);
if (taskOrchestration == null)
{
throw TraceHelper.TraceExceptionInstance(
TraceEventType.Error,
"TaskOrchestrationDispatcher-TypeMissing",
runtimeState.OrchestrationInstance,
new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})"));
}
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(runtimeState.OrchestrationInstance);
dispatchContext.SetProperty(taskOrchestration);
dispatchContext.SetProperty(runtimeState);
dispatchContext.SetProperty(workItem);
var executor = new TaskOrchestrationExecutor(runtimeState, taskOrchestration, this.orchestrationService.EventBehaviourForContinueAsNew);
IEnumerable <OrchestratorAction> decisions = Enumerable.Empty <OrchestratorAction>();
await this.dispatchPipeline.RunAsync(dispatchContext, _ => {
decisions = executor.Execute();
return(CompletedTask);
});
return(new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions));
}
async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem)
{
try {
if (workItem.Session == null)
{
// Legacy behavior
await this.OnProcessWorkItemAsync(workItem);
return;
}
var isExtendedSession = false;
CorrelationTraceClient.Propagate(
() => {
// Check if it is extended session.
isExtendedSession = this.concurrentSessionLock.Acquire();
this.concurrentSessionLock.Release();
workItem.IsExtendedSession = isExtendedSession;
});
var processCount = 0;
try {
while (true)
{
// If the provider provided work items, execute them.
if (workItem.NewMessages?.Count > 0)
{
bool isCompletedOrInterrupted = await this.OnProcessWorkItemAsync(workItem);
if (isCompletedOrInterrupted)
{
break;
}
processCount++;
}
// Fetches beyond the first require getting an extended session lock, used to prevent starvation.
if (processCount > 0 && !isExtendedSession)
{
isExtendedSession = this.concurrentSessionLock.Acquire();
if (!isExtendedSession)
{
TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-MaxOperations", "Failed to acquire concurrent session lock.");
break;
}
}
TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-StartFetch", "Starting fetch of existing session.");
Stopwatch timer = Stopwatch.StartNew();
// Wait for new messages to arrive for the session. This call is expected to block (asynchronously)
// until either new messages are available or until a provider-specific timeout has expired.
workItem.NewMessages = await workItem.Session.FetchNewOrchestrationMessagesAsync(workItem);
if (workItem.NewMessages == null)
{
break;
}
TraceHelper.Trace(
TraceEventType.Verbose,
"OnProcessWorkItemSession-EndFetch",
$"Fetched {workItem.NewMessages.Count} new message(s) after {timer.ElapsedMilliseconds} ms from existing session.");
workItem.OrchestrationRuntimeState.NewEvents.Clear();
}
} finally {
if (isExtendedSession)
{
TraceHelper.Trace(
TraceEventType.Verbose,
"OnProcessWorkItemSession-Release",
$"Releasing extended session after {processCount} batch(es).");
this.concurrentSessionLock.Release();
}
}
} catch (SessionAbortedException e) {
// Either the orchestration or the orchestration service explicitly abandoned the session.
OrchestrationInstance instance = workItem.OrchestrationRuntimeState?.OrchestrationInstance ?? new OrchestrationInstance {
InstanceId = workItem.InstanceId
};
this.logHelper.OrchestrationAborted(instance, e.Message);
TraceHelper.TraceInstance(TraceEventType.Warning, "TaskOrchestrationDispatcher-ExecutionAborted", instance, "{0}", e.Message);
await this.orchestrationService.AbandonTaskOrchestrationWorkItemAsync(workItem);
}
}
public async Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
await(workItem.Session as GrpcClientOrchestrationSession).Abandon(workItem);
}
public async Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
await(workItem.Session as GrpcClientOrchestrationSession).Release(workItem);
}
static bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem)
{
foreach (TaskMessage message in workItem.NewMessages)
{
OrchestrationInstance orchestrationInstance = message.OrchestrationInstance;
if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
{
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskOrchestrationDispatcher-OrchestrationInstanceMissing",
new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
}
if (workItem.OrchestrationRuntimeState.Events.Count == 1 && message.Event.EventType != EventType.ExecutionStarted)
{
// we get here because of:
// i) responses for scheduled tasks after the orchestrations have been completed
// ii) responses for explicitly deleted orchestrations
return(false);
}
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ProcessEvent",
orchestrationInstance,
"Processing new event with Id {0} and type {1}",
message.Event.EventId,
message.Event.EventType);
if (message.Event.EventType == EventType.ExecutionStarted)
{
if (workItem.OrchestrationRuntimeState.Events.Count > 1)
{
// this was caused due to a dupe execution started event, swallow this one
TraceHelper.TraceInstance(
TraceEventType.Warning,
"TaskOrchestrationDispatcher-DuplicateStartEvent",
orchestrationInstance,
"Duplicate start event. Ignoring event with Id {0} and type {1} ",
message.Event.EventId,
message.Event.EventType);
continue;
}
}
else if (!string.IsNullOrWhiteSpace(orchestrationInstance.ExecutionId)
&&
!string.Equals(orchestrationInstance.ExecutionId,
workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId))
{
// eat up any events for previous executions
TraceHelper.TraceInstance(
TraceEventType.Warning,
"TaskOrchestrationDispatcher-ExecutionIdMismatch",
orchestrationInstance,
"ExecutionId of event does not match current executionId. Ignoring event with Id {0} and type {1} ",
message.Event.EventId,
message.Event.EventType);
continue;
}
workItem.OrchestrationRuntimeState.AddEvent(message.Event);
}
return(true);
}
/// <summary>
/// Method to process a new work item
/// </summary>
/// <param name="workItem">The work item to process</param>
protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
var messagesToSend = new List <TaskMessage>();
var timerMessages = new List <TaskMessage>();
var subOrchestrationMessages = new List <TaskMessage>();
bool isCompleted = false;
bool continuedAsNew = false;
ExecutionStartedEvent continueAsNewExecutionStarted = null;
TaskMessage continuedAsNewMessage = null;
OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState;
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
if (!ReconcileMessagesWithState(workItem))
{
// TODO : mark an orchestration as faulted if there is data corruption
TraceHelper.TraceSession(
TraceEventType.Error,
"TaskOrchestrationDispatcher-DeletedOrchestration",
runtimeState.OrchestrationInstance?.InstanceId,
"Received result for a deleted orchestration");
isCompleted = true;
}
else
{
TraceHelper.TraceInstance(
TraceEventType.Verbose,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin",
runtimeState.OrchestrationInstance,
"Executing user orchestration: {0}",
DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true));
IList <OrchestratorAction> decisions = (await ExecuteOrchestrationAsync(runtimeState)).ToList();
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-End",
runtimeState.OrchestrationInstance,
"Executed user orchestration. Received {0} orchestrator actions: {1}",
decisions.Count(),
string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType)));
foreach (OrchestratorAction decision in decisions)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ProcessOrchestratorAction",
runtimeState.OrchestrationInstance,
"Processing orchestrator action of type {0}",
decision.OrchestratorActionType);
switch (decision.OrchestratorActionType)
{
case OrchestratorActionType.ScheduleOrchestrator:
messagesToSend.Add(
ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState,
IncludeParameters));
break;
case OrchestratorActionType.CreateTimer:
var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision;
timerMessages.Add(ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState));
break;
case OrchestratorActionType.CreateSubOrchestration:
var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision;
subOrchestrationMessages.Add(
ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction,
runtimeState, IncludeParameters));
break;
case OrchestratorActionType.OrchestrationComplete:
TaskMessage workflowInstanceCompletedMessage =
ProcessWorkflowCompletedTaskDecision((OrchestrationCompleteOrchestratorAction)decision, runtimeState, IncludeDetails, out continuedAsNew);
if (workflowInstanceCompletedMessage != null)
{
// Send complete message to parent workflow or to itself to start a new execution
// Store the event so we can rebuild the state
if (continuedAsNew)
{
continuedAsNewMessage = workflowInstanceCompletedMessage;
continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent;
}
else
{
subOrchestrationMessages.Add(workflowInstanceCompletedMessage);
}
}
isCompleted = !continuedAsNew;
break;
default:
throw TraceHelper.TraceExceptionInstance(
TraceEventType.Error,
"TaskOrchestrationDispatcher-UnsupportedDecisionType",
runtimeState.OrchestrationInstance,
new NotSupportedException("decision type not supported"));
}
// Underlying orchestration service provider may have a limit of messages per call, to avoid the situation
// we keep on asking the provider if message count is ok and stop processing new decisions if not.
//
// We also put in a fake timer to force next orchestration task for remaining messages
int totalMessages = messagesToSend.Count + subOrchestrationMessages.Count + timerMessages.Count;
if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState))
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-MaxMessageCountReached",
runtimeState.OrchestrationInstance,
"MaxMessageCount reached. Adding timer to process remaining events in next attempt.");
if (isCompleted || continuedAsNew)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted",
runtimeState.OrchestrationInstance,
"Orchestration already completed. Skip adding timer for splitting messages.");
break;
}
var dummyTimer = new CreateTimerOrchestratorAction
{
Id = FrameworkConstants.FakeTimerIdToSplitDecision,
FireAt = DateTime.UtcNow
};
timerMessages.Add(ProcessCreateTimerDecision(dummyTimer, runtimeState));
break;
}
}
}
// finish up processing of the work item
if (!continuedAsNew)
{
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
}
OrchestrationRuntimeState newOrchestrationRuntimeState = workItem.OrchestrationRuntimeState;
OrchestrationState instanceState = null;
if (isCompleted)
{
TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state");
if (newOrchestrationRuntimeState.ExecutionStartedEvent != null)
{
instanceState = Utils.BuildOrchestrationState(newOrchestrationRuntimeState);
}
newOrchestrationRuntimeState = null;
}
else
{
instanceState = Utils.BuildOrchestrationState(newOrchestrationRuntimeState);
if (continuedAsNew)
{
TraceHelper.TraceSession(
TraceEventType.Information,
"TaskOrchestrationDispatcher-UpdatingStateForContinuation",
workItem.InstanceId,
"Updating state for continuation");
newOrchestrationRuntimeState = new OrchestrationRuntimeState();
newOrchestrationRuntimeState.AddEvent(new OrchestratorStartedEvent(-1));
newOrchestrationRuntimeState.AddEvent(continueAsNewExecutionStarted);
newOrchestrationRuntimeState.AddEvent(new OrchestratorCompletedEvent(-1));
}
}
await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync(
workItem,
newOrchestrationRuntimeState,
continuedAsNew?null : messagesToSend,
subOrchestrationMessages,
continuedAsNew?null : timerMessages,
continuedAsNewMessage,
instanceState);
}
async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem)
{
if (workItem.Session == null)
{
// Legacy behavior
await OnProcessWorkItemAsync(workItem);
return;
}
var isExtendedSession = false;
var processCount = 0;
try
{
while (true)
{
// If the provider provided work items, execute them.
if (workItem.NewMessages?.Count > 0)
{
bool isCompletedOrInterrupted = await OnProcessWorkItemAsync(workItem);
if (isCompletedOrInterrupted)
{
break;
}
processCount++;
}
// Fetches beyond the first require getting an extended session lock, used to prevent starvation.
if (processCount > 0 && !isExtendedSession)
{
isExtendedSession = this.concurrentSessionLock.Acquire();
if (!isExtendedSession)
{
TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-MaxOperations", "Failed to acquire concurrent session lock.");
break;
}
}
TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-StartFetch", "Starting fetch of existing session.");
Stopwatch timer = Stopwatch.StartNew();
// Wait for new messages to arrive for the session. This call is expected to block (asynchronously)
// until either new messages are available or until a provider-specific timeout has expired.
workItem.NewMessages = await workItem.Session.FetchNewOrchestrationMessagesAsync(workItem);
if (workItem.NewMessages == null)
{
break;
}
TraceHelper.Trace(
TraceEventType.Verbose,
"OnProcessWorkItemSession-EndFetch",
$"Fetched {workItem.NewMessages.Count} new message(s) after {timer.ElapsedMilliseconds} ms from existing session.");
workItem.OrchestrationRuntimeState.NewEvents.Clear();
}
}
finally
{
if (isExtendedSession)
{
TraceHelper.Trace(
TraceEventType.Verbose,
"OnProcessWorkItemSession-Release",
$"Releasing extended session after {processCount} batch(es).");
this.concurrentSessionLock.Release();
}
}
}
/// <summary>
/// Method to process a new work item
/// </summary>
/// <param name="workItem">The work item to process</param>
protected async Task <bool> OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
var messagesToSend = new List <TaskMessage>();
var timerMessages = new List <TaskMessage>();
var orchestratorMessages = new List <TaskMessage>();
var isCompleted = false;
var continuedAsNew = false;
var isInterrupted = false;
// correlation
CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext);
ExecutionStartedEvent continueAsNewExecutionStarted = null;
TaskMessage continuedAsNewMessage = null;
IList <HistoryEvent> carryOverEvents = null;
string carryOverStatus = null;
OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState;
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
OrchestrationRuntimeState originalOrchestrationRuntimeState = runtimeState;
OrchestrationState instanceState = null;
if (!ReconcileMessagesWithState(workItem))
{
// TODO : mark an orchestration as faulted if there is data corruption
TraceHelper.TraceSession(
TraceEventType.Error,
"TaskOrchestrationDispatcher-DeletedOrchestration",
runtimeState.OrchestrationInstance?.InstanceId,
"Received result for a deleted orchestration");
isCompleted = true;
}
else
{
do
{
continuedAsNew = false;
continuedAsNewMessage = null;
TraceHelper.TraceInstance(
TraceEventType.Verbose,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin",
runtimeState.OrchestrationInstance,
"Executing user orchestration: {0}",
DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true));
if (workItem.Cursor == null)
{
workItem.Cursor = await ExecuteOrchestrationAsync(runtimeState);
}
else
{
await ResumeOrchestrationAsync(workItem.Cursor);
}
IReadOnlyList <OrchestratorAction> decisions = workItem.Cursor.LatestDecisions.ToList();
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-End",
runtimeState.OrchestrationInstance,
"Executed user orchestration. Received {0} orchestrator actions: {1}",
decisions.Count,
string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType)));
foreach (OrchestratorAction decision in decisions)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ProcessOrchestratorAction",
runtimeState.OrchestrationInstance,
"Processing orchestrator action of type {0}",
decision.OrchestratorActionType);
switch (decision.OrchestratorActionType)
{
case OrchestratorActionType.ScheduleOrchestrator:
messagesToSend.Add(
ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState,
IncludeParameters));
break;
case OrchestratorActionType.CreateTimer:
var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision;
timerMessages.Add(ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState));
break;
case OrchestratorActionType.CreateSubOrchestration:
var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision;
orchestratorMessages.Add(
ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction,
runtimeState, IncludeParameters));
break;
case OrchestratorActionType.SendEvent:
var sendEventAction = (SendEventOrchestratorAction)decision;
orchestratorMessages.Add(
ProcessSendEventDecision(sendEventAction, runtimeState));
break;
case OrchestratorActionType.OrchestrationComplete:
OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision;
TaskMessage workflowInstanceCompletedMessage =
ProcessWorkflowCompletedTaskDecision(completeDecision, runtimeState, IncludeDetails, out continuedAsNew);
if (workflowInstanceCompletedMessage != null)
{
// Send complete message to parent workflow or to itself to start a new execution
// Store the event so we can rebuild the state
carryOverEvents = null;
if (continuedAsNew)
{
continuedAsNewMessage = workflowInstanceCompletedMessage;
continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent;
if (completeDecision.CarryoverEvents.Any())
{
carryOverEvents = completeDecision.CarryoverEvents.ToList();
completeDecision.CarryoverEvents.Clear();
}
}
else
{
orchestratorMessages.Add(workflowInstanceCompletedMessage);
}
}
isCompleted = !continuedAsNew;
break;
default:
throw TraceHelper.TraceExceptionInstance(
TraceEventType.Error,
"TaskOrchestrationDispatcher-UnsupportedDecisionType",
runtimeState.OrchestrationInstance,
new NotSupportedException("decision type not supported"));
}
// Underlying orchestration service provider may have a limit of messages per call, to avoid the situation
// we keep on asking the provider if message count is ok and stop processing new decisions if not.
//
// We also put in a fake timer to force next orchestration task for remaining messages
int totalMessages = messagesToSend.Count + orchestratorMessages.Count + timerMessages.Count;
if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState))
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-MaxMessageCountReached",
runtimeState.OrchestrationInstance,
"MaxMessageCount reached. Adding timer to process remaining events in next attempt.");
if (isCompleted || continuedAsNew)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted",
runtimeState.OrchestrationInstance,
"Orchestration already completed. Skip adding timer for splitting messages.");
break;
}
var dummyTimer = new CreateTimerOrchestratorAction
{
Id = FrameworkConstants.FakeTimerIdToSplitDecision,
FireAt = DateTime.UtcNow
};
timerMessages.Add(ProcessCreateTimerDecision(dummyTimer, runtimeState));
isInterrupted = true;
break;
}
}
// finish up processing of the work item
if (!continuedAsNew && runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted)
{
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
}
if (isCompleted)
{
TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state");
if (runtimeState.ExecutionStartedEvent != null)
{
instanceState = Utils.BuildOrchestrationState(runtimeState);
}
}
else
{
if (continuedAsNew)
{
TraceHelper.TraceSession(
TraceEventType.Information,
"TaskOrchestrationDispatcher-UpdatingStateForContinuation",
workItem.InstanceId,
"Updating state for continuation");
runtimeState = new OrchestrationRuntimeState();
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
runtimeState.AddEvent(continueAsNewExecutionStarted);
runtimeState.Status = workItem.OrchestrationRuntimeState.Status ?? carryOverStatus;
carryOverStatus = workItem.OrchestrationRuntimeState.Status;
if (carryOverEvents != null)
{
foreach (var historyEvent in carryOverEvents)
{
runtimeState.AddEvent(historyEvent);
}
}
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
workItem.OrchestrationRuntimeState = runtimeState;
TaskOrchestration newOrchestration = this.objectManager.GetObject(
runtimeState.Name,
continueAsNewExecutionStarted.Version);
workItem.Cursor = new OrchestrationExecutionCursor(
runtimeState,
newOrchestration,
new TaskOrchestrationExecutor(
runtimeState,
newOrchestration,
orchestrationService.EventBehaviourForContinueAsNew),
latestDecisions: null);
if (workItem.LockedUntilUtc < DateTime.UtcNow.AddMinutes(1))
{
await orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem);
}
}
instanceState = Utils.BuildOrchestrationState(runtimeState);
}
} while (continuedAsNew);
}
workItem.OrchestrationRuntimeState = originalOrchestrationRuntimeState;
runtimeState.Status = runtimeState.Status ?? carryOverStatus;
await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync(
workItem,
runtimeState,
continuedAsNew?null : messagesToSend,
orchestratorMessages,
continuedAsNew?null : timerMessages,
continuedAsNewMessage,
instanceState);
workItem.OrchestrationRuntimeState = runtimeState;
return(isCompleted || continuedAsNew || isInterrupted);
}
public async Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem)
{
await(workItem.Session as GrpcClientOrchestrationSession).Renew(workItem);
}
public override async Task LockNextTaskOrchestrationWorkItem(IAsyncStreamReader <TaskOrchestrationRequest> requestStream, IServerStreamWriter <TaskOrchestrationResponse> responseStream, ServerCallContext context)
{
try
{
TaskOrchestrationWorkItem workItem = null;
// Receive and reply each message
await foreach (var message in requestStream.ReadAllAsync(context.CancellationToken))
{
switch (message.MessageCase)
{
case TaskOrchestrationRequest.MessageOneofCase.LockRequest:
var lockRequest = message.LockRequest;
var orchestrations = lockRequest.Orchestrations.Select(x => new NameVersion(x.Name, x.Version)).ToArray();
workItem = await(lockRequest.AllOrchestrations
? _orchestrationService
.LockNextTaskOrchestrationWorkItemAsync(lockRequest.ReceiveTimeout.ToTimeSpan(), context.CancellationToken)
: (_extendedOrchestrationService ?? throw DistributedWorkersNotSupported())
.LockNextTaskOrchestrationWorkItemAsync(lockRequest.ReceiveTimeout.ToTimeSpan(), orchestrations, context.CancellationToken)
);
var lockResponse = new TaskOrchestrationResponse
{
LockResponse = new LockNextTaskOrchestrationWorkItemResponse
{
WorkItem = workItem == null ? null : new DurableTaskGrpc.TaskOrchestrationWorkItem
{
InstanceId = workItem.InstanceId,
LockedUntilUtc = Timestamp.FromDateTime(workItem.LockedUntilUtc),
Events = { workItem.OrchestrationRuntimeState.Events.Select(_options.DataConverter.Serialize) },
NewMessages = { workItem.NewMessages.Select(_options.DataConverter.Serialize) }
}
}
};
context.CancellationToken.ThrowIfCancellationRequested();
await responseStream.WriteAsync(lockResponse);
break;
case TaskOrchestrationRequest.MessageOneofCase.RenewRequest:
var renewRequest = message.RenewRequest;
await _orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem);
var renewResponse = new TaskOrchestrationResponse
{
RenewResponse = new RenewTaskOrchestrationWorkItemLockResponse
{
LockedUntilUtc = Timestamp.FromDateTime(workItem.LockedUntilUtc)
}
};
context.CancellationToken.ThrowIfCancellationRequested();
await responseStream.WriteAsync(renewResponse);
break;
case TaskOrchestrationRequest.MessageOneofCase.CompleteRequest:
var completeRequest = message.CompleteRequest;
var outboundMessages = completeRequest.OutboundMessages.Select(x => _options.DataConverter.Deserialize <TaskMessage>(x)).ToArray();
var timerMessages = completeRequest.TimerMessages.Select(x => _options.DataConverter.Deserialize <TaskMessage>(x)).ToArray();
var orchestratorMessages = completeRequest.OrchestratorMessages.Select(x => _options.DataConverter.Deserialize <TaskMessage>(x)).ToArray();
var continuedAsNewMessage = string.IsNullOrEmpty(completeRequest.ContinuedAsNewMessage)
? null
: _options.DataConverter.Deserialize <TaskMessage>(completeRequest.ContinuedAsNewMessage);
var newEvents = completeRequest.NewEvents.Select(x => _options.DataConverter.Deserialize <HistoryEvent>(x)).ToArray();
workItem.OrchestrationRuntimeState ??= new OrchestrationRuntimeState();
foreach (var newEvent in newEvents)
{
workItem.OrchestrationRuntimeState.AddEvent(newEvent);
}
workItem.OrchestrationRuntimeState.Status = completeRequest.NewStatus;
var newOrchestrationRuntimeState = workItem.OrchestrationRuntimeState;
var newOrchestrationRuntimeStateEvents = completeRequest.NewOrchestrationEvents.Select(x => _options.DataConverter.Deserialize <HistoryEvent>(x)).ToArray();
if (newOrchestrationRuntimeStateEvents.Length > 0)
{
newOrchestrationRuntimeState = new OrchestrationRuntimeState();
foreach (var newEvent in newOrchestrationRuntimeStateEvents)
{
newOrchestrationRuntimeState.AddEvent(newEvent);
}
newOrchestrationRuntimeState.Status = completeRequest.NewOrchestrationStatus;
}
var orchestrationState = Utils.BuildOrchestrationState(newOrchestrationRuntimeState);
await _orchestrationService.CompleteTaskOrchestrationWorkItemAsync(
workItem,
newOrchestrationRuntimeState,
outboundMessages,
orchestratorMessages,
timerMessages,
continuedAsNewMessage,
orchestrationState);
newOrchestrationRuntimeState.NewEvents.Clear();
workItem.OrchestrationRuntimeState = newOrchestrationRuntimeState;
context.CancellationToken.ThrowIfCancellationRequested();
await responseStream.WriteAsync(new TaskOrchestrationResponse
{
CompleteResponse = new CompleteTaskOrchestrationWorkItemResponse()
});
break;
case TaskOrchestrationRequest.MessageOneofCase.FetchRequest:
var fetchRequest = message.FetchRequest;
if (workItem.Session == null)
{
var fetchResponse = new TaskOrchestrationResponse
{
FetchResponse = new FetchNewOrchestrationMessagesResponse
{
NewMessages = null
}
};
context.CancellationToken.ThrowIfCancellationRequested();
await responseStream.WriteAsync(fetchResponse);
}
else
{
var newMessages = await workItem.Session.FetchNewOrchestrationMessagesAsync(workItem);
var fetchResponse = new TaskOrchestrationResponse
{
FetchResponse = new FetchNewOrchestrationMessagesResponse
{
NewMessages = newMessages == null ? null : new OrchestrationMessages
{
Messages = { newMessages.Select(_options.DataConverter.Serialize) }
}
}
};
context.CancellationToken.ThrowIfCancellationRequested();
await responseStream.WriteAsync(fetchResponse);
}
break;
case TaskOrchestrationRequest.MessageOneofCase.ReleaseRequest:
var releaseRequest = message.ReleaseRequest;
await _orchestrationService.ReleaseTaskOrchestrationWorkItemAsync(workItem);
context.CancellationToken.ThrowIfCancellationRequested();
await responseStream.WriteAsync(new TaskOrchestrationResponse
{
ReleaseResponse = new ReleaseTaskOrchestrationWorkItemResponse()
});
break;
case TaskOrchestrationRequest.MessageOneofCase.AbandonRequest:
var abandonRequest = message.AbandonRequest;
await _orchestrationService.AbandonTaskOrchestrationWorkItemAsync(workItem);
context.CancellationToken.ThrowIfCancellationRequested();
await responseStream.WriteAsync(new TaskOrchestrationResponse
{
AbandonResponse = new AbandonTaskOrchestrationWorkItemLockResponse()
});
break;
}
}
}
catch (OperationCanceledException) when(context.CancellationToken.IsCancellationRequested)
{
// Avoid exceptions when clients cancel request
}
}
async Task <OrchestrationExecutionCursor> ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem)
{
// Get the TaskOrchestration implementation. If it's not found, it either means that the developer never
// registered it (which is an error, and we'll throw for this further down) or it could be that some custom
// middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic.
TaskOrchestration?taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version !);
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(runtimeState.OrchestrationInstance);
dispatchContext.SetProperty(taskOrchestration);
dispatchContext.SetProperty(runtimeState);
dispatchContext.SetProperty(workItem);
TaskOrchestrationExecutor?executor = null;
await this.dispatchPipeline.RunAsync(dispatchContext, _ =>
{
// Check to see if the custom middleware intercepted and substituted the orchestration execution
// with its own execution behavior, providing us with the end results. If so, we can terminate
// the dispatch pipeline here.
var resultFromMiddleware = dispatchContext.GetProperty <OrchestratorExecutionResult>();
if (resultFromMiddleware != null)
{
return(CompletedTask);
}
if (taskOrchestration == null)
{
throw TraceHelper.TraceExceptionInstance(
TraceEventType.Error,
"TaskOrchestrationDispatcher-TypeMissing",
runtimeState.OrchestrationInstance,
new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})"));
}
executor = new TaskOrchestrationExecutor(
runtimeState,
taskOrchestration,
this.orchestrationService.EventBehaviourForContinueAsNew,
this.errorPropagationMode);
OrchestratorExecutionResult resultFromOrchestrator = executor.Execute();
dispatchContext.SetProperty(resultFromOrchestrator);
return(CompletedTask);
});
var result = dispatchContext.GetProperty <OrchestratorExecutionResult>();
IEnumerable <OrchestratorAction> decisions = result?.Actions ?? Enumerable.Empty <OrchestratorAction>();
runtimeState.Status = result?.CustomStatus;
return(new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions));
}
