public void FailOrchestration(Exception failure)
{
if (failure == null)
{
throw new ArgumentNullException(nameof(failure));
}
string reason = failure.Message;
string details;
// correlation
CorrelationTraceClient.Propagate(
() =>
{
CorrelationTraceClient.TrackException(failure);
});
if (failure is OrchestrationFailureException orchestrationFailureException)
{
details = orchestrationFailureException.Details;
}
else
{
details = $"Unhandled exception while executing orchestration: {failure}\n\t{failure.StackTrace}";
}
CompleteOrchestration(reason, details, OrchestrationStatus.Failed);
}
void CreateAndTrackDependencyTelemetry(TraceContextBase requestTraceContext)
{
TraceContextBase dependencyTraceContext = TraceContextFactory.Create(TraceConstants.Client);
dependencyTraceContext.TelemetryType = TelemetryType.Dependency;
dependencyTraceContext.SetParentAndStart(requestTraceContext);
CorrelationTraceContext.Current = dependencyTraceContext;
// Correlation
CorrelationTraceClient.TrackDepencencyTelemetry(dependencyTraceContext);
CorrelationTraceClient.TrackRequestTelemetry(requestTraceContext);
}
public void FailOrchestration(Exception failure)
{
if (failure == null)
{
throw new ArgumentNullException(nameof(failure));
}
string reason = failure.Message;
// string details is legacy, FailureDetails is the newer way to share failure information
string details = null;
FailureDetails failureDetails = null;
// correlation
CorrelationTraceClient.Propagate(
() =>
{
CorrelationTraceClient.TrackException(failure);
});
if (failure is OrchestrationFailureException orchestrationFailureException)
{
if (this.ErrorPropagationMode == ErrorPropagationMode.UseFailureDetails)
{
// When not serializing exceptions, we instead construct FailureDetails objects
failureDetails = orchestrationFailureException.FailureDetails;
}
else
{
details = orchestrationFailureException.Details;
}
}
else
{
if (this.ErrorPropagationMode == ErrorPropagationMode.UseFailureDetails)
{
failureDetails = new FailureDetails(failure);
}
else
{
details = $"Unhandled exception while executing orchestration: {failure}\n\t{failure.StackTrace}";
}
}
CompleteOrchestration(reason, details, OrchestrationStatus.Failed, failureDetails);
}
/// <summary>
/// Method to process a new work item
/// </summary>
/// <param name="workItem">The work item to process</param>
protected async Task <bool> OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
var messagesToSend = new List <TaskMessage>();
var timerMessages = new List <TaskMessage>();
var orchestratorMessages = new List <TaskMessage>();
var isCompleted = false;
var continuedAsNew = false;
var isInterrupted = false;
// correlation
CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext);
ExecutionStartedEvent continueAsNewExecutionStarted = null;
TaskMessage continuedAsNewMessage = null;
IList <HistoryEvent> carryOverEvents = null;
string carryOverStatus = null;
OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState;
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
OrchestrationRuntimeState originalOrchestrationRuntimeState = runtimeState;
OrchestrationState instanceState = null;
if (!ReconcileMessagesWithState(workItem))
{
// TODO : mark an orchestration as faulted if there is data corruption
TraceHelper.TraceSession(
TraceEventType.Error,
"TaskOrchestrationDispatcher-DeletedOrchestration",
runtimeState.OrchestrationInstance?.InstanceId,
"Received result for a deleted orchestration");
isCompleted = true;
}
else
{
do
{
continuedAsNew = false;
continuedAsNewMessage = null;
TraceHelper.TraceInstance(
TraceEventType.Verbose,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin",
runtimeState.OrchestrationInstance,
"Executing user orchestration: {0}",
DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true));
if (workItem.Cursor == null)
{
workItem.Cursor = await ExecuteOrchestrationAsync(runtimeState);
}
else
{
await ResumeOrchestrationAsync(workItem.Cursor);
}
IReadOnlyList <OrchestratorAction> decisions = workItem.Cursor.LatestDecisions.ToList();
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-End",
runtimeState.OrchestrationInstance,
"Executed user orchestration. Received {0} orchestrator actions: {1}",
decisions.Count,
string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType)));
foreach (OrchestratorAction decision in decisions)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ProcessOrchestratorAction",
runtimeState.OrchestrationInstance,
"Processing orchestrator action of type {0}",
decision.OrchestratorActionType);
switch (decision.OrchestratorActionType)
{
case OrchestratorActionType.ScheduleOrchestrator:
messagesToSend.Add(
ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState,
IncludeParameters));
break;
case OrchestratorActionType.CreateTimer:
var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision;
timerMessages.Add(ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState));
break;
case OrchestratorActionType.CreateSubOrchestration:
var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision;
orchestratorMessages.Add(
ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction,
runtimeState, IncludeParameters));
break;
case OrchestratorActionType.SendEvent:
var sendEventAction = (SendEventOrchestratorAction)decision;
orchestratorMessages.Add(
ProcessSendEventDecision(sendEventAction, runtimeState));
break;
case OrchestratorActionType.OrchestrationComplete:
OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision;
TaskMessage workflowInstanceCompletedMessage =
ProcessWorkflowCompletedTaskDecision(completeDecision, runtimeState, IncludeDetails, out continuedAsNew);
if (workflowInstanceCompletedMessage != null)
{
// Send complete message to parent workflow or to itself to start a new execution
// Store the event so we can rebuild the state
carryOverEvents = null;
if (continuedAsNew)
{
continuedAsNewMessage = workflowInstanceCompletedMessage;
continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent;
if (completeDecision.CarryoverEvents.Any())
{
carryOverEvents = completeDecision.CarryoverEvents.ToList();
completeDecision.CarryoverEvents.Clear();
}
}
else
{
orchestratorMessages.Add(workflowInstanceCompletedMessage);
}
}
isCompleted = !continuedAsNew;
break;
default:
throw TraceHelper.TraceExceptionInstance(
TraceEventType.Error,
"TaskOrchestrationDispatcher-UnsupportedDecisionType",
runtimeState.OrchestrationInstance,
new NotSupportedException("decision type not supported"));
}
// Underlying orchestration service provider may have a limit of messages per call, to avoid the situation
// we keep on asking the provider if message count is ok and stop processing new decisions if not.
//
// We also put in a fake timer to force next orchestration task for remaining messages
int totalMessages = messagesToSend.Count + orchestratorMessages.Count + timerMessages.Count;
if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState))
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-MaxMessageCountReached",
runtimeState.OrchestrationInstance,
"MaxMessageCount reached. Adding timer to process remaining events in next attempt.");
if (isCompleted || continuedAsNew)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted",
runtimeState.OrchestrationInstance,
"Orchestration already completed. Skip adding timer for splitting messages.");
break;
}
var dummyTimer = new CreateTimerOrchestratorAction
{
Id = FrameworkConstants.FakeTimerIdToSplitDecision,
FireAt = DateTime.UtcNow
};
timerMessages.Add(ProcessCreateTimerDecision(dummyTimer, runtimeState));
isInterrupted = true;
break;
}
}
// finish up processing of the work item
if (!continuedAsNew && runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted)
{
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
}
if (isCompleted)
{
TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state");
if (runtimeState.ExecutionStartedEvent != null)
{
instanceState = Utils.BuildOrchestrationState(runtimeState);
}
}
else
{
if (continuedAsNew)
{
TraceHelper.TraceSession(
TraceEventType.Information,
"TaskOrchestrationDispatcher-UpdatingStateForContinuation",
workItem.InstanceId,
"Updating state for continuation");
runtimeState = new OrchestrationRuntimeState();
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
runtimeState.AddEvent(continueAsNewExecutionStarted);
runtimeState.Status = workItem.OrchestrationRuntimeState.Status ?? carryOverStatus;
carryOverStatus = workItem.OrchestrationRuntimeState.Status;
if (carryOverEvents != null)
{
foreach (var historyEvent in carryOverEvents)
{
runtimeState.AddEvent(historyEvent);
}
}
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
workItem.OrchestrationRuntimeState = runtimeState;
TaskOrchestration newOrchestration = this.objectManager.GetObject(
runtimeState.Name,
continueAsNewExecutionStarted.Version);
workItem.Cursor = new OrchestrationExecutionCursor(
runtimeState,
newOrchestration,
new TaskOrchestrationExecutor(
runtimeState,
newOrchestration,
orchestrationService.EventBehaviourForContinueAsNew),
latestDecisions: null);
if (workItem.LockedUntilUtc < DateTime.UtcNow.AddMinutes(1))
{
await orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem);
}
}
instanceState = Utils.BuildOrchestrationState(runtimeState);
}
} while (continuedAsNew);
}
workItem.OrchestrationRuntimeState = originalOrchestrationRuntimeState;
runtimeState.Status = runtimeState.Status ?? carryOverStatus;
await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync(
workItem,
runtimeState,
continuedAsNew?null : messagesToSend,
orchestratorMessages,
continuedAsNew?null : timerMessages,
continuedAsNewMessage,
instanceState);
workItem.OrchestrationRuntimeState = runtimeState;
return(isCompleted || continuedAsNew || isInterrupted);
}
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
{
Task renewTask = null;
var renewCancellationTokenSource = new CancellationTokenSource();
TaskMessage taskMessage = workItem.TaskMessage;
OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
TaskScheduledEvent scheduledEvent = null;
Activity diagnosticActivity = null;
try
{
if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received a message that does not have any OrchestrationInstance information.");
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingOrchestrationInstance",
new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
}
if (taskMessage.Event.EventType != EventType.TaskScheduled)
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported.");
throw TraceHelper.TraceException(
TraceEventType.Critical,
"TaskActivityDispatcher-UnsupportedEventType",
new NotSupportedException("Activity worker does not support event of type: " +
taskMessage.Event.EventType));
}
// call and get return message
scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent);
TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
if (taskActivity == null)
{
throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
}
if (workItem.LockedUntilUtc < DateTime.MaxValue)
{
// start a task to run RenewUntil
renewTask = Task.Factory.StartNew(
() => this.RenewUntil(workItem, renewCancellationTokenSource.Token),
renewCancellationTokenSource.Token);
}
// TODO : pass workflow instance data
var context = new TaskContext(taskMessage.OrchestrationInstance);
HistoryEvent eventToRespond = null;
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
dispatchContext.SetProperty(taskActivity);
dispatchContext.SetProperty(scheduledEvent);
// correlation
CorrelationTraceClient.Propagate(() =>
{
workItem.TraceContextBase?.SetActivityToCurrent();
diagnosticActivity = workItem.TraceContextBase?.CurrentActivity;
});
await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
{
try
{
string output = await taskActivity.RunAsync(context, scheduledEvent.Input);
eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
}
catch (TaskFailureException e)
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
string details = this.IncludeDetails ? e.Details : null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e);
CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e));
}
catch (Exception e) when(!Utils.IsFatal(e) && !Utils.IsExecutionAborting(e))
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
string details = this.IncludeDetails
? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}"
: null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e);
}
if (eventToRespond is TaskCompletedEvent completedEvent)
{
this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent);
}
});
var responseTaskMessage = new TaskMessage
{
Event = eventToRespond,
OrchestrationInstance = orchestrationInstance
};
await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
}
catch (SessionAbortedException e)
{
// The activity aborted its execution
this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message);
TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent.Name, e.Message);
await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem);
}
finally
{
diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out.
if (renewTask != null)
{
renewCancellationTokenSource.Cancel();
try
{
// wait the renewTask finish
await renewTask;
}
catch (OperationCanceledException)
{
// ignore
}
}
}
}
async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem)
{
try {
if (workItem.Session == null)
{
// Legacy behavior
await this.OnProcessWorkItemAsync(workItem);
return;
}
var isExtendedSession = false;
CorrelationTraceClient.Propagate(
() => {
// Check if it is extended session.
isExtendedSession = this.concurrentSessionLock.Acquire();
this.concurrentSessionLock.Release();
workItem.IsExtendedSession = isExtendedSession;
});
var processCount = 0;
try {
while (true)
{
// If the provider provided work items, execute them.
if (workItem.NewMessages?.Count > 0)
{
bool isCompletedOrInterrupted = await this.OnProcessWorkItemAsync(workItem);
if (isCompletedOrInterrupted)
{
break;
}
processCount++;
}
// Fetches beyond the first require getting an extended session lock, used to prevent starvation.
if (processCount > 0 && !isExtendedSession)
{
isExtendedSession = this.concurrentSessionLock.Acquire();
if (!isExtendedSession)
{
TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-MaxOperations", "Failed to acquire concurrent session lock.");
break;
}
}
TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-StartFetch", "Starting fetch of existing session.");
Stopwatch timer = Stopwatch.StartNew();
// Wait for new messages to arrive for the session. This call is expected to block (asynchronously)
// until either new messages are available or until a provider-specific timeout has expired.
workItem.NewMessages = await workItem.Session.FetchNewOrchestrationMessagesAsync(workItem);
if (workItem.NewMessages == null)
{
break;
}
TraceHelper.Trace(
TraceEventType.Verbose,
"OnProcessWorkItemSession-EndFetch",
$"Fetched {workItem.NewMessages.Count} new message(s) after {timer.ElapsedMilliseconds} ms from existing session.");
workItem.OrchestrationRuntimeState.NewEvents.Clear();
}
} finally {
if (isExtendedSession)
{
TraceHelper.Trace(
TraceEventType.Verbose,
"OnProcessWorkItemSession-Release",
$"Releasing extended session after {processCount} batch(es).");
this.concurrentSessionLock.Release();
}
}
} catch (SessionAbortedException e) {
// Either the orchestration or the orchestration service explicitly abandoned the session.
OrchestrationInstance instance = workItem.OrchestrationRuntimeState?.OrchestrationInstance ?? new OrchestrationInstance {
InstanceId = workItem.InstanceId
};
this.logHelper.OrchestrationAborted(instance, e.Message);
TraceHelper.TraceInstance(TraceEventType.Warning, "TaskOrchestrationDispatcher-ExecutionAborted", instance, "{0}", e.Message);
await this.orchestrationService.AbandonTaskOrchestrationWorkItemAsync(workItem);
}
}
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
{
Task?renewTask = null;
using var renewCancellationTokenSource = new CancellationTokenSource();
TaskMessage taskMessage = workItem.TaskMessage;
OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
TaskScheduledEvent? scheduledEvent = null;
Activity?diagnosticActivity = null;
try
{
if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received a message that does not have any OrchestrationInstance information.");
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingOrchestrationInstance",
new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
}
if (taskMessage.Event.EventType != EventType.TaskScheduled)
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported.");
throw TraceHelper.TraceException(
TraceEventType.Critical,
"TaskActivityDispatcher-UnsupportedEventType",
new NotSupportedException("Activity worker does not support event of type: " +
taskMessage.Event.EventType));
}
scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
if (scheduledEvent.Name == null)
{
string message = $"The activity worker received a {nameof(EventType.TaskScheduled)} event that does not specify an activity name.";
this.logHelper.TaskActivityDispatcherError(workItem, message);
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingActivityName",
new InvalidOperationException(message));
}
this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent);
TaskActivity?taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
if (workItem.LockedUntilUtc < DateTime.MaxValue)
{
// start a task to run RenewUntil
renewTask = Task.Factory.StartNew(
() => this.RenewUntil(workItem, renewCancellationTokenSource.Token),
renewCancellationTokenSource.Token);
}
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
dispatchContext.SetProperty(taskActivity);
dispatchContext.SetProperty(scheduledEvent);
// correlation
CorrelationTraceClient.Propagate(() =>
{
workItem.TraceContextBase?.SetActivityToCurrent();
diagnosticActivity = workItem.TraceContextBase?.CurrentActivity;
});
ActivityExecutionResult?result;
try
{
await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
{
if (taskActivity == null)
{
// This likely indicates a deployment error of some kind. Because these unhandled exceptions are
// automatically retried, resolving this may require redeploying the app code so that the activity exists again.
// CONSIDER: Should this be changed into a permanent error that fails the orchestration? Perhaps
// the app owner doesn't care to preserve existing instances when doing code deployments?
throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
}
var context = new TaskContext(taskMessage.OrchestrationInstance);
context.ErrorPropagationMode = this.errorPropagationMode;
HistoryEvent?responseEvent;
try
{
string?output = await taskActivity.RunAsync(context, scheduledEvent.Input);
responseEvent = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
}
catch (Exception e) when(e is not TaskFailureException && !Utils.IsFatal(e) && !Utils.IsExecutionAborting(e))
{
// These are unexpected exceptions that occur in the task activity abstraction. Normal exceptions from
// activities are expected to be translated into TaskFailureException and handled outside the middleware
// context (see further below).
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
string?details = this.IncludeDetails
? $"Unhandled exception while executing task: {e}"
: null;
responseEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, new FailureDetails(e));
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)responseEvent, e);
}
var result = new ActivityExecutionResult {
ResponseEvent = responseEvent
};
dispatchContext.SetProperty(result);
});
result = dispatchContext.GetProperty <ActivityExecutionResult>();
}
catch (TaskFailureException e)
{
// These are normal task activity failures. They can come from Activity implementations or from middleware.
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
string?details = this.IncludeDetails ? e.Details : null;
var failureEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, e.FailureDetails);
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, failureEvent, e);
CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e));
result = new ActivityExecutionResult {
ResponseEvent = failureEvent
};
}
catch (Exception middlewareException) when(!Utils.IsFatal(middlewareException))
{
// These are considered retriable
this.logHelper.TaskActivityDispatcherError(workItem, $"Unhandled exception in activity middleware pipeline: {middlewareException}");
throw;
}
HistoryEvent?eventToRespond = result?.ResponseEvent;
if (eventToRespond is TaskCompletedEvent completedEvent)
{
this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent);
}
else if (eventToRespond is null)
{
// Default response if middleware prevents a response from being generated
eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, null);
}
var responseTaskMessage = new TaskMessage
{
Event = eventToRespond,
OrchestrationInstance = orchestrationInstance
};
await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
}
catch (SessionAbortedException e)
{
// The activity aborted its execution
this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message);
TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent?.Name, e.Message);
await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem);
}
finally
{
diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out.
if (renewTask != null)
{
renewCancellationTokenSource.Cancel();
try
{
// wait the renewTask finish
await renewTask;
}
catch (OperationCanceledException)
{
// ignore
}
}
}
}
async Task <OrchestrationInstance> InternalCreateOrchestrationInstanceWithRaisedEventAsync(
string orchestrationName,
string orchestrationVersion,
string orchestrationInstanceId,
object orchestrationInput,
IDictionary <string, string> orchestrationTags,
OrchestrationStatus[] dedupeStatuses,
string eventName,
object eventData,
DateTime?startAt = null)
{
TraceContextBase requestTraceContext = null;
// correlation
CorrelationTraceClient.Propagate(() => { requestTraceContext = this.CreateOrExtractRequestTraceContext(eventName); });
if (string.IsNullOrWhiteSpace(orchestrationInstanceId))
{
orchestrationInstanceId = Guid.NewGuid().ToString("N");
}
var orchestrationInstance = new OrchestrationInstance
{
InstanceId = orchestrationInstanceId,
ExecutionId = Guid.NewGuid().ToString("N"),
};
string serializedOrchestrationData = this.defaultConverter.Serialize(orchestrationInput);
var startedEvent = new ExecutionStartedEvent(-1, serializedOrchestrationData)
{
Tags = orchestrationTags,
Name = orchestrationName,
Version = orchestrationVersion,
OrchestrationInstance = orchestrationInstance,
ScheduledStartTime = startAt
};
var startMessage = new TaskMessage
{
OrchestrationInstance = orchestrationInstance,
Event = startedEvent
};
this.logHelper.SchedulingOrchestration(startedEvent);
CorrelationTraceClient.Propagate(() => this.CreateAndTrackDependencyTelemetry(requestTraceContext));
// Raised events and create orchestration calls use different methods so get handled separately
await this.ServiceClient.CreateTaskOrchestrationAsync(startMessage, dedupeStatuses);
if (eventData != null)
{
string serializedEventData = this.defaultConverter.Serialize(eventData);
var eventRaisedEvent = new EventRaisedEvent(-1, serializedEventData)
{
Name = eventName
};
this.logHelper.RaisingEvent(orchestrationInstance, eventRaisedEvent);
var eventMessage = new TaskMessage
{
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = orchestrationInstanceId,
// to ensure that the event gets raised on the running
// orchestration instance, null the execution id
// so that it will find out which execution
// it should use for processing
ExecutionId = null
},
Event = eventRaisedEvent,
};
await this.ServiceClient.SendTaskOrchestrationMessageAsync(eventMessage);
}
return(orchestrationInstance);
}
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
{
Task renewTask = null;
var renewCancellationTokenSource = new CancellationTokenSource();
try
{
TaskMessage taskMessage = workItem.TaskMessage;
OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
{
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingOrchestrationInstance",
new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
}
if (taskMessage.Event.EventType != EventType.TaskScheduled)
{
throw TraceHelper.TraceException(
TraceEventType.Critical,
"TaskActivityDispatcher-UnsupportedEventType",
new NotSupportedException("Activity worker does not support event of type: " +
taskMessage.Event.EventType));
}
// call and get return message
var scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
if (taskActivity == null)
{
throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
}
renewTask = Task.Factory.StartNew(() => RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token);
// TODO : pass workflow instance data
var context = new TaskContext(taskMessage.OrchestrationInstance);
HistoryEvent eventToRespond = null;
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
dispatchContext.SetProperty(taskActivity);
dispatchContext.SetProperty(scheduledEvent);
await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
{
// correlation
CorrelationTraceClient.Propagate(() => workItem.TraceContextBase?.SetActivityToCurrent());
try
{
string output = await taskActivity.RunAsync(context, scheduledEvent.Input);
eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
}
catch (TaskFailureException e)
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
string details = IncludeDetails ? e.Details : null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e));
}
catch (Exception e) when(!Utils.IsFatal(e))
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
string details = IncludeDetails
? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}"
: null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
}
});
var responseTaskMessage = new TaskMessage
{
Event = eventToRespond,
OrchestrationInstance = orchestrationInstance
};
await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
}
finally
{
if (renewTask != null)
{
renewCancellationTokenSource.Cancel();
try
{
await renewTask;
}
catch (OperationCanceledException)
{
// ignore
}
}
}
}
async Task <OrchestrationInstance> InternalCreateOrchestrationInstanceWithRaisedEventAsync(
string orchestrationName,
string orchestrationVersion,
string orchestrationInstanceId,
object orchestrationInput,
IDictionary <string, string> orchestrationTags,
OrchestrationStatus[] dedupeStatuses,
string eventName,
object eventData)
{
TraceContextBase requestTraceContext = null;
// correlation
CorrelationTraceClient.Propagate(() => { requestTraceContext = CreateOrExtractRequestTraceContext(eventName); });
if (string.IsNullOrWhiteSpace(orchestrationInstanceId))
{
orchestrationInstanceId = Guid.NewGuid().ToString("N");
}
var orchestrationInstance = new OrchestrationInstance
{
InstanceId = orchestrationInstanceId,
ExecutionId = Guid.NewGuid().ToString("N"),
};
string serializedOrchestrationData = this.defaultConverter.Serialize(orchestrationInput);
var startedEvent = new ExecutionStartedEvent(-1, serializedOrchestrationData)
{
Tags = orchestrationTags,
Name = orchestrationName,
Version = orchestrationVersion,
OrchestrationInstance = orchestrationInstance
};
var taskMessages = new List <TaskMessage>
{
new TaskMessage
{
OrchestrationInstance = orchestrationInstance,
Event = startedEvent
}
};
if (eventData != null)
{
string serializedEventData = this.defaultConverter.Serialize(eventData);
taskMessages.Add(new TaskMessage
{
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = orchestrationInstanceId,
// to ensure that the event gets raised on the running
// orchestration instance, null the execution id
// so that it will find out which execution
// it should use for processing
ExecutionId = null
},
Event = new EventRaisedEvent(-1, serializedEventData)
{
Name = eventName
}
});
}
CorrelationTraceClient.Propagate(() => CreateAndTrackDependencyTelemetry(requestTraceContext));
// Raised events and create orchestration calls use different methods so get handled separately
await Task.WhenAll(taskMessages.Where(t => !(t.Event is EventRaisedEvent)).Select(sEvent => ServiceClient.CreateTaskOrchestrationAsync(sEvent, dedupeStatuses)));
await ServiceClient.SendTaskOrchestrationMessageBatchAsync(taskMessages.Where(t => (t.Event is EventRaisedEvent)).ToArray());
return(orchestrationInstance);
}
