/// <summary> /// Wait for an orchestration to reach any terminal state within the given timeout /// </summary> /// <param name="orchestrationInstance">Instance to terminate</param> /// <param name="timeout">Max timeout to wait</param> public Task <OrchestrationState> WaitForOrchestrationAsync( OrchestrationInstance orchestrationInstance, TimeSpan timeout) { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { throw new ArgumentException(nameof(orchestrationInstance)); } this.logHelper.WaitingForInstance(orchestrationInstance, timeout); return(this.ServiceClient.WaitForOrchestrationAsync( orchestrationInstance.InstanceId, orchestrationInstance.ExecutionId, timeout, CancellationToken.None)); }
/// <summary> /// Wait for an orchestration to reach any terminal state within the given timeout /// </summary> /// <param name="orchestrationInstance">Instance to terminate</param> /// <param name="timeout">Max timeout to wait</param> /// <param name="cancellationToken">Task cancellation token</param> public Task <OrchestrationState> WaitForOrchestrationAsync( OrchestrationInstance orchestrationInstance, TimeSpan timeout, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { throw new ArgumentException(nameof(orchestrationInstance)); } return(ServiceClient.WaitForOrchestrationAsync( orchestrationInstance.InstanceId, orchestrationInstance.ExecutionId, timeout, cancellationToken)); }
public TaskOrchestrationContext( OrchestrationInstance orchestrationInstance, TaskScheduler taskScheduler, ErrorPropagationMode errorPropagationMode = ErrorPropagationMode.SerializeExceptions) { Utils.UnusedParameter(taskScheduler); this.openTasks = new Dictionary <int, OpenTaskInfo>(); this.orchestratorActionsMap = new SortedDictionary <int, OrchestratorAction>(); this.idCounter = 0; this.MessageDataConverter = JsonDataConverter.Default; this.ErrorDataConverter = JsonDataConverter.Default; OrchestrationInstance = orchestrationInstance; IsReplaying = false; ErrorPropagationMode = errorPropagationMode; }
/// <summary> /// Raises an event in the specified orchestration instance, which eventually causes the OnEvent() method in the /// orchestration to fire. /// </summary> /// <param name="orchestrationInstance">Instance in which to raise the event</param> /// <param name="eventName">Name of the event</param> /// <param name="eventData">Data for the event</param> public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { throw new ArgumentException("orchestrationInstance"); } string serializedInput = defaultConverter.Serialize(eventData); var taskMessage = new TaskMessage { OrchestrationInstance = orchestrationInstance, Event = new EventRaisedEvent(-1, serializedInput) { Name = eventName } }; await this.serviceClient.SendTaskOrchestrationMessageAsync(taskMessage); }
public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { throw new ArgumentException(nameof(orchestrationInstance)); } int id = this.idCounter++; string serializedEventData = this.dataConverter.Serialize(eventData); var action = new SendEventOrchestratorAction { Id = id, Instance = orchestrationInstance, EventName = eventName, EventData = serializedEventData, }; this.orchestratorActionsMap.Add(id, action); }
static bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) { foreach (TaskMessage message in workItem.NewMessages) { OrchestrationInstance orchestrationInstance = message.OrchestrationInstance; if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { throw TraceHelper.TraceException( TraceEventType.Error, "TaskOrchestrationDispatcher-OrchestrationInstanceMissing", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (workItem.OrchestrationRuntimeState.Events.Count == 1 && message.Event.EventType != EventType.ExecutionStarted) { // we get here because of: // i) responses for scheduled tasks after the orchestrations have been completed // ii) responses for explicitly deleted orchestrations return(false); } TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-ProcessEvent", orchestrationInstance, "Processing new event with Id {0} and type {1}", message.Event.EventId, message.Event.EventType); if (message.Event.EventType == EventType.ExecutionStarted) { if (workItem.OrchestrationRuntimeState.Events.Count > 1) { // this was caused due to a dupe execution started event, swallow this one TraceHelper.TraceInstance( TraceEventType.Warning, "TaskOrchestrationDispatcher-DuplicateStartEvent", orchestrationInstance, "Duplicate start event. Ignoring event with Id {0} and type {1} ", message.Event.EventId, message.Event.EventType); continue; } } else if (!string.IsNullOrWhiteSpace(orchestrationInstance.ExecutionId) && !string.Equals(orchestrationInstance.ExecutionId, workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId)) { // eat up any events for previous executions TraceHelper.TraceInstance( TraceEventType.Warning, "TaskOrchestrationDispatcher-ExecutionIdMismatch", orchestrationInstance, "ExecutionId of event does not match current executionId. Ignoring event with Id {0} and type {1} ", message.Event.EventId, message.Event.EventType); continue; } workItem.OrchestrationRuntimeState.AddEvent(message.Event); } return(true); }
/// <summary> /// Get a list of orchestration states from the instance storage table for the /// most current execution (generation) of the specified instance. /// </summary> /// <param name="instance">Instance</param> /// <returns>The OrchestrationState of the specified instanceId or null if not found</returns> /// <exception cref="InvalidOperationException">Thrown if instance store not configured</exception> public Task <OrchestrationState> GetOrchestrationStateAsync(OrchestrationInstance instance) { return(GetOrchestrationStateAsync(instance.InstanceId, instance.ExecutionId)); }
/// <summary> /// Forcefully terminate the specified orchestration instance /// </summary> /// <param name="orchestrationInstance">Instance to terminate</param> public Task TerminateInstanceAsync(OrchestrationInstance orchestrationInstance) { return(TerminateInstanceAsync(orchestrationInstance, string.Empty)); }
async Task <OrchestrationInstance> InternalCreateOrchestrationInstanceWithRaisedEventAsync( string orchestrationName, string orchestrationVersion, string orchestrationInstanceId, object orchestrationInput, IDictionary <string, string> orchestrationTags, OrchestrationStatus[] dedupeStatuses, string eventName, object eventData) { if (string.IsNullOrWhiteSpace(orchestrationInstanceId)) { orchestrationInstanceId = Guid.NewGuid().ToString("N"); } var orchestrationInstance = new OrchestrationInstance { InstanceId = orchestrationInstanceId, ExecutionId = Guid.NewGuid().ToString("N"), }; string serializedOrchestrationData = defaultConverter.Serialize(orchestrationInput); var startedEvent = new ExecutionStartedEvent(-1, serializedOrchestrationData) { Tags = orchestrationTags, Name = orchestrationName, Version = orchestrationVersion, OrchestrationInstance = orchestrationInstance }; var taskMessages = new List <TaskMessage> { new TaskMessage { OrchestrationInstance = orchestrationInstance, Event = startedEvent } }; if (eventData != null) { string serializedEventData = defaultConverter.Serialize(eventData); taskMessages.Add(new TaskMessage { OrchestrationInstance = new OrchestrationInstance { InstanceId = orchestrationInstanceId, // to ensure that the event gets raised on the running // orchestration instance, null the execution id // so that it will find out which execution // it should use for processing ExecutionId = null }, Event = new EventRaisedEvent(-1, serializedEventData) { Name = eventName } }); } // Raised events and create orchestration calls use different methods so get handled separately await Task.WhenAll(taskMessages.Where(t => !(t.Event is EventRaisedEvent)).Select(sEvent => this.serviceClient.CreateTaskOrchestrationAsync(sEvent, dedupeStatuses))); await this.serviceClient.SendTaskOrchestrationMessageBatchAsync(taskMessages.Where(t => (t.Event is EventRaisedEvent)).ToArray()); return(orchestrationInstance); }
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem) { Task renewTask = null; var renewCancellationTokenSource = new CancellationTokenSource(); TaskMessage taskMessage = workItem.TaskMessage; OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; TaskScheduledEvent scheduledEvent = null; Activity diagnosticActivity = null; try { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received a message that does not have any OrchestrationInstance information."); throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingOrchestrationInstance", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported."); throw TraceHelper.TraceException( TraceEventType.Critical, "TaskActivityDispatcher-UnsupportedEventType", new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } // call and get return message scheduledEvent = (TaskScheduledEvent)taskMessage.Event; this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent); TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (taskActivity == null) { throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found"); } if (workItem.LockedUntilUtc < DateTime.MaxValue) { // start a task to run RenewUntil renewTask = Task.Factory.StartNew( () => this.RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); } // TODO : pass workflow instance data var context = new TaskContext(taskMessage.OrchestrationInstance); HistoryEvent eventToRespond = null; var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(taskMessage.OrchestrationInstance); dispatchContext.SetProperty(taskActivity); dispatchContext.SetProperty(scheduledEvent); // correlation CorrelationTraceClient.Propagate(() => { workItem.TraceContextBase?.SetActivityToCurrent(); diagnosticActivity = workItem.TraceContextBase?.CurrentActivity; }); await this.dispatchPipeline.RunAsync(dispatchContext, async _ => { try { string output = await taskActivity.RunAsync(context, scheduledEvent.Input); eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (TaskFailureException e) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e); string details = this.IncludeDetails ? e.Details : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e); CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e)); } catch (Exception e) when(!Utils.IsFatal(e) && !Utils.IsExecutionAborting(e)) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e); string details = this.IncludeDetails ? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}" : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e); } if (eventToRespond is TaskCompletedEvent completedEvent) { this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent); } }); var responseTaskMessage = new TaskMessage { Event = eventToRespond, OrchestrationInstance = orchestrationInstance }; await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage); } catch (SessionAbortedException e) { // The activity aborted its execution this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message); TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent.Name, e.Message); await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem); } finally { diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out. if (renewTask != null) { renewCancellationTokenSource.Cancel(); try { // wait the renewTask finish await renewTask; } catch (OperationCanceledException) { // ignore } } } }
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem) { Task renewTask = null; var renewCancellationTokenSource = new CancellationTokenSource(); try { TaskMessage taskMessage = workItem.TaskMessage; OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingOrchestrationInstance", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { throw TraceHelper.TraceException( TraceEventType.Critical, "TaskActivityDispatcher-UnsupportedEventType", new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } // call and get return message var scheduledEvent = (TaskScheduledEvent)taskMessage.Event; TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (taskActivity == null) { throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found"); } renewTask = Task.Factory.StartNew(() => RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); // TODO : pass workflow instance data var context = new TaskContext(taskMessage.OrchestrationInstance); HistoryEvent eventToRespond = null; var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(taskMessage.OrchestrationInstance); dispatchContext.SetProperty(taskActivity); dispatchContext.SetProperty(scheduledEvent); await this.dispatchPipeline.RunAsync(dispatchContext, async _ => { try { string output = await taskActivity.RunAsync(context, scheduledEvent.Input); eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (TaskFailureException e) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e); string details = IncludeDetails ? e.Details : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); } catch (Exception e) when(!Utils.IsFatal(e)) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e); string details = IncludeDetails ? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}" : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); } }); var responseTaskMessage = new TaskMessage { Event = eventToRespond, OrchestrationInstance = orchestrationInstance }; await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage); } finally { if (renewTask != null) { renewCancellationTokenSource.Cancel(); renewTask.Wait(renewCancellationTokenSource.Token); } } }
async Task <OrchestrationInstance> InternalCreateOrchestrationInstanceWithRaisedEventAsync( string orchestrationName, string orchestrationVersion, string orchestrationInstanceId, object orchestrationInput, IDictionary <string, string> orchestrationTags, OrchestrationStatus[] dedupeStatuses, string eventName, object eventData, DateTime?startAt = null) { if (string.IsNullOrWhiteSpace(orchestrationInstanceId)) { orchestrationInstanceId = Guid.NewGuid().ToString("N"); } var orchestrationInstance = new OrchestrationInstance { InstanceId = orchestrationInstanceId, ExecutionId = Guid.NewGuid().ToString("N"), }; string serializedOrchestrationData = this.defaultConverter.Serialize(orchestrationInput); var startedEvent = new ExecutionStartedEvent(-1, serializedOrchestrationData) { Tags = orchestrationTags, Name = orchestrationName, Version = orchestrationVersion, OrchestrationInstance = orchestrationInstance, ScheduledStartTime = startAt }; var startMessage = new TaskMessage { OrchestrationInstance = orchestrationInstance, Event = startedEvent }; this.logHelper.SchedulingOrchestration(startedEvent); // Raised events and create orchestration calls use different methods so get handled separately await this.ServiceClient.CreateTaskOrchestrationAsync(startMessage, dedupeStatuses); if (eventData != null) { string serializedEventData = this.defaultConverter.Serialize(eventData); var eventRaisedEvent = new EventRaisedEvent(-1, serializedEventData) { Name = eventName }; this.logHelper.RaisingEvent(orchestrationInstance, eventRaisedEvent); var eventMessage = new TaskMessage { OrchestrationInstance = new OrchestrationInstance { InstanceId = orchestrationInstanceId, // to ensure that the event gets raised on the running // orchestration instance, null the execution id // so that it will find out which execution // it should use for processing ExecutionId = null }, Event = eventRaisedEvent, }; await this.ServiceClient.SendTaskOrchestrationMessageAsync(eventMessage); } return(orchestrationInstance); }
/// <summary> /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the /// orchestration to fire. /// </summary> /// <param name="orchestrationInstance">Instance in which to raise the event</param> /// <param name="eventName">Name of the event</param> /// <param name="eventData">Data for the event</param> public abstract void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData);
async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem) { try { if (workItem.Session == null) { // Legacy behavior await this.OnProcessWorkItemAsync(workItem); return; } var isExtendedSession = false; CorrelationTraceClient.Propagate( () => { // Check if it is extended session. isExtendedSession = this.concurrentSessionLock.Acquire(); this.concurrentSessionLock.Release(); workItem.IsExtendedSession = isExtendedSession; }); var processCount = 0; try { while (true) { // If the provider provided work items, execute them. if (workItem.NewMessages?.Count > 0) { bool isCompletedOrInterrupted = await this.OnProcessWorkItemAsync(workItem); if (isCompletedOrInterrupted) { break; } processCount++; } // Fetches beyond the first require getting an extended session lock, used to prevent starvation. if (processCount > 0 && !isExtendedSession) { isExtendedSession = this.concurrentSessionLock.Acquire(); if (!isExtendedSession) { TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-MaxOperations", "Failed to acquire concurrent session lock."); break; } } TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-StartFetch", "Starting fetch of existing session."); Stopwatch timer = Stopwatch.StartNew(); // Wait for new messages to arrive for the session. This call is expected to block (asynchronously) // until either new messages are available or until a provider-specific timeout has expired. workItem.NewMessages = await workItem.Session.FetchNewOrchestrationMessagesAsync(workItem); if (workItem.NewMessages == null) { break; } TraceHelper.Trace( TraceEventType.Verbose, "OnProcessWorkItemSession-EndFetch", $"Fetched {workItem.NewMessages.Count} new message(s) after {timer.ElapsedMilliseconds} ms from existing session."); workItem.OrchestrationRuntimeState.NewEvents.Clear(); } } finally { if (isExtendedSession) { TraceHelper.Trace( TraceEventType.Verbose, "OnProcessWorkItemSession-Release", $"Releasing extended session after {processCount} batch(es)."); this.concurrentSessionLock.Release(); } } } catch (SessionAbortedException e) { // Either the orchestration or the orchestration service explicitly abandoned the session. OrchestrationInstance instance = workItem.OrchestrationRuntimeState?.OrchestrationInstance ?? new OrchestrationInstance { InstanceId = workItem.InstanceId }; this.logHelper.OrchestrationAborted(instance, e.Message); TraceHelper.TraceInstance(TraceEventType.Warning, "TaskOrchestrationDispatcher-ExecutionAborted", instance, "{0}", e.Message); await this.orchestrationService.AbandonTaskOrchestrationWorkItemAsync(workItem); } }
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem) { Task?renewTask = null; using var renewCancellationTokenSource = new CancellationTokenSource(); TaskMessage taskMessage = workItem.TaskMessage; OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; TaskScheduledEvent? scheduledEvent = null; Activity?diagnosticActivity = null; try { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received a message that does not have any OrchestrationInstance information."); throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingOrchestrationInstance", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported."); throw TraceHelper.TraceException( TraceEventType.Critical, "TaskActivityDispatcher-UnsupportedEventType", new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } scheduledEvent = (TaskScheduledEvent)taskMessage.Event; if (scheduledEvent.Name == null) { string message = $"The activity worker received a {nameof(EventType.TaskScheduled)} event that does not specify an activity name."; this.logHelper.TaskActivityDispatcherError(workItem, message); throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingActivityName", new InvalidOperationException(message)); } this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent); TaskActivity?taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (workItem.LockedUntilUtc < DateTime.MaxValue) { // start a task to run RenewUntil renewTask = Task.Factory.StartNew( () => this.RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); } var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(taskMessage.OrchestrationInstance); dispatchContext.SetProperty(taskActivity); dispatchContext.SetProperty(scheduledEvent); // correlation CorrelationTraceClient.Propagate(() => { workItem.TraceContextBase?.SetActivityToCurrent(); diagnosticActivity = workItem.TraceContextBase?.CurrentActivity; }); ActivityExecutionResult?result; try { await this.dispatchPipeline.RunAsync(dispatchContext, async _ => { if (taskActivity == null) { // This likely indicates a deployment error of some kind. Because these unhandled exceptions are // automatically retried, resolving this may require redeploying the app code so that the activity exists again. // CONSIDER: Should this be changed into a permanent error that fails the orchestration? Perhaps // the app owner doesn't care to preserve existing instances when doing code deployments? throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found"); } var context = new TaskContext(taskMessage.OrchestrationInstance); context.ErrorPropagationMode = this.errorPropagationMode; HistoryEvent?responseEvent; try { string?output = await taskActivity.RunAsync(context, scheduledEvent.Input); responseEvent = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (Exception e) when(e is not TaskFailureException && !Utils.IsFatal(e) && !Utils.IsExecutionAborting(e)) { // These are unexpected exceptions that occur in the task activity abstraction. Normal exceptions from // activities are expected to be translated into TaskFailureException and handled outside the middleware // context (see further below). TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e); string?details = this.IncludeDetails ? $"Unhandled exception while executing task: {e}" : null; responseEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, new FailureDetails(e)); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)responseEvent, e); } var result = new ActivityExecutionResult { ResponseEvent = responseEvent }; dispatchContext.SetProperty(result); }); result = dispatchContext.GetProperty <ActivityExecutionResult>(); } catch (TaskFailureException e) { // These are normal task activity failures. They can come from Activity implementations or from middleware. TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e); string?details = this.IncludeDetails ? e.Details : null; var failureEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, e.FailureDetails); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, failureEvent, e); CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e)); result = new ActivityExecutionResult { ResponseEvent = failureEvent }; } catch (Exception middlewareException) when(!Utils.IsFatal(middlewareException)) { // These are considered retriable this.logHelper.TaskActivityDispatcherError(workItem, $"Unhandled exception in activity middleware pipeline: {middlewareException}"); throw; } HistoryEvent?eventToRespond = result?.ResponseEvent; if (eventToRespond is TaskCompletedEvent completedEvent) { this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent); } else if (eventToRespond is null) { // Default response if middleware prevents a response from being generated eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, null); } var responseTaskMessage = new TaskMessage { Event = eventToRespond, OrchestrationInstance = orchestrationInstance }; await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage); } catch (SessionAbortedException e) { // The activity aborted its execution this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message); TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent?.Name, e.Message); await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem); } finally { diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out. if (renewTask != null) { renewCancellationTokenSource.Cancel(); try { // wait the renewTask finish await renewTask; } catch (OperationCanceledException) { // ignore } } } }