Ejemplo n.º 1
0
        async Task <OrchestrationExecutionCursor> ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState)
        {
            TaskOrchestration taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version);

            if (taskOrchestration == null)
            {
                throw TraceHelper.TraceExceptionInstance(
                          TraceEventType.Error,
                          "TaskOrchestrationDispatcher-TypeMissing",
                          runtimeState.OrchestrationInstance,
                          new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})"));
            }

            var dispatchContext = new DispatchMiddlewareContext();

            dispatchContext.SetProperty(runtimeState.OrchestrationInstance);
            dispatchContext.SetProperty(taskOrchestration);
            dispatchContext.SetProperty(runtimeState);

            var executor = new TaskOrchestrationExecutor(runtimeState, taskOrchestration, orchestrationService.EventBehaviourForContinueAsNew);

            IEnumerable <OrchestratorAction> decisions = Enumerable.Empty <OrchestratorAction>();

            await this.dispatchPipeline.RunAsync(dispatchContext, _ =>
            {
                decisions = executor.Execute();
                return(CompletedTask);
            });

            return(new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions));
        }
Ejemplo n.º 2
0
        async Task <IEnumerable <OrchestratorAction> > ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState)
        {
            TaskOrchestration taskOrchestration = objectManager.GetObject(runtimeState.Name, runtimeState.Version);

            if (taskOrchestration == null)
            {
                throw TraceHelper.TraceExceptionInstance(
                          TraceEventType.Error,
                          "TaskOrchestrationDispatcher-TypeMissing",
                          runtimeState.OrchestrationInstance,
                          new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})"));
            }

            var dispatchContext = new DispatchMiddlewareContext();

            dispatchContext.SetProperty(runtimeState.OrchestrationInstance);
            dispatchContext.SetProperty(taskOrchestration);
            dispatchContext.SetProperty(runtimeState);

            IEnumerable <OrchestratorAction> decisions = null;

            await this.dispatchPipeline.RunAsync(dispatchContext, _ =>
            {
                var taskOrchestrationExecutor = new TaskOrchestrationExecutor(runtimeState, taskOrchestration);
                decisions = taskOrchestrationExecutor.Execute();

                return(Task.FromResult(0));
            });

            return(decisions);
        }
Ejemplo n.º 3
0
        async Task <OrchestrationExecutionCursor> ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem)
        {
            // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never
            // registered it (which is an error, and we'll throw for this further down) or it could be that some custom
            // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic.
            TaskOrchestration?taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version !);

            var dispatchContext = new DispatchMiddlewareContext();

            dispatchContext.SetProperty(runtimeState.OrchestrationInstance);
            dispatchContext.SetProperty(taskOrchestration);
            dispatchContext.SetProperty(runtimeState);
            dispatchContext.SetProperty(workItem);

            TaskOrchestrationExecutor?executor = null;

            await this.dispatchPipeline.RunAsync(dispatchContext, _ =>
            {
                // Check to see if the custom middleware intercepted and substituted the orchestration execution
                // with its own execution behavior, providing us with the end results. If so, we can terminate
                // the dispatch pipeline here.
                var resultFromMiddleware = dispatchContext.GetProperty <OrchestratorExecutionResult>();
                if (resultFromMiddleware != null)
                {
                    return(CompletedTask);
                }

                if (taskOrchestration == null)
                {
                    throw TraceHelper.TraceExceptionInstance(
                        TraceEventType.Error,
                        "TaskOrchestrationDispatcher-TypeMissing",
                        runtimeState.OrchestrationInstance,
                        new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})"));
                }

                executor = new TaskOrchestrationExecutor(
                    runtimeState,
                    taskOrchestration,
                    this.orchestrationService.EventBehaviourForContinueAsNew,
                    this.errorPropagationMode);
                OrchestratorExecutionResult resultFromOrchestrator = executor.Execute();
                dispatchContext.SetProperty(resultFromOrchestrator);
                return(CompletedTask);
            });

            var result = dispatchContext.GetProperty <OrchestratorExecutionResult>();
            IEnumerable <OrchestratorAction> decisions = result?.Actions ?? Enumerable.Empty <OrchestratorAction>();

            runtimeState.Status = result?.CustomStatus;

            return(new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions));
        }
Ejemplo n.º 4
0
        Task ResumeOrchestrationAsync(OrchestrationExecutionCursor cursor)
        {
            var dispatchContext = new DispatchMiddlewareContext();

            dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance);
            dispatchContext.SetProperty(cursor.TaskOrchestration);
            dispatchContext.SetProperty(cursor.RuntimeState);

            return(this.dispatchPipeline.RunAsync(dispatchContext, _ =>
            {
                cursor.LatestDecisions = cursor.OrchestrationExecutor.ExecuteNewEvents();
                return CompletedTask;
            }));
        }
Ejemplo n.º 5
0
        Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem)
        {
            OrchestrationExecutionCursor cursor = workItem.Cursor;

            var dispatchContext = new DispatchMiddlewareContext();

            dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance);
            dispatchContext.SetProperty(cursor.TaskOrchestration);
            dispatchContext.SetProperty(cursor.RuntimeState);
            dispatchContext.SetProperty(workItem);

            cursor.LatestDecisions = Enumerable.Empty <OrchestratorAction>();
            return(this.dispatchPipeline.RunAsync(dispatchContext, _ => {
                cursor.LatestDecisions = cursor.OrchestrationExecutor.ExecuteNewEvents();
                return CompletedTask;
            }));
        }
Ejemplo n.º 6
0
        /// <inheritdoc />
        public Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next)
        {
            var customInstance = OrchestrationInstanceEx.Get(context.GetProperty <OrchestrationInstance>());

            context.SetProperty <OrchestrationInstance>(customInstance);

            // Do something with the session data, such as starting a logging scope with correlation id property.

            return(next());
        }
Ejemplo n.º 7
0
        /// <inheritdoc />
        public Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next)
        {
            // Initialize the OrchestrationInstance with our session data. Or if it already exists,
            // then set it to the appropriate context properties.
            OrchestrationRuntimeState runtimeState = context.GetProperty <OrchestrationRuntimeState>();
            var customInstance = OrchestrationInstanceEx.Initialize(runtimeState);

            context.SetProperty <OrchestrationInstance>(customInstance);

            // Do something with the session data, such as starting a logging scope with correlation id property.

            return(next());
        }
        public async Task Execute(DispatchMiddlewareContext context, Func <Task> next)
        {
            using (var serviceScope = _serviceScopeFactory.CreateScope())
            {
                context.SetProperty(serviceScope.ServiceProvider);

                if (context.GetProperty <TaskActivity>() is ServiceProviderTaskActivity initializable)
                {
                    initializable.Initialize(serviceScope.ServiceProvider);
                }

                await next();
            }
        }
        public async Task Execute(DispatchMiddlewareContext context, Func <Task> next)
        {
            var orchestrationInstance = context.GetProperty <OrchestrationInstance>();

            var serviceScope = WorkerOrchestrationService.OrchestrationsServiceScopes[orchestrationInstance.InstanceId];

            context.SetProperty(serviceScope.ServiceProvider);

            if (context.GetProperty <TaskOrchestration>() is ServiceProviderTaskOrchestration initializable)
            {
                initializable.Initialize(serviceScope.ServiceProvider);
            }

            await next();
        }
Ejemplo n.º 10
0
        async Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem)
        {
            OrchestrationExecutionCursor cursor = workItem.Cursor;

            var dispatchContext = new DispatchMiddlewareContext();

            dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance);
            dispatchContext.SetProperty(cursor.TaskOrchestration);
            dispatchContext.SetProperty(cursor.RuntimeState);
            dispatchContext.SetProperty(workItem);

            cursor.LatestDecisions = Enumerable.Empty <OrchestratorAction>();
            await this.dispatchPipeline.RunAsync(dispatchContext, _ =>
            {
                OrchestratorExecutionResult result = cursor.OrchestrationExecutor.ExecuteNewEvents();
                dispatchContext.SetProperty(result);
                return(CompletedTask);
            });

            var result = dispatchContext.GetProperty <OrchestratorExecutionResult>();

            cursor.LatestDecisions     = result?.Actions ?? Enumerable.Empty <OrchestratorAction>();
            cursor.RuntimeState.Status = result?.CustomStatus;
        }
        /// <inheritdoc />
        public async Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next)
        {
            Check.NotNull(context, nameof(context));
            Check.NotNull(next, nameof(next));

            TaskOrchestration taskOrchestration = context.GetProperty <TaskOrchestration>();

            if (taskOrchestration is WrapperOrchestration wrapper)
            {
                wrapper.Initialize(_serviceProvider);

                // update the context task orchestration with the real one.
                context.SetProperty(wrapper.InnerOrchestration);
                await next().ConfigureAwait(false);

                return;
            }

            await next().ConfigureAwait(false);
        }
        /// <inheritdoc />
        public async Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next)
        {
            Check.NotNull(context, nameof(context));
            Check.NotNull(next, nameof(next));

            TaskActivity taskActivity = context.GetProperty <TaskActivity>();

            if (taskActivity is WrapperActivity wrapper)
            {
                wrapper.InnerActivity = (TaskActivity)_serviceProvider
                                        .GetServiceOrCreateInstance(wrapper.InnerActivityType);

                // update the context task activity with the real one.
                context.SetProperty(wrapper.InnerActivity);
                await next().ConfigureAwait(false);

                return;
            }

            await next().ConfigureAwait(false);
        }
        async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
        {
            Task renewTask = null;
            var  renewCancellationTokenSource = new CancellationTokenSource();

            TaskMessage           taskMessage           = workItem.TaskMessage;
            OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
            TaskScheduledEvent    scheduledEvent        = null;
            Activity diagnosticActivity = null;

            try
            {
                if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
                {
                    this.logHelper.TaskActivityDispatcherError(
                        workItem,
                        $"The activity worker received a message that does not have any OrchestrationInstance information.");
                    throw TraceHelper.TraceException(
                              TraceEventType.Error,
                              "TaskActivityDispatcher-MissingOrchestrationInstance",
                              new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
                }

                if (taskMessage.Event.EventType != EventType.TaskScheduled)
                {
                    this.logHelper.TaskActivityDispatcherError(
                        workItem,
                        $"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported.");
                    throw TraceHelper.TraceException(
                              TraceEventType.Critical,
                              "TaskActivityDispatcher-UnsupportedEventType",
                              new NotSupportedException("Activity worker does not support event of type: " +
                                                        taskMessage.Event.EventType));
                }

                // call and get return message
                scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
                this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent);
                TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
                if (taskActivity == null)
                {
                    throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
                }

                if (workItem.LockedUntilUtc < DateTime.MaxValue)
                {
                    // start a task to run RenewUntil
                    renewTask = Task.Factory.StartNew(
                        () => this.RenewUntil(workItem, renewCancellationTokenSource.Token),
                        renewCancellationTokenSource.Token);
                }

                // TODO : pass workflow instance data
                var          context        = new TaskContext(taskMessage.OrchestrationInstance);
                HistoryEvent eventToRespond = null;

                var dispatchContext = new DispatchMiddlewareContext();
                dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
                dispatchContext.SetProperty(taskActivity);
                dispatchContext.SetProperty(scheduledEvent);

                // correlation
                CorrelationTraceClient.Propagate(() =>
                {
                    workItem.TraceContextBase?.SetActivityToCurrent();
                    diagnosticActivity = workItem.TraceContextBase?.CurrentActivity;
                });

                await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
                {
                    try
                    {
                        string output  = await taskActivity.RunAsync(context, scheduledEvent.Input);
                        eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
                    }
                    catch (TaskFailureException e)
                    {
                        TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
                        string details = this.IncludeDetails ? e.Details : null;
                        eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
                        this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e);
                        CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e));
                    }
                    catch (Exception e) when(!Utils.IsFatal(e) && !Utils.IsExecutionAborting(e))
                    {
                        TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
                        string details = this.IncludeDetails
                            ? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}"
                            : null;
                        eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
                        this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e);
                    }

                    if (eventToRespond is TaskCompletedEvent completedEvent)
                    {
                        this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent);
                    }
                });

                var responseTaskMessage = new TaskMessage
                {
                    Event = eventToRespond,
                    OrchestrationInstance = orchestrationInstance
                };

                await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
            }
            catch (SessionAbortedException e)
            {
                // The activity aborted its execution
                this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message);
                TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent.Name, e.Message);
                await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem);
            }
            finally
            {
                diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out.
                if (renewTask != null)
                {
                    renewCancellationTokenSource.Cancel();
                    try
                    {
                        // wait the renewTask finish
                        await renewTask;
                    }
                    catch (OperationCanceledException)
                    {
                        // ignore
                    }
                }
            }
        }
Ejemplo n.º 14
0
        async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
        {
            Task renewTask = null;
            var  renewCancellationTokenSource = new CancellationTokenSource();

            try
            {
                TaskMessage           taskMessage           = workItem.TaskMessage;
                OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
                if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
                {
                    throw TraceHelper.TraceException(
                              TraceEventType.Error,
                              "TaskActivityDispatcher-MissingOrchestrationInstance",
                              new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
                }

                if (taskMessage.Event.EventType != EventType.TaskScheduled)
                {
                    throw TraceHelper.TraceException(
                              TraceEventType.Critical,
                              "TaskActivityDispatcher-UnsupportedEventType",
                              new NotSupportedException("Activity worker does not support event of type: " +
                                                        taskMessage.Event.EventType));
                }

                // call and get return message
                var          scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
                TaskActivity taskActivity   = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
                if (taskActivity == null)
                {
                    throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
                }

                renewTask = Task.Factory.StartNew(() => RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token);

                // TODO : pass workflow instance data
                var          context        = new TaskContext(taskMessage.OrchestrationInstance);
                HistoryEvent eventToRespond = null;

                var dispatchContext = new DispatchMiddlewareContext();
                dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
                dispatchContext.SetProperty(taskActivity);
                dispatchContext.SetProperty(scheduledEvent);

                await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
                {
                    try
                    {
                        string output  = await taskActivity.RunAsync(context, scheduledEvent.Input);
                        eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
                    }
                    catch (TaskFailureException e)
                    {
                        TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
                        string details = IncludeDetails ? e.Details : null;
                        eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
                    }
                    catch (Exception e) when(!Utils.IsFatal(e))
                    {
                        TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
                        string details = IncludeDetails
                            ? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}"
                            : null;
                        eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
                    }
                });

                var responseTaskMessage = new TaskMessage
                {
                    Event = eventToRespond,
                    OrchestrationInstance = orchestrationInstance
                };

                await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
            }
            finally
            {
                if (renewTask != null)
                {
                    renewCancellationTokenSource.Cancel();
                    renewTask.Wait(renewCancellationTokenSource.Token);
                }
            }
        }
Ejemplo n.º 15
0
        async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
        {
            Task?renewTask = null;

            using var renewCancellationTokenSource = new CancellationTokenSource();

            TaskMessage           taskMessage           = workItem.TaskMessage;
            OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
            TaskScheduledEvent?   scheduledEvent        = null;
            Activity?diagnosticActivity = null;

            try
            {
                if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
                {
                    this.logHelper.TaskActivityDispatcherError(
                        workItem,
                        $"The activity worker received a message that does not have any OrchestrationInstance information.");
                    throw TraceHelper.TraceException(
                              TraceEventType.Error,
                              "TaskActivityDispatcher-MissingOrchestrationInstance",
                              new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
                }

                if (taskMessage.Event.EventType != EventType.TaskScheduled)
                {
                    this.logHelper.TaskActivityDispatcherError(
                        workItem,
                        $"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported.");
                    throw TraceHelper.TraceException(
                              TraceEventType.Critical,
                              "TaskActivityDispatcher-UnsupportedEventType",
                              new NotSupportedException("Activity worker does not support event of type: " +
                                                        taskMessage.Event.EventType));
                }

                scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
                if (scheduledEvent.Name == null)
                {
                    string message = $"The activity worker received a {nameof(EventType.TaskScheduled)} event that does not specify an activity name.";
                    this.logHelper.TaskActivityDispatcherError(workItem, message);
                    throw TraceHelper.TraceException(
                              TraceEventType.Error,
                              "TaskActivityDispatcher-MissingActivityName",
                              new InvalidOperationException(message));
                }

                this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent);
                TaskActivity?taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);

                if (workItem.LockedUntilUtc < DateTime.MaxValue)
                {
                    // start a task to run RenewUntil
                    renewTask = Task.Factory.StartNew(
                        () => this.RenewUntil(workItem, renewCancellationTokenSource.Token),
                        renewCancellationTokenSource.Token);
                }

                var dispatchContext = new DispatchMiddlewareContext();
                dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
                dispatchContext.SetProperty(taskActivity);
                dispatchContext.SetProperty(scheduledEvent);

                // correlation
                CorrelationTraceClient.Propagate(() =>
                {
                    workItem.TraceContextBase?.SetActivityToCurrent();
                    diagnosticActivity = workItem.TraceContextBase?.CurrentActivity;
                });

                ActivityExecutionResult?result;
                try
                {
                    await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
                    {
                        if (taskActivity == null)
                        {
                            // This likely indicates a deployment error of some kind. Because these unhandled exceptions are
                            // automatically retried, resolving this may require redeploying the app code so that the activity exists again.
                            // CONSIDER: Should this be changed into a permanent error that fails the orchestration? Perhaps
                            //           the app owner doesn't care to preserve existing instances when doing code deployments?
                            throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
                        }

                        var context = new TaskContext(taskMessage.OrchestrationInstance);
                        context.ErrorPropagationMode = this.errorPropagationMode;

                        HistoryEvent?responseEvent;

                        try
                        {
                            string?output = await taskActivity.RunAsync(context, scheduledEvent.Input);
                            responseEvent = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
                        }
                        catch (Exception e) when(e is not TaskFailureException && !Utils.IsFatal(e) && !Utils.IsExecutionAborting(e))
                        {
                            // These are unexpected exceptions that occur in the task activity abstraction. Normal exceptions from
                            // activities are expected to be translated into TaskFailureException and handled outside the middleware
                            // context (see further below).
                            TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
                            string?details = this.IncludeDetails
                                ? $"Unhandled exception while executing task: {e}"
                                : null;
                            responseEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, new FailureDetails(e));
                            this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)responseEvent, e);
                        }

                        var result = new ActivityExecutionResult {
                            ResponseEvent = responseEvent
                        };
                        dispatchContext.SetProperty(result);
                    });

                    result = dispatchContext.GetProperty <ActivityExecutionResult>();
                }
                catch (TaskFailureException e)
                {
                    // These are normal task activity failures. They can come from Activity implementations or from middleware.
                    TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
                    string?details      = this.IncludeDetails ? e.Details : null;
                    var    failureEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, e.FailureDetails);
                    this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, failureEvent, e);
                    CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e));
                    result = new ActivityExecutionResult {
                        ResponseEvent = failureEvent
                    };
                }
                catch (Exception middlewareException) when(!Utils.IsFatal(middlewareException))
                {
                    // These are considered retriable
                    this.logHelper.TaskActivityDispatcherError(workItem, $"Unhandled exception in activity middleware pipeline: {middlewareException}");
                    throw;
                }

                HistoryEvent?eventToRespond = result?.ResponseEvent;

                if (eventToRespond is TaskCompletedEvent completedEvent)
                {
                    this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent);
                }
                else if (eventToRespond is null)
                {
                    // Default response if middleware prevents a response from being generated
                    eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, null);
                }

                var responseTaskMessage = new TaskMessage
                {
                    Event = eventToRespond,
                    OrchestrationInstance = orchestrationInstance
                };

                await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
            }
            catch (SessionAbortedException e)
            {
                // The activity aborted its execution
                this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message);
                TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent?.Name, e.Message);
                await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem);
            }
            finally
            {
                diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out.
                if (renewTask != null)
                {
                    renewCancellationTokenSource.Cancel();
                    try
                    {
                        // wait the renewTask finish
                        await renewTask;
                    }
                    catch (OperationCanceledException)
                    {
                        // ignore
                    }
                }
            }
        }