async Task <OrchestrationExecutionCursor> ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState) { TaskOrchestration taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version); if (taskOrchestration == null) { throw TraceHelper.TraceExceptionInstance( TraceEventType.Error, "TaskOrchestrationDispatcher-TypeMissing", runtimeState.OrchestrationInstance, new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})")); } var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(runtimeState.OrchestrationInstance); dispatchContext.SetProperty(taskOrchestration); dispatchContext.SetProperty(runtimeState); var executor = new TaskOrchestrationExecutor(runtimeState, taskOrchestration, orchestrationService.EventBehaviourForContinueAsNew); IEnumerable <OrchestratorAction> decisions = Enumerable.Empty <OrchestratorAction>(); await this.dispatchPipeline.RunAsync(dispatchContext, _ => { decisions = executor.Execute(); return(CompletedTask); }); return(new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions)); }
async Task <IEnumerable <OrchestratorAction> > ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState) { TaskOrchestration taskOrchestration = objectManager.GetObject(runtimeState.Name, runtimeState.Version); if (taskOrchestration == null) { throw TraceHelper.TraceExceptionInstance( TraceEventType.Error, "TaskOrchestrationDispatcher-TypeMissing", runtimeState.OrchestrationInstance, new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})")); } var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(runtimeState.OrchestrationInstance); dispatchContext.SetProperty(taskOrchestration); dispatchContext.SetProperty(runtimeState); IEnumerable <OrchestratorAction> decisions = null; await this.dispatchPipeline.RunAsync(dispatchContext, _ => { var taskOrchestrationExecutor = new TaskOrchestrationExecutor(runtimeState, taskOrchestration); decisions = taskOrchestrationExecutor.Execute(); return(Task.FromResult(0)); }); return(decisions); }
async Task <OrchestrationExecutionCursor> ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem) { // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never // registered it (which is an error, and we'll throw for this further down) or it could be that some custom // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic. TaskOrchestration?taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version !); var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(runtimeState.OrchestrationInstance); dispatchContext.SetProperty(taskOrchestration); dispatchContext.SetProperty(runtimeState); dispatchContext.SetProperty(workItem); TaskOrchestrationExecutor?executor = null; await this.dispatchPipeline.RunAsync(dispatchContext, _ => { // Check to see if the custom middleware intercepted and substituted the orchestration execution // with its own execution behavior, providing us with the end results. If so, we can terminate // the dispatch pipeline here. var resultFromMiddleware = dispatchContext.GetProperty <OrchestratorExecutionResult>(); if (resultFromMiddleware != null) { return(CompletedTask); } if (taskOrchestration == null) { throw TraceHelper.TraceExceptionInstance( TraceEventType.Error, "TaskOrchestrationDispatcher-TypeMissing", runtimeState.OrchestrationInstance, new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})")); } executor = new TaskOrchestrationExecutor( runtimeState, taskOrchestration, this.orchestrationService.EventBehaviourForContinueAsNew, this.errorPropagationMode); OrchestratorExecutionResult resultFromOrchestrator = executor.Execute(); dispatchContext.SetProperty(resultFromOrchestrator); return(CompletedTask); }); var result = dispatchContext.GetProperty <OrchestratorExecutionResult>(); IEnumerable <OrchestratorAction> decisions = result?.Actions ?? Enumerable.Empty <OrchestratorAction>(); runtimeState.Status = result?.CustomStatus; return(new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions)); }
Task ResumeOrchestrationAsync(OrchestrationExecutionCursor cursor) { var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance); dispatchContext.SetProperty(cursor.TaskOrchestration); dispatchContext.SetProperty(cursor.RuntimeState); return(this.dispatchPipeline.RunAsync(dispatchContext, _ => { cursor.LatestDecisions = cursor.OrchestrationExecutor.ExecuteNewEvents(); return CompletedTask; })); }
Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem) { OrchestrationExecutionCursor cursor = workItem.Cursor; var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance); dispatchContext.SetProperty(cursor.TaskOrchestration); dispatchContext.SetProperty(cursor.RuntimeState); dispatchContext.SetProperty(workItem); cursor.LatestDecisions = Enumerable.Empty <OrchestratorAction>(); return(this.dispatchPipeline.RunAsync(dispatchContext, _ => { cursor.LatestDecisions = cursor.OrchestrationExecutor.ExecuteNewEvents(); return CompletedTask; })); }
/// <inheritdoc /> public Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next) { var customInstance = OrchestrationInstanceEx.Get(context.GetProperty <OrchestrationInstance>()); context.SetProperty <OrchestrationInstance>(customInstance); // Do something with the session data, such as starting a logging scope with correlation id property. return(next()); }
/// <inheritdoc /> public Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next) { // Initialize the OrchestrationInstance with our session data. Or if it already exists, // then set it to the appropriate context properties. OrchestrationRuntimeState runtimeState = context.GetProperty <OrchestrationRuntimeState>(); var customInstance = OrchestrationInstanceEx.Initialize(runtimeState); context.SetProperty <OrchestrationInstance>(customInstance); // Do something with the session data, such as starting a logging scope with correlation id property. return(next()); }
public async Task Execute(DispatchMiddlewareContext context, Func <Task> next) { using (var serviceScope = _serviceScopeFactory.CreateScope()) { context.SetProperty(serviceScope.ServiceProvider); if (context.GetProperty <TaskActivity>() is ServiceProviderTaskActivity initializable) { initializable.Initialize(serviceScope.ServiceProvider); } await next(); } }
public async Task Execute(DispatchMiddlewareContext context, Func <Task> next) { var orchestrationInstance = context.GetProperty <OrchestrationInstance>(); var serviceScope = WorkerOrchestrationService.OrchestrationsServiceScopes[orchestrationInstance.InstanceId]; context.SetProperty(serviceScope.ServiceProvider); if (context.GetProperty <TaskOrchestration>() is ServiceProviderTaskOrchestration initializable) { initializable.Initialize(serviceScope.ServiceProvider); } await next(); }
async Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem) { OrchestrationExecutionCursor cursor = workItem.Cursor; var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance); dispatchContext.SetProperty(cursor.TaskOrchestration); dispatchContext.SetProperty(cursor.RuntimeState); dispatchContext.SetProperty(workItem); cursor.LatestDecisions = Enumerable.Empty <OrchestratorAction>(); await this.dispatchPipeline.RunAsync(dispatchContext, _ => { OrchestratorExecutionResult result = cursor.OrchestrationExecutor.ExecuteNewEvents(); dispatchContext.SetProperty(result); return(CompletedTask); }); var result = dispatchContext.GetProperty <OrchestratorExecutionResult>(); cursor.LatestDecisions = result?.Actions ?? Enumerable.Empty <OrchestratorAction>(); cursor.RuntimeState.Status = result?.CustomStatus; }
/// <inheritdoc /> public async Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next) { Check.NotNull(context, nameof(context)); Check.NotNull(next, nameof(next)); TaskOrchestration taskOrchestration = context.GetProperty <TaskOrchestration>(); if (taskOrchestration is WrapperOrchestration wrapper) { wrapper.Initialize(_serviceProvider); // update the context task orchestration with the real one. context.SetProperty(wrapper.InnerOrchestration); await next().ConfigureAwait(false); return; } await next().ConfigureAwait(false); }
/// <inheritdoc /> public async Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next) { Check.NotNull(context, nameof(context)); Check.NotNull(next, nameof(next)); TaskActivity taskActivity = context.GetProperty <TaskActivity>(); if (taskActivity is WrapperActivity wrapper) { wrapper.InnerActivity = (TaskActivity)_serviceProvider .GetServiceOrCreateInstance(wrapper.InnerActivityType); // update the context task activity with the real one. context.SetProperty(wrapper.InnerActivity); await next().ConfigureAwait(false); return; } await next().ConfigureAwait(false); }
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem) { Task renewTask = null; var renewCancellationTokenSource = new CancellationTokenSource(); TaskMessage taskMessage = workItem.TaskMessage; OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; TaskScheduledEvent scheduledEvent = null; Activity diagnosticActivity = null; try { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received a message that does not have any OrchestrationInstance information."); throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingOrchestrationInstance", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported."); throw TraceHelper.TraceException( TraceEventType.Critical, "TaskActivityDispatcher-UnsupportedEventType", new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } // call and get return message scheduledEvent = (TaskScheduledEvent)taskMessage.Event; this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent); TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (taskActivity == null) { throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found"); } if (workItem.LockedUntilUtc < DateTime.MaxValue) { // start a task to run RenewUntil renewTask = Task.Factory.StartNew( () => this.RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); } // TODO : pass workflow instance data var context = new TaskContext(taskMessage.OrchestrationInstance); HistoryEvent eventToRespond = null; var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(taskMessage.OrchestrationInstance); dispatchContext.SetProperty(taskActivity); dispatchContext.SetProperty(scheduledEvent); // correlation CorrelationTraceClient.Propagate(() => { workItem.TraceContextBase?.SetActivityToCurrent(); diagnosticActivity = workItem.TraceContextBase?.CurrentActivity; }); await this.dispatchPipeline.RunAsync(dispatchContext, async _ => { try { string output = await taskActivity.RunAsync(context, scheduledEvent.Input); eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (TaskFailureException e) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e); string details = this.IncludeDetails ? e.Details : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e); CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e)); } catch (Exception e) when(!Utils.IsFatal(e) && !Utils.IsExecutionAborting(e)) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e); string details = this.IncludeDetails ? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}" : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e); } if (eventToRespond is TaskCompletedEvent completedEvent) { this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent); } }); var responseTaskMessage = new TaskMessage { Event = eventToRespond, OrchestrationInstance = orchestrationInstance }; await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage); } catch (SessionAbortedException e) { // The activity aborted its execution this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message); TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent.Name, e.Message); await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem); } finally { diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out. if (renewTask != null) { renewCancellationTokenSource.Cancel(); try { // wait the renewTask finish await renewTask; } catch (OperationCanceledException) { // ignore } } } }
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem) { Task renewTask = null; var renewCancellationTokenSource = new CancellationTokenSource(); try { TaskMessage taskMessage = workItem.TaskMessage; OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingOrchestrationInstance", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { throw TraceHelper.TraceException( TraceEventType.Critical, "TaskActivityDispatcher-UnsupportedEventType", new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } // call and get return message var scheduledEvent = (TaskScheduledEvent)taskMessage.Event; TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (taskActivity == null) { throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found"); } renewTask = Task.Factory.StartNew(() => RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); // TODO : pass workflow instance data var context = new TaskContext(taskMessage.OrchestrationInstance); HistoryEvent eventToRespond = null; var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(taskMessage.OrchestrationInstance); dispatchContext.SetProperty(taskActivity); dispatchContext.SetProperty(scheduledEvent); await this.dispatchPipeline.RunAsync(dispatchContext, async _ => { try { string output = await taskActivity.RunAsync(context, scheduledEvent.Input); eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (TaskFailureException e) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e); string details = IncludeDetails ? e.Details : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); } catch (Exception e) when(!Utils.IsFatal(e)) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e); string details = IncludeDetails ? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}" : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); } }); var responseTaskMessage = new TaskMessage { Event = eventToRespond, OrchestrationInstance = orchestrationInstance }; await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage); } finally { if (renewTask != null) { renewCancellationTokenSource.Cancel(); renewTask.Wait(renewCancellationTokenSource.Token); } } }
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem) { Task?renewTask = null; using var renewCancellationTokenSource = new CancellationTokenSource(); TaskMessage taskMessage = workItem.TaskMessage; OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; TaskScheduledEvent? scheduledEvent = null; Activity?diagnosticActivity = null; try { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received a message that does not have any OrchestrationInstance information."); throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingOrchestrationInstance", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported."); throw TraceHelper.TraceException( TraceEventType.Critical, "TaskActivityDispatcher-UnsupportedEventType", new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } scheduledEvent = (TaskScheduledEvent)taskMessage.Event; if (scheduledEvent.Name == null) { string message = $"The activity worker received a {nameof(EventType.TaskScheduled)} event that does not specify an activity name."; this.logHelper.TaskActivityDispatcherError(workItem, message); throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingActivityName", new InvalidOperationException(message)); } this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent); TaskActivity?taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (workItem.LockedUntilUtc < DateTime.MaxValue) { // start a task to run RenewUntil renewTask = Task.Factory.StartNew( () => this.RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); } var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(taskMessage.OrchestrationInstance); dispatchContext.SetProperty(taskActivity); dispatchContext.SetProperty(scheduledEvent); // correlation CorrelationTraceClient.Propagate(() => { workItem.TraceContextBase?.SetActivityToCurrent(); diagnosticActivity = workItem.TraceContextBase?.CurrentActivity; }); ActivityExecutionResult?result; try { await this.dispatchPipeline.RunAsync(dispatchContext, async _ => { if (taskActivity == null) { // This likely indicates a deployment error of some kind. Because these unhandled exceptions are // automatically retried, resolving this may require redeploying the app code so that the activity exists again. // CONSIDER: Should this be changed into a permanent error that fails the orchestration? Perhaps // the app owner doesn't care to preserve existing instances when doing code deployments? throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found"); } var context = new TaskContext(taskMessage.OrchestrationInstance); context.ErrorPropagationMode = this.errorPropagationMode; HistoryEvent?responseEvent; try { string?output = await taskActivity.RunAsync(context, scheduledEvent.Input); responseEvent = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (Exception e) when(e is not TaskFailureException && !Utils.IsFatal(e) && !Utils.IsExecutionAborting(e)) { // These are unexpected exceptions that occur in the task activity abstraction. Normal exceptions from // activities are expected to be translated into TaskFailureException and handled outside the middleware // context (see further below). TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e); string?details = this.IncludeDetails ? $"Unhandled exception while executing task: {e}" : null; responseEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, new FailureDetails(e)); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)responseEvent, e); } var result = new ActivityExecutionResult { ResponseEvent = responseEvent }; dispatchContext.SetProperty(result); }); result = dispatchContext.GetProperty <ActivityExecutionResult>(); } catch (TaskFailureException e) { // These are normal task activity failures. They can come from Activity implementations or from middleware. TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e); string?details = this.IncludeDetails ? e.Details : null; var failureEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, e.FailureDetails); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, failureEvent, e); CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e)); result = new ActivityExecutionResult { ResponseEvent = failureEvent }; } catch (Exception middlewareException) when(!Utils.IsFatal(middlewareException)) { // These are considered retriable this.logHelper.TaskActivityDispatcherError(workItem, $"Unhandled exception in activity middleware pipeline: {middlewareException}"); throw; } HistoryEvent?eventToRespond = result?.ResponseEvent; if (eventToRespond is TaskCompletedEvent completedEvent) { this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent); } else if (eventToRespond is null) { // Default response if middleware prevents a response from being generated eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, null); } var responseTaskMessage = new TaskMessage { Event = eventToRespond, OrchestrationInstance = orchestrationInstance }; await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage); } catch (SessionAbortedException e) { // The activity aborted its execution this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message); TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent?.Name, e.Message); await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem); } finally { diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out. if (renewTask != null) { renewCancellationTokenSource.Cancel(); try { // wait the renewTask finish await renewTask; } catch (OperationCanceledException) { // ignore } } } }