public async Task InvokeAsync_NotWrapped_Continues()
{
// arrange
var activity = new TestOrchestration();
var serviceProvider = new Mock <IServiceProvider>();
DispatchMiddlewareContext context = CreateContext(activity);
var middleware = new ServiceProviderOrchestrationMiddleware(serviceProvider.Object);
// act
await middleware.InvokeAsync(context, () => Task.CompletedTask);
// assert
TaskOrchestration actual = context.GetProperty <TaskOrchestration>();
actual.Should().NotBeNull();
actual.Should().Be(activity);
serviceProvider.Verify(m => m.GetService(It.IsAny <Type>()), Times.Never);
}
Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem)
{
OrchestrationExecutionCursor cursor = workItem.Cursor;
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance);
dispatchContext.SetProperty(cursor.TaskOrchestration);
dispatchContext.SetProperty(cursor.RuntimeState);
dispatchContext.SetProperty(workItem);
cursor.LatestDecisions = Enumerable.Empty <OrchestratorAction>();
return(this.dispatchPipeline.RunAsync(dispatchContext, _ =>
{
cursor.LatestDecisions = cursor.OrchestrationExecutor.ExecuteNewEvents();
return CompletedTask;
}));
}
private async Task OrchestrationMiddleware(DispatchMiddlewareContext dispatchContext, Func <Task> next)
{
TaskOrchestrationShim shim = (TaskOrchestrationShim)dispatchContext.GetProperty <TaskOrchestration>();
DurableOrchestrationContext context = shim.Context;
FunctionName orchestratorFunction = new FunctionName(context.Name, context.Version);
ITriggeredFunctionExecutor executor;
if (!this.registeredOrchestrators.TryGetValue(orchestratorFunction, out executor))
{
throw new InvalidOperationException($"Orchestrator function '{orchestratorFunction}' does not exist.");
}
// 1. Start the functions invocation pipeline (billing, logging, bindings, and timeout tracking).
FunctionResult result = await executor.TryExecuteAsync(
new TriggeredFunctionData
{
TriggerValue = context,
#pragma warning disable CS0618 // Approved for use by this extension
InvokeHandler = userCodeInvoker =>
{
// 2. Configure the shim with the inner invoker to execute the user code.
shim.SetFunctionInvocationCallback(userCodeInvoker);
// 3. Move to the next stage of the DTFx pipeline to trigger the orchestrator shim.
return(next());
},
#pragma warning restore CS0618
},
CancellationToken.None);
if (!context.IsCompleted)
{
this.TraceHelper.FunctionAwaited(
context.HubName,
context.Name,
context.Version,
context.InstanceId,
context.IsReplaying);
}
}
public async Task InvokeAsync_Wrapped_SetsActivity()
{
// arrange
var serviceProvider = new Mock <IServiceProvider>();
serviceProvider.Setup(m => m.GetService(typeof(TestActivity))).Returns(new TestActivity());
var wrapper = new WrapperActivity(new TaskActivityDescriptor(typeof(TestActivity)));
DispatchMiddlewareContext context = CreateContext(wrapper);
var middleware = new ServiceProviderActivityMiddleware(serviceProvider.Object);
// act
await middleware.InvokeAsync(context, () => Task.CompletedTask);
// assert
TaskActivity activity = context.GetProperty <TaskActivity>();
activity.Should().NotBeNull();
activity.Should().Be(wrapper.InnerActivity);
serviceProvider.Verify(m => m.GetService(typeof(TestActivity)), Times.Once);
}
/// <inheritdoc />
public async Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next)
{
Check.NotNull(context, nameof(context));
Check.NotNull(next, nameof(next));
TaskOrchestration taskOrchestration = context.GetProperty <TaskOrchestration>();
if (taskOrchestration is WrapperOrchestration wrapper)
{
wrapper.Initialize(_serviceProvider);
// update the context task orchestration with the real one.
context.SetProperty(wrapper.InnerOrchestration);
await next().ConfigureAwait(false);
return;
}
await next().ConfigureAwait(false);
}
/// <inheritdoc />
public async Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next)
{
Check.NotNull(context, nameof(context));
Check.NotNull(next, nameof(next));
TaskActivity taskActivity = context.GetProperty <TaskActivity>();
if (taskActivity is WrapperActivity wrapper)
{
wrapper.InnerActivity = (TaskActivity)_serviceProvider
.GetServiceOrCreateInstance(wrapper.InnerActivityType);
// update the context task activity with the real one.
context.SetProperty(wrapper.InnerActivity);
await next().ConfigureAwait(false);
return;
}
await next().ConfigureAwait(false);
}
private async Task OrchestrationMiddleware(DispatchMiddlewareContext dispatchContext, Func <Task> next)
{
TaskOrchestrationShim shim = (TaskOrchestrationShim)dispatchContext.GetProperty <TaskOrchestration>();
DurableOrchestrationContext context = shim.Context;
string executorKey = GetFunctionKey(context.Name, context.Version);
ITriggeredFunctionExecutor functionsPipelineInvoker;
if (!orchestratorFunctionExecutors.TryGetValue(executorKey, out functionsPipelineInvoker))
{
throw new ArgumentException($"No orchestration function named '{context.Name}' with version '{context.Version}' is registered.");
}
// 1. Start the functions invocation pipeline (billing, logging, bindings, and timeout tracking).
FunctionResult result = await functionsPipelineInvoker.TryExecuteAsync(
new TriggeredFunctionData
{
TriggerValue = context,
InvokeHandler = userCodeInvoker =>
{
// 2. Configure the shim with the inner invoker to execute the user code.
shim.SetFunctionInvocationCallback(userCodeInvoker);
// 3. Move to the next stage of the DTFx pipeline to trigger the orchestrator shim.
return(next());
}
},
CancellationToken.None);
if (!context.IsCompleted)
{
this.config.TraceHelper.FunctionAwaited(
context.HubName,
context.Name,
context.Version,
context.InstanceId,
context.IsReplaying);
}
}
/// <summary>
/// Runs the middleware described by <see cref="TaskMiddlewareDescriptor"/>.
/// </summary>
/// <param name="descriptor">The task middleware to find and run.</param>
/// <param name="context">The dispatch middleware context.</param>
/// <param name="next">The next middleware callback.</param>
/// <returns>A task that completes when the middleware has finished executing.</returns>
public static Task RunAsync(
TaskMiddlewareDescriptor descriptor, DispatchMiddlewareContext context, Func <Task> next)
{
Check.NotNull(descriptor, nameof(descriptor));
Check.NotNull(context, nameof(context));
Check.NotNull(next, nameof(next));
IServiceProvider serviceProvider = context.GetProperty <IServiceProvider>();
ITaskMiddleware middleware = GetMiddleware(descriptor, serviceProvider);
if (middleware is null)
{
// This _should not_ be possible, as TaskMiddlewareDescriptor is designed to only be constructable with
// valid values. But a good sanity check here.
ILogger logger = serviceProvider.CreateLogger(typeof(TaskMiddlewareRunner));
string message = Strings.MiddlewareCreationFailed(descriptor);
logger.LogError(message);
throw new InvalidOperationException(message); // TODO: maybe a better exception type.
}
return(middleware.InvokeAsync(context, next));
}
async Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem)
{
OrchestrationExecutionCursor cursor = workItem.Cursor;
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance);
dispatchContext.SetProperty(cursor.TaskOrchestration);
dispatchContext.SetProperty(cursor.RuntimeState);
dispatchContext.SetProperty(workItem);
cursor.LatestDecisions = Enumerable.Empty <OrchestratorAction>();
await this.dispatchPipeline.RunAsync(dispatchContext, _ =>
{
OrchestratorExecutionResult result = cursor.OrchestrationExecutor.ExecuteNewEvents();
dispatchContext.SetProperty(result);
return(CompletedTask);
});
var result = dispatchContext.GetProperty <OrchestratorExecutionResult>();
cursor.LatestDecisions = result?.Actions ?? Enumerable.Empty <OrchestratorAction>();
cursor.RuntimeState.Status = result?.CustomStatus;
}
private async Task OrchestrationMiddleware(DispatchMiddlewareContext dispatchContext, Func <Task> next)
{
TaskOrchestrationShim shim = dispatchContext.GetProperty <TaskOrchestration>() as TaskOrchestrationShim;
if (shim == null)
{
// This is not an orchestration - skip.
await next();
return;
}
DurableOrchestrationContext context = (DurableOrchestrationContext)shim.Context;
OrchestrationRuntimeState orchestrationRuntimeState = dispatchContext.GetProperty <OrchestrationRuntimeState>();
if (orchestrationRuntimeState.ParentInstance != null)
{
context.ParentInstanceId = orchestrationRuntimeState.ParentInstance.OrchestrationInstance.InstanceId;
}
context.InstanceId = orchestrationRuntimeState.OrchestrationInstance.InstanceId;
context.ExecutionId = orchestrationRuntimeState.OrchestrationInstance.ExecutionId;
context.IsReplaying = orchestrationRuntimeState.ExecutionStartedEvent.IsPlayed;
context.History = orchestrationRuntimeState.Events;
context.RawInput = orchestrationRuntimeState.Input;
var info = shim.GetFunctionInfo();
if (info == null)
{
string message = this.GetInvalidOrchestratorFunctionMessage(context.FunctionName);
this.TraceHelper.ExtensionWarningEvent(
this.Options.HubName,
orchestrationRuntimeState.Name,
orchestrationRuntimeState.OrchestrationInstance.InstanceId,
message);
throw new InvalidOperationException(message);
}
// 1. Start the functions invocation pipeline (billing, logging, bindings, and timeout tracking).
FunctionResult result = await info.Executor.TryExecuteAsync(
new TriggeredFunctionData
{
TriggerValue = context,
#pragma warning disable CS0618 // Approved for use by this extension
InvokeHandler = async userCodeInvoker =>
{
// 2. Configure the shim with the inner invoker to execute the user code.
shim.SetFunctionInvocationCallback(userCodeInvoker);
// 3. Move to the next stage of the DTFx pipeline to trigger the orchestrator shim.
await next();
// 4. If an activity failed, indicate to the functions Host that this execution failed via an exception
if (context.IsCompleted && context.OrchestrationException != null)
{
context.OrchestrationException.Throw();
}
},
#pragma warning restore CS0618
},
CancellationToken.None);
if (!context.IsCompleted)
{
this.TraceHelper.FunctionAwaited(
context.HubName,
context.Name,
FunctionType.Orchestrator,
context.InstanceId,
context.IsReplaying);
}
if (context.IsCompleted &&
context.PreserveUnprocessedEvents)
{
// Reschedule any unprocessed external events so that they can be picked up
// in the next iteration.
context.RescheduleBufferedExternalEvents();
}
await context.RunDeferredTasks();
}
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
{
Task renewTask = null;
var renewCancellationTokenSource = new CancellationTokenSource();
try
{
TaskMessage taskMessage = workItem.TaskMessage;
OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
{
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingOrchestrationInstance",
new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
}
if (taskMessage.Event.EventType != EventType.TaskScheduled)
{
throw TraceHelper.TraceException(
TraceEventType.Critical,
"TaskActivityDispatcher-UnsupportedEventType",
new NotSupportedException("Activity worker does not support event of type: " +
taskMessage.Event.EventType));
}
// call and get return message
var scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
if (taskActivity == null)
{
throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
}
renewTask = Task.Factory.StartNew(() => RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token);
// TODO : pass workflow instance data
var context = new TaskContext(taskMessage.OrchestrationInstance);
HistoryEvent eventToRespond = null;
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
dispatchContext.SetProperty(taskActivity);
dispatchContext.SetProperty(scheduledEvent);
await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
{
try
{
string output = await taskActivity.RunAsync(context, scheduledEvent.Input);
eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
}
catch (TaskFailureException e)
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
string details = IncludeDetails ? e.Details : null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
}
catch (Exception e) when(!Utils.IsFatal(e))
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
string details = IncludeDetails
? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}"
: null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
}
});
var responseTaskMessage = new TaskMessage
{
Event = eventToRespond,
OrchestrationInstance = orchestrationInstance
};
await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
}
finally
{
if (renewTask != null)
{
renewCancellationTokenSource.Cancel();
renewTask.Wait(renewCancellationTokenSource.Token);
}
}
}
private async Task OrchestrationMiddleware(DispatchMiddlewareContext dispatchContext, Func <Task> next)
{
TaskOrchestrationShim shim = (TaskOrchestrationShim)dispatchContext.GetProperty <TaskOrchestration>();
DurableOrchestrationContext context = shim.Context;
OrchestrationRuntimeState orchestrationRuntimeState = dispatchContext.GetProperty <OrchestrationRuntimeState>();
if (orchestrationRuntimeState.ParentInstance != null)
{
context.ParentInstanceId = orchestrationRuntimeState.ParentInstance.OrchestrationInstance.InstanceId;
}
context.InstanceId = orchestrationRuntimeState.OrchestrationInstance.InstanceId;
context.IsReplaying = orchestrationRuntimeState.ExecutionStartedEvent.IsPlayed;
context.History = orchestrationRuntimeState.Events;
context.SetInput(orchestrationRuntimeState.Input);
FunctionName orchestratorFunction = new FunctionName(context.Name);
RegisteredFunctionInfo info;
if (!this.knownOrchestrators.TryGetValue(orchestratorFunction, out info))
{
string message = this.GetInvalidOrchestratorFunctionMessage(orchestratorFunction.Name);
this.TraceHelper.ExtensionWarningEvent(
this.Options.HubName,
orchestratorFunction.Name,
orchestrationRuntimeState.OrchestrationInstance.InstanceId,
message);
throw new InvalidOperationException(message);
}
// 1. Start the functions invocation pipeline (billing, logging, bindings, and timeout tracking).
FunctionResult result = await info.Executor.TryExecuteAsync(
new TriggeredFunctionData
{
TriggerValue = context,
#pragma warning disable CS0618 // Approved for use by this extension
InvokeHandler = async userCodeInvoker =>
{
// 2. Configure the shim with the inner invoker to execute the user code.
shim.SetFunctionInvocationCallback(userCodeInvoker);
// 3. Move to the next stage of the DTFx pipeline to trigger the orchestrator shim.
await next();
// 4. If an activity failed, indicate to the functions Host that this execution failed via an exception
if (context.IsCompleted && context.OrchestrationException != null)
{
context.OrchestrationException.Throw();
}
},
#pragma warning restore CS0618
},
CancellationToken.None);
if (!context.IsCompleted)
{
this.TraceHelper.FunctionAwaited(
context.HubName,
context.Name,
context.InstanceId,
context.IsReplaying);
}
await context.RunDeferredTasks();
}
public Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next) => _func.Invoke(context, next);
public Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next)
{
throw new NotImplementedException();
}
/// <inheritdoc />
public Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next)
{
_console.WriteLine("In sample middleware. Dependency Injection works.");
return(next());
}
private async Task EntityMiddleware(DispatchMiddlewareContext dispatchContext, Func <Task> next)
{
var entityShim = dispatchContext.GetProperty <TaskOrchestration>() as TaskEntityShim;
if (entityShim == null)
{
// This is not an entity - skip.
await next();
return;
}
OrchestrationRuntimeState runtimeState = dispatchContext.GetProperty <OrchestrationRuntimeState>();
DurableEntityContext entityContext = (DurableEntityContext)entityShim.Context;
entityContext.InstanceId = runtimeState.OrchestrationInstance.InstanceId;
entityContext.ExecutionId = runtimeState.OrchestrationInstance.ExecutionId;
entityContext.History = runtimeState.Events;
entityContext.RawInput = runtimeState.Input;
try
{
// 1. First time through the history
// we count events, add any under-lock op to the batch, and process lock releases
foreach (HistoryEvent e in runtimeState.Events)
{
switch (e.EventType)
{
case EventType.ExecutionStarted:
entityShim.Rehydrate(runtimeState.Input);
break;
case EventType.EventRaised:
EventRaisedEvent eventRaisedEvent = (EventRaisedEvent)e;
this.TraceHelper.DeliveringEntityMessage(
entityContext.InstanceId,
entityContext.ExecutionId,
e.EventId,
eventRaisedEvent.Name,
eventRaisedEvent.Input);
entityShim.NumberEventsToReceive++;
if (eventRaisedEvent.Name == "op")
{
// we are receiving an operation request or a lock request
var requestMessage = JsonConvert.DeserializeObject <RequestMessage>(eventRaisedEvent.Input);
// run this through the message sorter to help with reordering and duplicate filtering
var deliverNow = entityContext.State.MessageSorter.ReceiveInOrder(requestMessage, entityContext.EntityMessageReorderWindow);
foreach (var message in deliverNow)
{
if (entityContext.State.LockedBy == message.ParentInstanceId)
{
// operation requests from the lock holder are processed immediately
entityShim.AddOperationToBatch(message);
}
else
{
// others go to the back of the queue
entityContext.State.Enqueue(message);
}
}
}
else
{
// we are receiving a lock release
var message = JsonConvert.DeserializeObject <ReleaseMessage>(eventRaisedEvent.Input);
if (entityContext.State.LockedBy == message.ParentInstanceId)
{
this.TraceHelper.EntityLockReleased(
entityContext.HubName,
entityContext.Name,
entityContext.InstanceId,
message.ParentInstanceId,
message.LockRequestId,
isReplay: false);
entityContext.State.LockedBy = null;
}
}
break;
}
}
// 2. We add as many requests from the queue to the batch as possible (stopping at lock requests)
while (entityContext.State.LockedBy == null &&
entityContext.State.TryDequeue(out var request))
{
if (request.IsLockRequest)
{
entityShim.AddLockRequestToBatch(request);
entityContext.State.LockedBy = request.ParentInstanceId;
}
else
{
entityShim.AddOperationToBatch(request);
}
}
}
catch (Exception e)
{
entityContext.CaptureInternalError(e);
}
// 3. Start the functions invocation pipeline (billing, logging, bindings, and timeout tracking).
FunctionResult result = await entityShim.GetFunctionInfo().Executor.TryExecuteAsync(
new TriggeredFunctionData
{
TriggerValue = entityShim.Context,
#pragma warning disable CS0618 // Approved for use by this extension
InvokeHandler = async userCodeInvoker =>
{
entityShim.SetFunctionInvocationCallback(userCodeInvoker);
// 3. Run all the operations in the batch
if (entityContext.InternalError == null)
{
try
{
await entityShim.ExecuteBatch();
}
catch (Exception e)
{
entityContext.CaptureInternalError(e);
}
}
// 4. Run the DTFx orchestration to persist the effects,
// send the outbox, and continue as new
await next();
// 5. If there were internal or application errors, indicate to the functions host
entityContext.ThrowInternalExceptionIfAny();
entityContext.ThrowApplicationExceptionsIfAny();
},
#pragma warning restore CS0618
},
CancellationToken.None);
await entityContext.RunDeferredTasks();
// If there were internal errors, do not commit the batch, but instead rethrow
// here so DTFx can abort the batch and back off the work item
entityContext.ThrowInternalExceptionIfAny();
}
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
{
Task renewTask = null;
var renewCancellationTokenSource = new CancellationTokenSource();
TaskMessage taskMessage = workItem.TaskMessage;
OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
TaskScheduledEvent scheduledEvent = null;
Activity diagnosticActivity = null;
try
{
if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received a message that does not have any OrchestrationInstance information.");
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingOrchestrationInstance",
new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
}
if (taskMessage.Event.EventType != EventType.TaskScheduled)
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported.");
throw TraceHelper.TraceException(
TraceEventType.Critical,
"TaskActivityDispatcher-UnsupportedEventType",
new NotSupportedException("Activity worker does not support event of type: " +
taskMessage.Event.EventType));
}
// call and get return message
scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent);
TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
if (taskActivity == null)
{
throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
}
if (workItem.LockedUntilUtc < DateTime.MaxValue)
{
// start a task to run RenewUntil
renewTask = Task.Factory.StartNew(
() => this.RenewUntil(workItem, renewCancellationTokenSource.Token),
renewCancellationTokenSource.Token);
}
// TODO : pass workflow instance data
var context = new TaskContext(taskMessage.OrchestrationInstance);
HistoryEvent eventToRespond = null;
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
dispatchContext.SetProperty(taskActivity);
dispatchContext.SetProperty(scheduledEvent);
// correlation
CorrelationTraceClient.Propagate(() =>
{
workItem.TraceContextBase?.SetActivityToCurrent();
diagnosticActivity = workItem.TraceContextBase?.CurrentActivity;
});
await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
{
try
{
string output = await taskActivity.RunAsync(context, scheduledEvent.Input);
eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
}
catch (TaskFailureException e)
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
string details = this.IncludeDetails ? e.Details : null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e);
CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e));
}
catch (Exception e) when(!Utils.IsFatal(e) && !Utils.IsExecutionAborting(e))
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
string details = this.IncludeDetails
? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}"
: null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e);
}
if (eventToRespond is TaskCompletedEvent completedEvent)
{
this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent);
}
});
var responseTaskMessage = new TaskMessage
{
Event = eventToRespond,
OrchestrationInstance = orchestrationInstance
};
await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
}
catch (SessionAbortedException e)
{
// The activity aborted its execution
this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message);
TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent.Name, e.Message);
await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem);
}
finally
{
diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out.
if (renewTask != null)
{
renewCancellationTokenSource.Cancel();
try
{
// wait the renewTask finish
await renewTask;
}
catch (OperationCanceledException)
{
// ignore
}
}
}
}
public Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next)
{
_tracker.ExecutionCount++;
return(next());
}
public abstract Task InvokeAsync(DispatchMiddlewareContext context, Func <Task> next);
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
{
Task?renewTask = null;
using var renewCancellationTokenSource = new CancellationTokenSource();
TaskMessage taskMessage = workItem.TaskMessage;
OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
TaskScheduledEvent? scheduledEvent = null;
Activity?diagnosticActivity = null;
try
{
if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received a message that does not have any OrchestrationInstance information.");
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingOrchestrationInstance",
new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
}
if (taskMessage.Event.EventType != EventType.TaskScheduled)
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported.");
throw TraceHelper.TraceException(
TraceEventType.Critical,
"TaskActivityDispatcher-UnsupportedEventType",
new NotSupportedException("Activity worker does not support event of type: " +
taskMessage.Event.EventType));
}
scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
if (scheduledEvent.Name == null)
{
string message = $"The activity worker received a {nameof(EventType.TaskScheduled)} event that does not specify an activity name.";
this.logHelper.TaskActivityDispatcherError(workItem, message);
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingActivityName",
new InvalidOperationException(message));
}
this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent);
TaskActivity?taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
if (workItem.LockedUntilUtc < DateTime.MaxValue)
{
// start a task to run RenewUntil
renewTask = Task.Factory.StartNew(
() => this.RenewUntil(workItem, renewCancellationTokenSource.Token),
renewCancellationTokenSource.Token);
}
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
dispatchContext.SetProperty(taskActivity);
dispatchContext.SetProperty(scheduledEvent);
// correlation
CorrelationTraceClient.Propagate(() =>
{
workItem.TraceContextBase?.SetActivityToCurrent();
diagnosticActivity = workItem.TraceContextBase?.CurrentActivity;
});
ActivityExecutionResult?result;
try
{
await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
{
if (taskActivity == null)
{
// This likely indicates a deployment error of some kind. Because these unhandled exceptions are
// automatically retried, resolving this may require redeploying the app code so that the activity exists again.
// CONSIDER: Should this be changed into a permanent error that fails the orchestration? Perhaps
// the app owner doesn't care to preserve existing instances when doing code deployments?
throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
}
var context = new TaskContext(taskMessage.OrchestrationInstance);
context.ErrorPropagationMode = this.errorPropagationMode;
HistoryEvent?responseEvent;
try
{
string?output = await taskActivity.RunAsync(context, scheduledEvent.Input);
responseEvent = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
}
catch (Exception e) when(e is not TaskFailureException && !Utils.IsFatal(e) && !Utils.IsExecutionAborting(e))
{
// These are unexpected exceptions that occur in the task activity abstraction. Normal exceptions from
// activities are expected to be translated into TaskFailureException and handled outside the middleware
// context (see further below).
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
string?details = this.IncludeDetails
? $"Unhandled exception while executing task: {e}"
: null;
responseEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, new FailureDetails(e));
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)responseEvent, e);
}
var result = new ActivityExecutionResult {
ResponseEvent = responseEvent
};
dispatchContext.SetProperty(result);
});
result = dispatchContext.GetProperty <ActivityExecutionResult>();
}
catch (TaskFailureException e)
{
// These are normal task activity failures. They can come from Activity implementations or from middleware.
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
string?details = this.IncludeDetails ? e.Details : null;
var failureEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, e.FailureDetails);
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, failureEvent, e);
CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e));
result = new ActivityExecutionResult {
ResponseEvent = failureEvent
};
}
catch (Exception middlewareException) when(!Utils.IsFatal(middlewareException))
{
// These are considered retriable
this.logHelper.TaskActivityDispatcherError(workItem, $"Unhandled exception in activity middleware pipeline: {middlewareException}");
throw;
}
HistoryEvent?eventToRespond = result?.ResponseEvent;
if (eventToRespond is TaskCompletedEvent completedEvent)
{
this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent);
}
else if (eventToRespond is null)
{
// Default response if middleware prevents a response from being generated
eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, null);
}
var responseTaskMessage = new TaskMessage
{
Event = eventToRespond,
OrchestrationInstance = orchestrationInstance
};
await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
}
catch (SessionAbortedException e)
{
// The activity aborted its execution
this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message);
TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent?.Name, e.Message);
await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem);
}
finally
{
diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out.
if (renewTask != null)
{
renewCancellationTokenSource.Cancel();
try
{
// wait the renewTask finish
await renewTask;
}
catch (OperationCanceledException)
{
// ignore
}
}
}
}
