private DurableTaskGrpc.TaskActivityWorkItem ToGrpcWorkItem(TaskActivityWorkItem workItem)
{
return(new DurableTaskGrpc.TaskActivityWorkItem
{
Id = workItem.Id,
LockedUntilUtc = Timestamp.FromDateTime(workItem.LockedUntilUtc),
TaskMessage = _options.DataConverter.Serialize(workItem.TaskMessage)
});
}
public async Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem)
{
var request = new AbandonTaskActivityWorkItemRequest
{
WorkItem = ToGrpcWorkItem(workItem)
};
await _client.AbandonTaskActivityWorkItemAsync(request);
}
async Task RenewUntil(TaskActivityWorkItem workItem, CancellationToken cancellationToken)
{
try
{
if (workItem.LockedUntilUtc < DateTime.UtcNow)
{
return;
}
DateTime renewAt = workItem.LockedUntilUtc.Subtract(TimeSpan.FromSeconds(30));
// service bus clock sku can really mess us up so just always renew every 30 secs regardless of
// what the message.LockedUntilUtc says. if the sku is negative then in the worst case we will be
// renewing every 5 secs
//
renewAt = this.AdjustRenewAt(renewAt);
while (!cancellationToken.IsCancellationRequested)
{
await Utils.DelayWithCancellation(TimeSpan.FromSeconds(5), cancellationToken);
if (DateTime.UtcNow < renewAt || cancellationToken.IsCancellationRequested)
{
continue;
}
try
{
this.logHelper.RenewActivityMessageStarting(workItem);
TraceHelper.Trace(TraceEventType.Information, "TaskActivityDispatcher-RenewLock", "Renewing lock for work item id {0}", workItem.Id);
workItem = await this.orchestrationService.RenewTaskActivityWorkItemLockAsync(workItem);
renewAt = workItem.LockedUntilUtc.Subtract(TimeSpan.FromSeconds(30));
renewAt = this.AdjustRenewAt(renewAt);
this.logHelper.RenewActivityMessageCompleted(workItem, renewAt);
TraceHelper.Trace(TraceEventType.Information, "TaskActivityDispatcher-RenewLockAt", "Next renew for work item id '{0}' at '{1}'", workItem.Id, renewAt);
}
catch (Exception exception) when(!Utils.IsFatal(exception))
{
// might have been completed
this.logHelper.RenewActivityMessageFailed(workItem, exception);
TraceHelper.TraceException(TraceEventType.Warning, "TaskActivityDispatcher-RenewLockFailure", exception, "Failed to renew lock for work item {0}", workItem.Id);
break;
}
}
}
catch (OperationCanceledException)
{
// cancellation was triggered
}
catch (ObjectDisposedException)
{
// brokered message is already disposed probably through
// a complete call in the main dispatcher thread
}
}
public async Task <TaskActivityWorkItem> RenewTaskActivityWorkItemLockAsync(TaskActivityWorkItem workItem)
{
var request = new RenewTaskActivityWorkItemLockRequest
{
WorkItem = ToGrpcWorkItem(workItem)
};
var response = await _client.RenewTaskActivityWorkItemLockAsync(request);
return(ToDurableTaskWorkItem(response.WorkItem));
}
public async Task CompleteTaskActivityWorkItemAsync(
TaskActivityWorkItem workItem,
TaskMessage responseMessage)
{
var request = new CompleteTaskActivityWorkItemRequest
{
WorkItem = ToGrpcWorkItem(workItem),
ResponseMessage = _options.DataConverter.Serialize(responseMessage)
};
await _client.CompleteTaskActivityWorkItemAsync(request);
}
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
{
Task renewTask = null;
var renewCancellationTokenSource = new CancellationTokenSource();
TaskMessage taskMessage = workItem.TaskMessage;
OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
TaskScheduledEvent scheduledEvent = null;
Activity diagnosticActivity = null;
try
{
if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received a message that does not have any OrchestrationInstance information.");
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingOrchestrationInstance",
new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
}
if (taskMessage.Event.EventType != EventType.TaskScheduled)
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported.");
throw TraceHelper.TraceException(
TraceEventType.Critical,
"TaskActivityDispatcher-UnsupportedEventType",
new NotSupportedException("Activity worker does not support event of type: " +
taskMessage.Event.EventType));
}
// call and get return message
scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent);
TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
if (taskActivity == null)
{
throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
}
if (workItem.LockedUntilUtc < DateTime.MaxValue)
{
// start a task to run RenewUntil
renewTask = Task.Factory.StartNew(
() => this.RenewUntil(workItem, renewCancellationTokenSource.Token),
renewCancellationTokenSource.Token);
}
// TODO : pass workflow instance data
var context = new TaskContext(taskMessage.OrchestrationInstance);
HistoryEvent eventToRespond = null;
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
dispatchContext.SetProperty(taskActivity);
dispatchContext.SetProperty(scheduledEvent);
// correlation
CorrelationTraceClient.Propagate(() =>
{
workItem.TraceContextBase?.SetActivityToCurrent();
diagnosticActivity = workItem.TraceContextBase?.CurrentActivity;
});
await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
{
try
{
string output = await taskActivity.RunAsync(context, scheduledEvent.Input);
eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
}
catch (TaskFailureException e)
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
string details = this.IncludeDetails ? e.Details : null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e);
CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e));
}
catch (Exception e) when(!Utils.IsFatal(e) && !Utils.IsExecutionAborting(e))
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
string details = this.IncludeDetails
? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}"
: null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e);
}
if (eventToRespond is TaskCompletedEvent completedEvent)
{
this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent);
}
});
var responseTaskMessage = new TaskMessage
{
Event = eventToRespond,
OrchestrationInstance = orchestrationInstance
};
await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
}
catch (SessionAbortedException e)
{
// The activity aborted its execution
this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message);
TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent.Name, e.Message);
await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem);
}
finally
{
diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out.
if (renewTask != null)
{
renewCancellationTokenSource.Cancel();
try
{
// wait the renewTask finish
await renewTask;
}
catch (OperationCanceledException)
{
// ignore
}
}
}
}
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
{
Task renewTask = null;
var renewCancellationTokenSource = new CancellationTokenSource();
try
{
TaskMessage taskMessage = workItem.TaskMessage;
OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
{
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingOrchestrationInstance",
new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
}
if (taskMessage.Event.EventType != EventType.TaskScheduled)
{
throw TraceHelper.TraceException(
TraceEventType.Critical,
"TaskActivityDispatcher-UnsupportedEventType",
new NotSupportedException("Activity worker does not support event of type: " +
taskMessage.Event.EventType));
}
// call and get return message
var scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
if (taskActivity == null)
{
throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
}
renewTask = Task.Factory.StartNew(() => RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token);
// TODO : pass workflow instance data
var context = new TaskContext(taskMessage.OrchestrationInstance);
HistoryEvent eventToRespond = null;
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
dispatchContext.SetProperty(taskActivity);
dispatchContext.SetProperty(scheduledEvent);
await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
{
try
{
string output = await taskActivity.RunAsync(context, scheduledEvent.Input);
eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
}
catch (TaskFailureException e)
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
string details = IncludeDetails ? e.Details : null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
}
catch (Exception e) when(!Utils.IsFatal(e))
{
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
string details = IncludeDetails
? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}"
: null;
eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details);
}
});
var responseTaskMessage = new TaskMessage
{
Event = eventToRespond,
OrchestrationInstance = orchestrationInstance
};
await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
}
finally
{
if (renewTask != null)
{
renewCancellationTokenSource.Cancel();
renewTask.Wait(renewCancellationTokenSource.Token);
}
}
}
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem)
{
Task?renewTask = null;
using var renewCancellationTokenSource = new CancellationTokenSource();
TaskMessage taskMessage = workItem.TaskMessage;
OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance;
TaskScheduledEvent? scheduledEvent = null;
Activity?diagnosticActivity = null;
try
{
if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId))
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received a message that does not have any OrchestrationInstance information.");
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingOrchestrationInstance",
new InvalidOperationException("Message does not contain any OrchestrationInstance information"));
}
if (taskMessage.Event.EventType != EventType.TaskScheduled)
{
this.logHelper.TaskActivityDispatcherError(
workItem,
$"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported.");
throw TraceHelper.TraceException(
TraceEventType.Critical,
"TaskActivityDispatcher-UnsupportedEventType",
new NotSupportedException("Activity worker does not support event of type: " +
taskMessage.Event.EventType));
}
scheduledEvent = (TaskScheduledEvent)taskMessage.Event;
if (scheduledEvent.Name == null)
{
string message = $"The activity worker received a {nameof(EventType.TaskScheduled)} event that does not specify an activity name.";
this.logHelper.TaskActivityDispatcherError(workItem, message);
throw TraceHelper.TraceException(
TraceEventType.Error,
"TaskActivityDispatcher-MissingActivityName",
new InvalidOperationException(message));
}
this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent);
TaskActivity?taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version);
if (workItem.LockedUntilUtc < DateTime.MaxValue)
{
// start a task to run RenewUntil
renewTask = Task.Factory.StartNew(
() => this.RenewUntil(workItem, renewCancellationTokenSource.Token),
renewCancellationTokenSource.Token);
}
var dispatchContext = new DispatchMiddlewareContext();
dispatchContext.SetProperty(taskMessage.OrchestrationInstance);
dispatchContext.SetProperty(taskActivity);
dispatchContext.SetProperty(scheduledEvent);
// correlation
CorrelationTraceClient.Propagate(() =>
{
workItem.TraceContextBase?.SetActivityToCurrent();
diagnosticActivity = workItem.TraceContextBase?.CurrentActivity;
});
ActivityExecutionResult?result;
try
{
await this.dispatchPipeline.RunAsync(dispatchContext, async _ =>
{
if (taskActivity == null)
{
// This likely indicates a deployment error of some kind. Because these unhandled exceptions are
// automatically retried, resolving this may require redeploying the app code so that the activity exists again.
// CONSIDER: Should this be changed into a permanent error that fails the orchestration? Perhaps
// the app owner doesn't care to preserve existing instances when doing code deployments?
throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found");
}
var context = new TaskContext(taskMessage.OrchestrationInstance);
context.ErrorPropagationMode = this.errorPropagationMode;
HistoryEvent?responseEvent;
try
{
string?output = await taskActivity.RunAsync(context, scheduledEvent.Input);
responseEvent = new TaskCompletedEvent(-1, scheduledEvent.EventId, output);
}
catch (Exception e) when(e is not TaskFailureException && !Utils.IsFatal(e) && !Utils.IsExecutionAborting(e))
{
// These are unexpected exceptions that occur in the task activity abstraction. Normal exceptions from
// activities are expected to be translated into TaskFailureException and handled outside the middleware
// context (see further below).
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e);
string?details = this.IncludeDetails
? $"Unhandled exception while executing task: {e}"
: null;
responseEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, new FailureDetails(e));
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)responseEvent, e);
}
var result = new ActivityExecutionResult {
ResponseEvent = responseEvent
};
dispatchContext.SetProperty(result);
});
result = dispatchContext.GetProperty <ActivityExecutionResult>();
}
catch (TaskFailureException e)
{
// These are normal task activity failures. They can come from Activity implementations or from middleware.
TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e);
string?details = this.IncludeDetails ? e.Details : null;
var failureEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, e.FailureDetails);
this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, failureEvent, e);
CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e));
result = new ActivityExecutionResult {
ResponseEvent = failureEvent
};
}
catch (Exception middlewareException) when(!Utils.IsFatal(middlewareException))
{
// These are considered retriable
this.logHelper.TaskActivityDispatcherError(workItem, $"Unhandled exception in activity middleware pipeline: {middlewareException}");
throw;
}
HistoryEvent?eventToRespond = result?.ResponseEvent;
if (eventToRespond is TaskCompletedEvent completedEvent)
{
this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent);
}
else if (eventToRespond is null)
{
// Default response if middleware prevents a response from being generated
eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, null);
}
var responseTaskMessage = new TaskMessage
{
Event = eventToRespond,
OrchestrationInstance = orchestrationInstance
};
await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage);
}
catch (SessionAbortedException e)
{
// The activity aborted its execution
this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message);
TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent?.Name, e.Message);
await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem);
}
finally
{
diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out.
if (renewTask != null)
{
renewCancellationTokenSource.Cancel();
try
{
// wait the renewTask finish
await renewTask;
}
catch (OperationCanceledException)
{
// ignore
}
}
}
}
