/// <exception cref="System.IO.IOException"/> public HistoryEvent Answer(InvocationOnMock invocation) { int eventId = numEventsRead.GetAndIncrement(); TaskID tid = tids[eventId & unchecked ((int)(0x1))]; if (eventId < 2) { return(new TaskStartedEvent(tid, 0, taskType, string.Empty)); } if (eventId < 4) { TaskFailedEvent tfe = new TaskFailedEvent(tid, 0, taskType, "failed", "FAILED", null , new Counters()); tfe.SetDatum(tfe.GetDatum()); return(tfe); } if (eventId < 5) { JobUnsuccessfulCompletionEvent juce = new JobUnsuccessfulCompletionEvent(jid, 100L , 2, 0, "JOB_FAILED", Sharpen.Collections.SingletonList("Task failed: " + tids[0 ].ToString())); return(juce); } return(null); }
/// <summary> /// Logs that an activity failed with an unhandled exception. /// </summary> /// <param name="instance">The orchestration instance that scheduled this task activity.</param> /// <param name="name">The name of the task activity.</param> /// <param name="failedEvent">The history event associated with this activity failure.</param> /// <param name="exception">The unhandled exception.</param> internal void TaskActivityFailure( OrchestrationInstance instance, string name, TaskFailedEvent failedEvent, Exception exception) { if (this.IsStructuredLoggingEnabled) { this.WriteStructuredLog( new LogEvents.TaskActivityFailure(instance, name, failedEvent, exception), exception); } }
int GetTaskScheduledId(HistoryEvent historyEvent) { TaskCompletedEvent tce = historyEvent as TaskCompletedEvent; if (tce != null) { return(tce.TaskScheduledId); } TaskFailedEvent tfe = historyEvent as TaskFailedEvent; if (tfe != null) { return(tfe.TaskScheduledId); } return(-1); }
public void HandleTaskFailedEvent(TaskFailedEvent failedEvent) { int taskId = failedEvent.TaskScheduledId; if (openTasks.ContainsKey(taskId)) { OpenTaskInfo info = openTasks[taskId]; Exception cause = Utils.RetrieveCause(failedEvent.Details, dataConverter); var taskFailedException = new TaskFailedException(failedEvent.EventId, taskId, info.Name, info.Version, failedEvent.Reason, cause); TaskCompletionSource <string> tcs = info.Result; tcs.SetException(taskFailedException); openTasks.Remove(taskId); } }
public void HandleTaskFailedEvent(TaskFailedEvent failedEvent) { int taskId = failedEvent.TaskScheduledId; if (this.openTasks.ContainsKey(taskId)) { OpenTaskInfo info = this.openTasks[taskId]; // When using ErrorPropagationMode.SerializeExceptions the "cause" is deserialized from history. // This isn't fully reliable because not all exception types can be serialized/deserialized. // When using ErrorPropagationMode.UseFailureDetails we instead use FailureDetails to convey // error information, which doesn't involve any serialization at all. Exception cause = this.ErrorPropagationMode == ErrorPropagationMode.SerializeExceptions ? Utils.RetrieveCause(failedEvent.Details, this.ErrorDataConverter) : null; var taskFailedException = new TaskFailedException( failedEvent.EventId, taskId, info.Name, info.Version, failedEvent.Reason, cause); taskFailedException.FailureDetails = failedEvent.FailureDetails; TaskCompletionSource <string> tcs = info.Result; tcs.SetException(taskFailedException); this.openTasks.Remove(taskId); } else { LogDuplicateEvent("TaskFailed", failedEvent, taskId); } }
HistoryEvent GenerateAbridgedEvent(HistoryEvent evt) { HistoryEvent returnedEvent = evt; if (evt is TaskScheduledEvent) { var sourceEvent = (TaskScheduledEvent) evt; returnedEvent = new TaskScheduledEvent(sourceEvent.EventId) { Timestamp = sourceEvent.Timestamp, IsPlayed = sourceEvent.IsPlayed, Name = sourceEvent.Name, Version = sourceEvent.Version, Input = "[..snipped..]", }; } else if (evt is TaskCompletedEvent) { var sourceEvent = (TaskCompletedEvent) evt; returnedEvent = new TaskCompletedEvent(sourceEvent.EventId, sourceEvent.TaskScheduledId, "[..snipped..]") { Timestamp = sourceEvent.Timestamp, IsPlayed = sourceEvent.IsPlayed, }; } else if (evt is SubOrchestrationInstanceCreatedEvent) { var sourceEvent = (SubOrchestrationInstanceCreatedEvent) evt; returnedEvent = new SubOrchestrationInstanceCreatedEvent(sourceEvent.EventId) { Timestamp = sourceEvent.Timestamp, IsPlayed = sourceEvent.IsPlayed, Name = sourceEvent.Name, Version = sourceEvent.Version, Input = "[..snipped..]", }; } else if (evt is SubOrchestrationInstanceCompletedEvent) { var sourceEvent = (SubOrchestrationInstanceCompletedEvent) evt; returnedEvent = new SubOrchestrationInstanceCompletedEvent(sourceEvent.EventId, sourceEvent.TaskScheduledId, "[..snipped..]") { Timestamp = sourceEvent.Timestamp, IsPlayed = sourceEvent.IsPlayed, }; } else if (evt is TaskFailedEvent) { var sourceEvent = (TaskFailedEvent) evt; returnedEvent = new TaskFailedEvent(sourceEvent.EventId, sourceEvent.TaskScheduledId, sourceEvent.Reason, "[..snipped..]") { Timestamp = sourceEvent.Timestamp, IsPlayed = sourceEvent.IsPlayed, }; } else if (evt is SubOrchestrationInstanceFailedEvent) { var sourceEvent = (SubOrchestrationInstanceFailedEvent) evt; returnedEvent = new SubOrchestrationInstanceFailedEvent(sourceEvent.EventId, sourceEvent.TaskScheduledId, sourceEvent.Reason, "[..snipped..]") { Timestamp = sourceEvent.Timestamp, IsPlayed = sourceEvent.IsPlayed, }; } else if (evt is ExecutionStartedEvent) { var sourceEvent = (ExecutionStartedEvent) evt; returnedEvent = new ExecutionStartedEvent(sourceEvent.EventId, "[..snipped..]") { Timestamp = sourceEvent.Timestamp, IsPlayed = sourceEvent.IsPlayed, }; } else if (evt is ExecutionCompletedEvent) { var sourceEvent = (ExecutionCompletedEvent) evt; returnedEvent = new ExecutionCompletedEvent(sourceEvent.EventId, "[..snipped..]", sourceEvent.OrchestrationStatus) { Timestamp = sourceEvent.Timestamp, IsPlayed = sourceEvent.IsPlayed, }; } else if (evt is ExecutionTerminatedEvent) { var sourceEvent = (ExecutionTerminatedEvent) evt; returnedEvent = new ExecutionTerminatedEvent(sourceEvent.EventId, "[..snipped..]") { Timestamp = sourceEvent.Timestamp, IsPlayed = sourceEvent.IsPlayed, }; } // ContinueAsNewEvent is covered by the ExecutionCompletedEvent block return returnedEvent; }
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem) { Task renewTask = null; var renewCancellationTokenSource = new CancellationTokenSource(); TaskMessage taskMessage = workItem.TaskMessage; OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; TaskScheduledEvent scheduledEvent = null; Activity diagnosticActivity = null; try { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received a message that does not have any OrchestrationInstance information."); throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingOrchestrationInstance", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported."); throw TraceHelper.TraceException( TraceEventType.Critical, "TaskActivityDispatcher-UnsupportedEventType", new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } // call and get return message scheduledEvent = (TaskScheduledEvent)taskMessage.Event; this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent); TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (taskActivity == null) { throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found"); } if (workItem.LockedUntilUtc < DateTime.MaxValue) { // start a task to run RenewUntil renewTask = Task.Factory.StartNew( () => this.RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); } // TODO : pass workflow instance data var context = new TaskContext(taskMessage.OrchestrationInstance); HistoryEvent eventToRespond = null; var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(taskMessage.OrchestrationInstance); dispatchContext.SetProperty(taskActivity); dispatchContext.SetProperty(scheduledEvent); // correlation CorrelationTraceClient.Propagate(() => { workItem.TraceContextBase?.SetActivityToCurrent(); diagnosticActivity = workItem.TraceContextBase?.CurrentActivity; }); await this.dispatchPipeline.RunAsync(dispatchContext, async _ => { try { string output = await taskActivity.RunAsync(context, scheduledEvent.Input); eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (TaskFailureException e) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e); string details = this.IncludeDetails ? e.Details : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e); CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e)); } catch (Exception e) when(!Utils.IsFatal(e) && !Utils.IsExecutionAborting(e)) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e); string details = this.IncludeDetails ? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}" : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)eventToRespond, e); } if (eventToRespond is TaskCompletedEvent completedEvent) { this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent); } }); var responseTaskMessage = new TaskMessage { Event = eventToRespond, OrchestrationInstance = orchestrationInstance }; await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage); } catch (SessionAbortedException e) { // The activity aborted its execution this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message); TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent.Name, e.Message); await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem); } finally { diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out. if (renewTask != null) { renewCancellationTokenSource.Cancel(); try { // wait the renewTask finish await renewTask; } catch (OperationCanceledException) { // ignore } } } }
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem) { Task renewTask = null; var renewCancellationTokenSource = new CancellationTokenSource(); try { TaskMessage taskMessage = workItem.TaskMessage; OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingOrchestrationInstance", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { throw TraceHelper.TraceException( TraceEventType.Critical, "TaskActivityDispatcher-UnsupportedEventType", new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } // call and get return message var scheduledEvent = (TaskScheduledEvent)taskMessage.Event; TaskActivity taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (taskActivity == null) { throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found"); } renewTask = Task.Factory.StartNew(() => RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); // TODO : pass workflow instance data var context = new TaskContext(taskMessage.OrchestrationInstance); HistoryEvent eventToRespond = null; var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(taskMessage.OrchestrationInstance); dispatchContext.SetProperty(taskActivity); dispatchContext.SetProperty(scheduledEvent); await this.dispatchPipeline.RunAsync(dispatchContext, async _ => { try { string output = await taskActivity.RunAsync(context, scheduledEvent.Input); eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (TaskFailureException e) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e); string details = IncludeDetails ? e.Details : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); } catch (Exception e) when(!Utils.IsFatal(e)) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e); string details = IncludeDetails ? $"Unhandled exception while executing task: {e}\n\t{e.StackTrace}" : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); } }); var responseTaskMessage = new TaskMessage { Event = eventToRespond, OrchestrationInstance = orchestrationInstance }; await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage); } finally { if (renewTask != null) { renewCancellationTokenSource.Cancel(); renewTask.Wait(renewCancellationTokenSource.Token); } } }
HistoryEvent GenerateAbridgedEvent(HistoryEvent evt) { HistoryEvent returnedEvent = evt; if (evt is TaskScheduledEvent taskScheduledEvent) { returnedEvent = new TaskScheduledEvent(taskScheduledEvent.EventId) { Timestamp = taskScheduledEvent.Timestamp, IsPlayed = taskScheduledEvent.IsPlayed, Name = taskScheduledEvent.Name, Version = taskScheduledEvent.Version, Input = "[..snipped..]", }; } else if (evt is TaskCompletedEvent taskCompletedEvent) { returnedEvent = new TaskCompletedEvent(taskCompletedEvent.EventId, taskCompletedEvent.TaskScheduledId, "[..snipped..]") { Timestamp = taskCompletedEvent.Timestamp, IsPlayed = taskCompletedEvent.IsPlayed, }; } else if (evt is SubOrchestrationInstanceCreatedEvent subOrchestrationInstanceCreatedEvent) { returnedEvent = new SubOrchestrationInstanceCreatedEvent(subOrchestrationInstanceCreatedEvent.EventId) { Timestamp = subOrchestrationInstanceCreatedEvent.Timestamp, IsPlayed = subOrchestrationInstanceCreatedEvent.IsPlayed, Name = subOrchestrationInstanceCreatedEvent.Name, Version = subOrchestrationInstanceCreatedEvent.Version, Input = "[..snipped..]", }; } else if (evt is SubOrchestrationInstanceCompletedEvent subOrchestrationInstanceCompletedEvent) { returnedEvent = new SubOrchestrationInstanceCompletedEvent(subOrchestrationInstanceCompletedEvent.EventId, subOrchestrationInstanceCompletedEvent.TaskScheduledId, "[..snipped..]") { Timestamp = subOrchestrationInstanceCompletedEvent.Timestamp, IsPlayed = subOrchestrationInstanceCompletedEvent.IsPlayed, }; } else if (evt is TaskFailedEvent taskFailedEvent) { returnedEvent = new TaskFailedEvent(taskFailedEvent.EventId, taskFailedEvent.TaskScheduledId, taskFailedEvent.Reason, "[..snipped..]") { Timestamp = taskFailedEvent.Timestamp, IsPlayed = taskFailedEvent.IsPlayed, }; } else if (evt is SubOrchestrationInstanceFailedEvent subOrchestrationInstanceFailedEvent) { returnedEvent = new SubOrchestrationInstanceFailedEvent(subOrchestrationInstanceFailedEvent.EventId, subOrchestrationInstanceFailedEvent.TaskScheduledId, subOrchestrationInstanceFailedEvent.Reason, "[..snipped..]") { Timestamp = subOrchestrationInstanceFailedEvent.Timestamp, IsPlayed = subOrchestrationInstanceFailedEvent.IsPlayed, }; } else if (evt is ExecutionStartedEvent executionStartedEvent) { returnedEvent = new ExecutionStartedEvent(executionStartedEvent.EventId, "[..snipped..]") { Timestamp = executionStartedEvent.Timestamp, IsPlayed = executionStartedEvent.IsPlayed, }; } else if (evt is ExecutionCompletedEvent executionCompletedEvent) { returnedEvent = new ExecutionCompletedEvent(executionCompletedEvent.EventId, "[..snipped..]", executionCompletedEvent.OrchestrationStatus) { Timestamp = executionCompletedEvent.Timestamp, IsPlayed = executionCompletedEvent.IsPlayed, }; } else if (evt is ExecutionTerminatedEvent executionTerminatedEvent) { returnedEvent = new ExecutionTerminatedEvent(executionTerminatedEvent.EventId, "[..snipped..]") { Timestamp = executionTerminatedEvent.Timestamp, IsPlayed = executionTerminatedEvent.IsPlayed, }; } // ContinueAsNewEvent is covered by the ExecutionCompletedEvent block return(returnedEvent); }
async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem) { Task?renewTask = null; using var renewCancellationTokenSource = new CancellationTokenSource(); TaskMessage taskMessage = workItem.TaskMessage; OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; TaskScheduledEvent? scheduledEvent = null; Activity?diagnosticActivity = null; try { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received a message that does not have any OrchestrationInstance information."); throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingOrchestrationInstance", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { this.logHelper.TaskActivityDispatcherError( workItem, $"The activity worker received an event of type '{taskMessage.Event.EventType}' but only '{EventType.TaskScheduled}' is supported."); throw TraceHelper.TraceException( TraceEventType.Critical, "TaskActivityDispatcher-UnsupportedEventType", new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } scheduledEvent = (TaskScheduledEvent)taskMessage.Event; if (scheduledEvent.Name == null) { string message = $"The activity worker received a {nameof(EventType.TaskScheduled)} event that does not specify an activity name."; this.logHelper.TaskActivityDispatcherError(workItem, message); throw TraceHelper.TraceException( TraceEventType.Error, "TaskActivityDispatcher-MissingActivityName", new InvalidOperationException(message)); } this.logHelper.TaskActivityStarting(orchestrationInstance, scheduledEvent); TaskActivity?taskActivity = this.objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (workItem.LockedUntilUtc < DateTime.MaxValue) { // start a task to run RenewUntil renewTask = Task.Factory.StartNew( () => this.RenewUntil(workItem, renewCancellationTokenSource.Token), renewCancellationTokenSource.Token); } var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(taskMessage.OrchestrationInstance); dispatchContext.SetProperty(taskActivity); dispatchContext.SetProperty(scheduledEvent); // correlation CorrelationTraceClient.Propagate(() => { workItem.TraceContextBase?.SetActivityToCurrent(); diagnosticActivity = workItem.TraceContextBase?.CurrentActivity; }); ActivityExecutionResult?result; try { await this.dispatchPipeline.RunAsync(dispatchContext, async _ => { if (taskActivity == null) { // This likely indicates a deployment error of some kind. Because these unhandled exceptions are // automatically retried, resolving this may require redeploying the app code so that the activity exists again. // CONSIDER: Should this be changed into a permanent error that fails the orchestration? Perhaps // the app owner doesn't care to preserve existing instances when doing code deployments? throw new TypeMissingException($"TaskActivity {scheduledEvent.Name} version {scheduledEvent.Version} was not found"); } var context = new TaskContext(taskMessage.OrchestrationInstance); context.ErrorPropagationMode = this.errorPropagationMode; HistoryEvent?responseEvent; try { string?output = await taskActivity.RunAsync(context, scheduledEvent.Input); responseEvent = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (Exception e) when(e is not TaskFailureException && !Utils.IsFatal(e) && !Utils.IsExecutionAborting(e)) { // These are unexpected exceptions that occur in the task activity abstraction. Normal exceptions from // activities are expected to be translated into TaskFailureException and handled outside the middleware // context (see further below). TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessException", taskMessage.OrchestrationInstance, e); string?details = this.IncludeDetails ? $"Unhandled exception while executing task: {e}" : null; responseEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, new FailureDetails(e)); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, (TaskFailedEvent)responseEvent, e); } var result = new ActivityExecutionResult { ResponseEvent = responseEvent }; dispatchContext.SetProperty(result); }); result = dispatchContext.GetProperty <ActivityExecutionResult>(); } catch (TaskFailureException e) { // These are normal task activity failures. They can come from Activity implementations or from middleware. TraceHelper.TraceExceptionInstance(TraceEventType.Error, "TaskActivityDispatcher-ProcessTaskFailure", taskMessage.OrchestrationInstance, e); string?details = this.IncludeDetails ? e.Details : null; var failureEvent = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details, e.FailureDetails); this.logHelper.TaskActivityFailure(orchestrationInstance, scheduledEvent.Name, failureEvent, e); CorrelationTraceClient.Propagate(() => CorrelationTraceClient.TrackException(e)); result = new ActivityExecutionResult { ResponseEvent = failureEvent }; } catch (Exception middlewareException) when(!Utils.IsFatal(middlewareException)) { // These are considered retriable this.logHelper.TaskActivityDispatcherError(workItem, $"Unhandled exception in activity middleware pipeline: {middlewareException}"); throw; } HistoryEvent?eventToRespond = result?.ResponseEvent; if (eventToRespond is TaskCompletedEvent completedEvent) { this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent); } else if (eventToRespond is null) { // Default response if middleware prevents a response from being generated eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, null); } var responseTaskMessage = new TaskMessage { Event = eventToRespond, OrchestrationInstance = orchestrationInstance }; await this.orchestrationService.CompleteTaskActivityWorkItemAsync(workItem, responseTaskMessage); } catch (SessionAbortedException e) { // The activity aborted its execution this.logHelper.TaskActivityAborted(orchestrationInstance, scheduledEvent, e.Message); TraceHelper.TraceInstance(TraceEventType.Warning, "TaskActivityDispatcher-ExecutionAborted", orchestrationInstance, "{0}: {1}", scheduledEvent?.Name, e.Message); await this.orchestrationService.AbandonTaskActivityWorkItemAsync(workItem); } finally { diagnosticActivity?.Stop(); // Ensure the activity is stopped here to prevent it from leaking out. if (renewTask != null) { renewCancellationTokenSource.Cancel(); try { // wait the renewTask finish await renewTask; } catch (OperationCanceledException) { // ignore } } } }
public static HistoryEvent GetHistoryEvent(this DbDataReader reader, bool isOrchestrationHistory = false) { string eventTypeString = (string)reader["EventType"]; if (!Enum.TryParse(eventTypeString, out EventType eventType)) { throw new InvalidOperationException($"Unknown event type '{eventTypeString}'."); } int eventId = GetTaskId(reader); HistoryEvent historyEvent; switch (eventType) { case EventType.ContinueAsNew: historyEvent = new ContinueAsNewEvent(eventId, GetPayloadText(reader)); break; case EventType.EventRaised: historyEvent = new EventRaisedEvent(eventId, GetPayloadText(reader)) { Name = GetName(reader), }; break; case EventType.EventSent: historyEvent = new EventSentEvent(eventId) { Input = GetPayloadText(reader), Name = GetName(reader), InstanceId = GetInstanceId(reader), }; break; case EventType.ExecutionCompleted: historyEvent = new ExecutionCompletedEvent( eventId, result: GetPayloadText(reader), orchestrationStatus: OrchestrationStatus.Completed); break; case EventType.ExecutionFailed: historyEvent = new ExecutionCompletedEvent( eventId, result: GetPayloadText(reader), orchestrationStatus: OrchestrationStatus.Failed); break; case EventType.ExecutionStarted: historyEvent = new ExecutionStartedEvent(eventId, GetPayloadText(reader)) { Name = GetName(reader), OrchestrationInstance = new OrchestrationInstance { InstanceId = GetInstanceId(reader), ExecutionId = GetExecutionId(reader), }, Tags = null, // TODO Version = GetVersion(reader), }; string?parentInstanceId = GetParentInstanceId(reader); if (parentInstanceId != null) { ((ExecutionStartedEvent)historyEvent).ParentInstance = new ParentInstance { OrchestrationInstance = new OrchestrationInstance { InstanceId = parentInstanceId }, TaskScheduleId = GetTaskId(reader) }; } break; case EventType.ExecutionTerminated: historyEvent = new ExecutionTerminatedEvent(eventId, GetPayloadText(reader)); break; case EventType.GenericEvent: historyEvent = new GenericEvent(eventId, GetPayloadText(reader)); break; case EventType.OrchestratorCompleted: historyEvent = new OrchestratorCompletedEvent(eventId); break; case EventType.OrchestratorStarted: historyEvent = new OrchestratorStartedEvent(eventId); break; case EventType.SubOrchestrationInstanceCompleted: historyEvent = new SubOrchestrationInstanceCompletedEvent(eventId, GetTaskId(reader), GetPayloadText(reader)); break; case EventType.SubOrchestrationInstanceCreated: historyEvent = new SubOrchestrationInstanceCreatedEvent(eventId) { Input = GetPayloadText(reader), InstanceId = null, // TODO Name = GetName(reader), Version = null, }; break; case EventType.SubOrchestrationInstanceFailed: historyEvent = new SubOrchestrationInstanceFailedEvent( eventId, taskScheduledId: GetTaskId(reader), reason: GetReason(reader), details: GetPayloadText(reader)); break; case EventType.TaskCompleted: historyEvent = new TaskCompletedEvent( eventId, taskScheduledId: GetTaskId(reader), result: GetPayloadText(reader)); break; case EventType.TaskFailed: historyEvent = new TaskFailedEvent( eventId, taskScheduledId: GetTaskId(reader), reason: GetReason(reader), details: GetPayloadText(reader)); break; case EventType.TaskScheduled: historyEvent = new TaskScheduledEvent(eventId) { Input = GetPayloadText(reader), Name = GetName(reader), Version = GetVersion(reader), }; break; case EventType.TimerCreated: historyEvent = new TimerCreatedEvent(eventId) { FireAt = GetVisibleTime(reader), }; break; case EventType.TimerFired: historyEvent = new TimerFiredEvent(eventId) { FireAt = GetVisibleTime(reader), TimerId = GetTaskId(reader), }; break; default: throw new InvalidOperationException($"Don't know how to interpret '{eventType}'."); } historyEvent.Timestamp = GetTimestamp(reader); historyEvent.IsPlayed = isOrchestrationHistory && (bool)reader["IsPlayed"]; return(historyEvent); }
protected override async Task OnProcessWorkItem(BrokeredMessage message) { Utils.CheckAndLogDeliveryCount(message, taskHubDescription.MaxTaskActivityDeliveryCount, this.orchestratorQueueName); Task renewTask = null; var renewCancellationTokenSource = new CancellationTokenSource(); try { TaskMessage taskMessage = await Utils.GetObjectFromBrokeredMessageAsync <TaskMessage>(message); OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; if (orchestrationInstance == null || string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) { throw TraceHelper.TraceException(TraceEventType.Error, new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { throw TraceHelper.TraceException(TraceEventType.Critical, new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } // call and get return message var scheduledEvent = (TaskScheduledEvent)taskMessage.Event; TaskActivity taskActivity = objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (taskActivity == null) { throw new TypeMissingException("TaskActivity " + scheduledEvent.Name + " version " + scheduledEvent.Version + " was not found"); } renewTask = Task.Factory.StartNew(() => RenewUntil(message, renewCancellationTokenSource.Token)); // TODO : pass workflow instance data var context = new TaskContext(taskMessage.OrchestrationInstance); HistoryEvent eventToRespond = null; try { string output = await taskActivity.RunAsync(context, scheduledEvent.Input); eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (TaskFailureException e) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, taskMessage.OrchestrationInstance, e); string details = IncludeDetails ? e.Details : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); } catch (Exception e) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, taskMessage.OrchestrationInstance, e); string details = IncludeDetails ? string.Format("Unhandled exception while executing task: {0}\n\t{1}", e, e.StackTrace) : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); } var responseTaskMessage = new TaskMessage(); responseTaskMessage.Event = eventToRespond; responseTaskMessage.OrchestrationInstance = orchestrationInstance; BrokeredMessage responseMessage = Utils.GetBrokeredMessageFromObject(responseTaskMessage, settings.MessageCompressionSettings, orchestrationInstance, "Response for " + message.MessageId); responseMessage.SessionId = orchestrationInstance.InstanceId; using (var ts = new TransactionScope()) { workerQueueClient.Complete(message.LockToken); deciderSender.Send(responseMessage); ts.Complete(); } } finally { if (renewTask != null) { renewCancellationTokenSource.Cancel(); renewTask.Wait(); } } }