/// <summary> /// Raises an event in the specified orchestration instance, which eventually causes the OnEvent() method in the /// orchestration to fire. /// </summary> /// <param name="orchestrationInstance">Instance in which to raise the event</param> /// <param name="eventName">Name of the event</param> /// <param name="eventData">Data for the event</param> public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { if (orchestrationInstance == null || string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) { throw new ArgumentException("orchestrationInstance"); } string serializedInput = defaultConverter.Serialize(eventData); var taskMessage = new TaskMessage { OrchestrationInstance = orchestrationInstance, Event = new EventRaisedEvent(-1, serializedInput) {Name = eventName} }; BrokeredMessage brokeredMessage = Utils.GetBrokeredMessageFromObject(taskMessage, settings.MessageCompressionSettings); brokeredMessage.SessionId = orchestrationInstance.InstanceId; MessageSender sender = await messagingFactory.CreateMessageSenderAsync(orchestratorEntityName).ConfigureAwait(false); await sender.SendAsync(brokeredMessage).ConfigureAwait(false); await sender.CloseAsync().ConfigureAwait(false); }
/// <summary> /// Forcefully terminate the specified orchestration instance with a reason /// </summary> /// <param name="orchestrationInstance">Instance to terminate</param> /// <param name="reason">Reason for terminating the instance</param> public async Task TerminateInstanceAsync(OrchestrationInstance orchestrationInstance, string reason) { if (orchestrationInstance == null || string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) { throw new ArgumentException("orchestrationInstance"); } string instanceId = orchestrationInstance.InstanceId; var taskMessage = new TaskMessage { OrchestrationInstance = orchestrationInstance, Event = new ExecutionTerminatedEvent(-1, reason) }; BrokeredMessage brokeredMessage = Utils.GetBrokeredMessageFromObject(taskMessage, settings.MessageCompressionSettings); brokeredMessage.SessionId = instanceId; MessageSender sender = await messagingFactory.CreateMessageSenderAsync(orchestratorEntityName).ConfigureAwait(false); await sender.SendAsync(brokeredMessage).ConfigureAwait(false); await sender.CloseAsync().ConfigureAwait(false); }
/// <summary> /// Create a new orchestration of the specified name and version /// </summary> /// <param name="name">Name of the orchestration as specified by the ObjectCreator</param> /// <param name="version">Name of the orchestration as specified by the ObjectCreator</param> /// <param name="instanceId">Instance id for the orchestration to be created, must be unique across the Task Hub</param> /// <param name="input">Input parameter to the specified TaskOrchestration</param> /// <param name="tags">Dictionary of key/value tags associated with this instance</param> /// <returns>OrchestrationInstance that represents the orchestration that was created</returns> public async Task<OrchestrationInstance> CreateOrchestrationInstanceAsync(string name, string version, string instanceId, object input, IDictionary<string, string> tags) { if (string.IsNullOrWhiteSpace(instanceId)) { instanceId = Guid.NewGuid().ToString("N"); } var orchestrationInstance = new OrchestrationInstance { InstanceId = instanceId, ExecutionId = Guid.NewGuid().ToString("N"), }; string serializedInput = defaultConverter.Serialize(input); string serializedtags = tags != null ? defaultConverter.Serialize(tags) : null; var startedEvent = new ExecutionStartedEvent(-1, serializedInput) { Tags = serializedtags, Name = name, Version = version, OrchestrationInstance = orchestrationInstance }; var taskMessage = new TaskMessage { OrchestrationInstance = orchestrationInstance, Event = startedEvent }; BrokeredMessage brokeredMessage = Utils.GetBrokeredMessageFromObject(taskMessage, settings.MessageCompressionSettings); brokeredMessage.SessionId = instanceId; MessageSender sender = await messagingFactory.CreateMessageSenderAsync(orchestratorEntityName).ConfigureAwait(false); await sender.SendAsync(brokeredMessage).ConfigureAwait(false); await sender.CloseAsync().ConfigureAwait(false); return orchestrationInstance; }
protected override async Task OnProcessWorkItem(SessionWorkItem sessionWorkItem) { MessageSession session = sessionWorkItem.Session; IEnumerable <BrokeredMessage> newMessages = sessionWorkItem.Messages; var historyEntities = new List <OrchestrationHistoryEventEntity>(); var stateEntities = new List <OrchestrationStateEntity>(); foreach (BrokeredMessage message in newMessages) { Utils.CheckAndLogDeliveryCount(message, taskHubDescription.MaxTrackingDeliveryCount, this.trackingEntityName); if (message.ContentType.Equals(FrameworkConstants.TaskMessageContentType, StringComparison.OrdinalIgnoreCase)) { object historyEventIndexObj; if (!message.Properties.TryGetValue(FrameworkConstants.HistoryEventIndexPropertyName, out historyEventIndexObj) || historyEventIndexObj == null) { throw new InvalidOperationException( "Could not find a valid history event index property on tracking message " + message.MessageId); } var historyEventIndex = (int)historyEventIndexObj; TaskMessage taskMessage = await Utils.GetObjectFromBrokeredMessageAsync <TaskMessage>(message); historyEntities.Add(new OrchestrationHistoryEventEntity( taskMessage.OrchestrationInstance.InstanceId, taskMessage.OrchestrationInstance.ExecutionId, historyEventIndex, DateTime.UtcNow, taskMessage.Event)); } else if (message.ContentType.Equals(FrameworkConstants.StateMessageContentType, StringComparison.OrdinalIgnoreCase)) { StateMessage stateMessage = await Utils.GetObjectFromBrokeredMessageAsync <StateMessage>(message); stateEntities.Add(new OrchestrationStateEntity(stateMessage.State)); } else { throw new InvalidOperationException("Invalid tracking message content type: " + message.ContentType); } } TraceEntities(TraceEventType.Verbose, "Writing tracking history event", historyEntities, GetNormalizedHistoryEventEntityTrace); TraceEntities(TraceEventType.Verbose, "Writing tracking state event", stateEntities, GetNormalizedStateEntityTrace); try { await Utils.ExecuteWithRetries(() => tableClient.WriteEntitesAsync(historyEntities), session.SessionId, string.Format("WriteHistoryEntities:{0}", session.SessionId), MaxRetriesTableStore, IntervalBetweenRetriesSecs); } catch (Exception) { TraceEntities(TraceEventType.Critical, "Failed to write history entity", historyEntities, GetNormalizedHistoryEventEntityTrace); throw; } try { foreach (OrchestrationStateEntity stateEntity in stateEntities) { await Utils.ExecuteWithRetries( () => tableClient.WriteEntitesAsync(new List <OrchestrationStateEntity> { stateEntity }), session.SessionId, string.Format("WriteStateEntities:{0}", session.SessionId), MaxRetriesTableStore, IntervalBetweenRetriesSecs); } } catch (Exception) { TraceEntities(TraceEventType.Critical, "Failed to write state entity", stateEntities, GetNormalizedStateEntityTrace); throw; } IEnumerable <Guid> lockTokens = newMessages.Select(m => m.LockToken); Utils.SyncExecuteWithRetries <object>(() => { session.CompleteBatch(lockTokens); return(null); }, session.SessionId, "Complete New Tracking Messages", MaxRetriesServiceBus, IntervalBetweenRetriesSecs); }
static TaskMessage ProcessWorkflowCompletedTaskDecision( OrchestrationCompleteOrchestratorAction completeOrchestratorAction, OrchestrationRuntimeState runtimeState, bool includeDetails, out bool continuedAsNew) { ExecutionCompletedEvent executionCompletedEvent; continuedAsNew = (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew); if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) { executionCompletedEvent = new ContinueAsNewEvent(completeOrchestratorAction.Id, completeOrchestratorAction.Result); } else { executionCompletedEvent = new ExecutionCompletedEvent(completeOrchestratorAction.Id, completeOrchestratorAction.Result, completeOrchestratorAction.OrchestrationStatus); } runtimeState.AddEvent(executionCompletedEvent); TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "Instance Id '{0}' completed in state {1} with result: {2}", runtimeState.OrchestrationInstance, runtimeState.OrchestrationStatus, completeOrchestratorAction.Result); string history = JsonConvert.SerializeObject(runtimeState.Events, Formatting.Indented, new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects }); TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, () => Utils.EscapeJson(history)); // Check to see if we need to start a new execution if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) { var taskMessage = new TaskMessage(); var startedEvent = new ExecutionStartedEvent(-1, completeOrchestratorAction.Result); startedEvent.OrchestrationInstance = new OrchestrationInstance { InstanceId = runtimeState.OrchestrationInstance.InstanceId, ExecutionId = Guid.NewGuid().ToString("N") }; startedEvent.Tags = runtimeState.Tags; startedEvent.ParentInstance = runtimeState.ParentInstance; startedEvent.Name = runtimeState.Name; startedEvent.Version = completeOrchestratorAction.NewVersion ?? runtimeState.Version; taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance; taskMessage.Event = startedEvent; return(taskMessage); } // If this is a Sub Orchestration than notify the parent by sending a complete message if (runtimeState.ParentInstance != null) { var taskMessage = new TaskMessage(); if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Completed) { var subOrchestrationCompletedEvent = new SubOrchestrationInstanceCompletedEvent(-1, runtimeState.ParentInstance.TaskScheduleId, completeOrchestratorAction.Result); taskMessage.Event = subOrchestrationCompletedEvent; } else if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Failed || completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Terminated) { var subOrchestrationFailedEvent = new SubOrchestrationInstanceFailedEvent(-1, runtimeState.ParentInstance.TaskScheduleId, completeOrchestratorAction.Result, includeDetails ? completeOrchestratorAction.Details : null); taskMessage.Event = subOrchestrationFailedEvent; } if (taskMessage.Event != null) { taskMessage.OrchestrationInstance = runtimeState.ParentInstance.OrchestrationInstance; return(taskMessage); } } return(null); }
protected override async Task OnProcessWorkItem(SessionWorkItem sessionWorkItem) { var messagesToSend = new List <MessageContainer>(); var timerMessages = new List <MessageContainer>(); var subOrchestrationMessages = new List <MessageContainer>(); bool isCompleted = false; bool continuedAsNew = false; BrokeredMessage continuedAsNewMessage = null; ExecutionStartedEvent continueAsNewExecutionStarted = null; MessageSession session = sessionWorkItem.Session; IEnumerable <BrokeredMessage> newMessages = sessionWorkItem.Messages ?? new List <BrokeredMessage>(); long rawSessionStateSize; long newSessionStateSize; bool isEmptySession = false; OrchestrationRuntimeState runtimeState; using (Stream rawSessionStream = await session.GetStateAsync()) using (Stream sessionStream = await Utils.GetDecompressedStreamAsync(rawSessionStream)) { isEmptySession = sessionStream == null; rawSessionStateSize = isEmptySession ? 0 : rawSessionStream.Length; newSessionStateSize = isEmptySession ? 0 : sessionStream.Length; runtimeState = GetOrCreateInstanceState(sessionStream, session.SessionId); } TraceHelper.TraceSession(TraceEventType.Information, session.SessionId, "Size of session state is {0}, compressed {1}", newSessionStateSize, rawSessionStateSize); runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); if (!(await ReconcileMessagesWithStateAsync(session.SessionId, runtimeState, newMessages))) { // TODO : mark an orchestration as faulted if there is data corruption TraceHelper.TraceSession(TraceEventType.Error, session.SessionId, "Received result for a deleted orchestration"); isCompleted = true; } else { TraceHelper.TraceInstance( TraceEventType.Verbose, runtimeState.OrchestrationInstance, "Executing user orchestration: {0}", JsonConvert.SerializeObject(runtimeState.GetOrchestrationRuntimeStateDump(), new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Auto, Formatting = Formatting.Indented })); IEnumerable <OrchestratorAction> decisions = ExecuteOrchestration(runtimeState); TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "Executed user orchestration. Received {0} orchestrator actions: {1}", decisions.Count(), string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); foreach (OrchestratorAction decision in decisions) { TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "Processing orchestrator action of type {0}, id {1}", decision.OrchestratorActionType, decision.Id.ToString()); switch (decision.OrchestratorActionType) { case OrchestratorActionType.ScheduleOrchestrator: TaskMessage taskMessage = ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState, IncludeParameters); BrokeredMessage brokeredMessage = Utils.GetBrokeredMessageFromObject( taskMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance, "ScheduleTask"); brokeredMessage.SessionId = session.SessionId; messagesToSend.Add(new MessageContainer(brokeredMessage, decision)); break; case OrchestratorActionType.CreateTimer: var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; TaskMessage timerMessage = ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState); BrokeredMessage brokeredTimerMessage = Utils.GetBrokeredMessageFromObject( timerMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance, "Timer"); brokeredTimerMessage.ScheduledEnqueueTimeUtc = timerOrchestratorAction.FireAt; brokeredTimerMessage.SessionId = session.SessionId; timerMessages.Add(new MessageContainer(brokeredTimerMessage, decision)); break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; TaskMessage createSubOrchestrationInstanceMessage = ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction, runtimeState, IncludeParameters); BrokeredMessage createSubOrchestrationMessage = Utils.GetBrokeredMessageFromObject( createSubOrchestrationInstanceMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance, "Schedule Suborchestration"); createSubOrchestrationMessage.SessionId = createSubOrchestrationInstanceMessage.OrchestrationInstance.InstanceId; subOrchestrationMessages.Add(new MessageContainer(createSubOrchestrationMessage, decision)); break; case OrchestratorActionType.OrchestrationComplete: TaskMessage workflowInstanceCompletedMessage = ProcessWorkflowCompletedTaskDecision((OrchestrationCompleteOrchestratorAction)decision, runtimeState, IncludeDetails, out continuedAsNew); if (workflowInstanceCompletedMessage != null) { // Send complete message to parent workflow or to itself to start a new execution BrokeredMessage workflowCompletedBrokeredMessage = Utils.GetBrokeredMessageFromObject( workflowInstanceCompletedMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance, "Complete Suborchestration"); workflowCompletedBrokeredMessage.SessionId = workflowInstanceCompletedMessage.OrchestrationInstance.InstanceId; // Store the event so we can rebuild the state if (continuedAsNew) { continuedAsNewMessage = workflowCompletedBrokeredMessage; continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; } else { subOrchestrationMessages.Add(new MessageContainer(workflowCompletedBrokeredMessage, decision)); } } isCompleted = !continuedAsNew; break; default: throw TraceHelper.TraceExceptionInstance(TraceEventType.Error, runtimeState.OrchestrationInstance, new NotSupportedException("decision type not supported")); } // We cannot send more than 100 messages within a transaction, to avoid the situation // we keep on checking the message count and stop processing the new decisions. // We also put in a fake timer to force next orchestration task for remaining messages int totalMessages = messagesToSend.Count + subOrchestrationMessages.Count + timerMessages.Count; // Also add tracking messages as they contribute to total messages within transaction if (isTrackingEnabled) { totalMessages += runtimeState.NewEvents.Count; } if (totalMessages > MaxMessageCount) { TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); if (isCompleted || continuedAsNew) { TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "Orchestration already completed. Skip adding timer for splitting messages."); break; } var dummyTimer = new CreateTimerOrchestratorAction { Id = FrameworkConstants.FakeTimerIdToSplitDecision, FireAt = DateTime.UtcNow }; TaskMessage timerMessage = ProcessCreateTimerDecision(dummyTimer, runtimeState); BrokeredMessage brokeredTimerMessage = Utils.GetBrokeredMessageFromObject( timerMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance, "MaxMessageCount Timer"); brokeredTimerMessage.ScheduledEnqueueTimeUtc = dummyTimer.FireAt; brokeredTimerMessage.SessionId = session.SessionId; timerMessages.Add(new MessageContainer(brokeredTimerMessage, null)); break; } } } // TODO : make async, transactions are a bit tricky using (var ts = new TransactionScope()) { Transaction.Current.TransactionCompleted += (o, e) => { TraceHelper.TraceInstance( e.Transaction.TransactionInformation.Status == TransactionStatus.Committed ? TraceEventType.Information : TraceEventType.Error, runtimeState.OrchestrationInstance, () => "Transaction " + e.Transaction.TransactionInformation.LocalIdentifier + " status: " + e.Transaction.TransactionInformation.Status); }; TraceHelper.TraceInstance( TraceEventType.Information, runtimeState.OrchestrationInstance, () => "Created new transaction - txnid: " + Transaction.Current.TransactionInformation.LocalIdentifier); bool isSessionSizeThresholdExceeded = false; if (!continuedAsNew) { runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); } using (var ms = new MemoryStream()) { if (isCompleted) { TraceHelper.TraceSession(TraceEventType.Information, session.SessionId, "Deleting session state"); // if session was already null and we finished the orchestration then no change is required if (!isEmptySession) { session.SetState(null); } runtimeState.Size = 0; runtimeState.CompressedSize = 0; } else { if (ms.Position != 0) { throw TraceHelper.TraceExceptionInstance(TraceEventType.Error, runtimeState.OrchestrationInstance, new ArgumentException("Instance state stream is partially consumed")); } IList <HistoryEvent> newState = runtimeState.Events; if (continuedAsNew) { TraceHelper.TraceSession(TraceEventType.Information, session.SessionId, "Updating state for continuation"); newState = new List <HistoryEvent>(); newState.Add(new OrchestratorStartedEvent(-1)); newState.Add(continueAsNewExecutionStarted); newState.Add(new OrchestratorCompletedEvent(-1)); } string serializedState = JsonConvert.SerializeObject(newState, new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Auto }); var writer = new StreamWriter(ms); writer.Write(serializedState); writer.Flush(); ms.Position = 0; using (Stream compressedState = settings.TaskOrchestrationDispatcherSettings.CompressOrchestrationState ? Utils.GetCompressedStream(ms) : ms) { runtimeState.Size = ms.Length; runtimeState.CompressedSize = compressedState.Length; if (runtimeState.CompressedSize > SessionStreamTerminationThresholdInBytes) { // basic idea is to simply enqueue a terminate message just like how we do it from taskhubclient // it is possible to have other messages in front of the queue and those will get processed before // the terminate message gets processed. but that is ok since in the worst case scenario we will // simply land in this if-block again and end up queuing up another terminate message. // // the interesting scenario is when the second time we *dont* land in this if-block because e.g. // the new messages that we processed caused a new generation to be created. in that case // it is still ok because the worst case scenario is that we will terminate a newly created generation // which shouldn't have been created at all in the first place isSessionSizeThresholdExceeded = true; TraceHelper.TraceSession(TraceEventType.Critical, session.SessionId, "Size of session state " + runtimeState.CompressedSize + " has exceeded termination threshold of " + SessionStreamTerminationThresholdInBytes); string reason = string.Format( "Session state size of {0} exceeded the termination threshold of {1} bytes", runtimeState.CompressedSize, SessionStreamTerminationThresholdInBytes); BrokeredMessage forcedTerminateMessage = CreateForcedTerminateMessage( runtimeState.OrchestrationInstance.InstanceId, reason); orchestratorQueueClient.Send(forcedTerminateMessage); } else { session.SetState(compressedState); } } writer.Close(); } } if (!isSessionSizeThresholdExceeded) { if (runtimeState.CompressedSize > SessionStreamWarningSizeInBytes) { TraceHelper.TraceSession(TraceEventType.Error, session.SessionId, "Size of session state is nearing session size limit of 256KB"); } if (!continuedAsNew) { if (messagesToSend.Count > 0) { messagesToSend.ForEach(m => workerSender.Send(m.Message)); this.LogSentMessages(session, "Worker outbound", messagesToSend); } if (timerMessages.Count > 0) { timerMessages.ForEach(m => orchestratorQueueClient.Send(m.Message)); this.LogSentMessages(session, "Timer Message", timerMessages); } } if (subOrchestrationMessages.Count > 0) { subOrchestrationMessages.ForEach(m => orchestratorQueueClient.Send(m.Message)); this.LogSentMessages(session, "Sub Orchestration", subOrchestrationMessages); } if (continuedAsNewMessage != null) { orchestratorQueueClient.Send(continuedAsNewMessage); this.LogSentMessages(session, "Continue as new", new List <MessageContainer> () { new MessageContainer(continuedAsNewMessage, null) }); } if (isTrackingEnabled) { List <MessageContainer> trackingMessages = CreateTrackingMessages(runtimeState); TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "Created {0} tracking messages", trackingMessages.Count); if (trackingMessages.Count > 0) { trackingMessages.ForEach(m => trackingSender.Send(m.Message)); this.LogSentMessages(session, "Tracking messages", trackingMessages); } } } TraceHelper.TraceInstance( TraceEventType.Information, runtimeState.OrchestrationInstance, () => { string allIds = string.Join(" ", newMessages.Select(m => $"[SEQ: {m.SequenceNumber} LT: {m.LockToken}]")); return("Completing msgs seq and locktokens: " + allIds); }); IEnumerable <Guid> lockTokens = newMessages.Select(m => m.LockToken); session.CompleteBatch(lockTokens); ts.Complete(); } }
protected override async Task OnProcessWorkItem(BrokeredMessage message) { Utils.CheckAndLogDeliveryCount(message, taskHubDescription.MaxTaskActivityDeliveryCount, this.orchestratorQueueName); Task renewTask = null; var renewCancellationTokenSource = new CancellationTokenSource(); try { TaskMessage taskMessage = await Utils.GetObjectFromBrokeredMessageAsync <TaskMessage>(message); OrchestrationInstance orchestrationInstance = taskMessage.OrchestrationInstance; if (orchestrationInstance == null || string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) { throw TraceHelper.TraceException(TraceEventType.Error, new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (taskMessage.Event.EventType != EventType.TaskScheduled) { throw TraceHelper.TraceException(TraceEventType.Critical, new NotSupportedException("Activity worker does not support event of type: " + taskMessage.Event.EventType)); } // call and get return message var scheduledEvent = (TaskScheduledEvent)taskMessage.Event; TaskActivity taskActivity = objectManager.GetObject(scheduledEvent.Name, scheduledEvent.Version); if (taskActivity == null) { throw new TypeMissingException("TaskActivity " + scheduledEvent.Name + " version " + scheduledEvent.Version + " was not found"); } renewTask = Task.Factory.StartNew(() => RenewUntil(message, renewCancellationTokenSource.Token)); // TODO : pass workflow instance data var context = new TaskContext(taskMessage.OrchestrationInstance); HistoryEvent eventToRespond = null; try { string output = await taskActivity.RunAsync(context, scheduledEvent.Input); eventToRespond = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } catch (TaskFailureException e) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, taskMessage.OrchestrationInstance, e); string details = IncludeDetails ? e.Details : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); } catch (Exception e) { TraceHelper.TraceExceptionInstance(TraceEventType.Error, taskMessage.OrchestrationInstance, e); string details = IncludeDetails ? string.Format("Unhandled exception while executing task: {0}\n\t{1}", e, e.StackTrace) : null; eventToRespond = new TaskFailedEvent(-1, scheduledEvent.EventId, e.Message, details); } var responseTaskMessage = new TaskMessage(); responseTaskMessage.Event = eventToRespond; responseTaskMessage.OrchestrationInstance = orchestrationInstance; BrokeredMessage responseMessage = Utils.GetBrokeredMessageFromObject(responseTaskMessage, settings.MessageCompressionSettings, orchestrationInstance, "Response for " + message.MessageId); responseMessage.SessionId = orchestrationInstance.InstanceId; using (var ts = new TransactionScope()) { workerQueueClient.Complete(message.LockToken); deciderSender.Send(responseMessage); ts.Complete(); } } finally { if (renewTask != null) { renewCancellationTokenSource.Cancel(); renewTask.Wait(); } } }