public override async Task <T> CreateTimer <T>(DateTime fireAt, T state, CancellationToken cancelToken) { int id = idCounter++; var createTimerOrchestratorAction = new CreateTimerOrchestratorAction { Id = id, FireAt = fireAt, }; orchestratorActionsMap.Add(id, createTimerOrchestratorAction); var tcs = new TaskCompletionSource <string>(); openTasks.Add(id, new OpenTaskInfo { Name = null, Version = null, Result = tcs }); if (cancelToken != CancellationToken.None) { cancelToken.Register(s => { if (tcs.TrySetCanceled()) { openTasks.Remove(id); } }, tcs); } await tcs.Task; return(state); }
TaskMessage ProcessCreateTimerDecision( CreateTimerOrchestratorAction createTimerOrchestratorAction, OrchestrationRuntimeState runtimeState, bool isInternal) { var taskMessage = new TaskMessage(); var timerCreatedEvent = new TimerCreatedEvent(createTimerOrchestratorAction.Id) { FireAt = createTimerOrchestratorAction.FireAt }; runtimeState.AddEvent(timerCreatedEvent); taskMessage.Event = new TimerFiredEvent(-1) { TimerId = createTimerOrchestratorAction.Id, FireAt = createTimerOrchestratorAction.FireAt }; this.logHelper.CreatingTimer( runtimeState.OrchestrationInstance, timerCreatedEvent, isInternal); taskMessage.OrchestrationInstance = runtimeState.OrchestrationInstance; return(taskMessage); }
static TaskMessage ProcessCreateTimerDecision( CreateTimerOrchestratorAction createTimerOrchestratorAction, OrchestrationRuntimeState runtimeState) { var taskMessage = new TaskMessage(); var timerCreatedEvent = new TimerCreatedEvent(createTimerOrchestratorAction.Id); timerCreatedEvent.FireAt = createTimerOrchestratorAction.FireAt; runtimeState.AddEvent(timerCreatedEvent); var timerFiredEvent = new TimerFiredEvent(-1); timerFiredEvent.TimerId = createTimerOrchestratorAction.Id; taskMessage.Event = timerFiredEvent; taskMessage.OrchestrationInstance = runtimeState.OrchestrationInstance; return(taskMessage); }
/// <summary> /// Method to process a new work item /// </summary> /// <param name="workItem">The work item to process</param> protected async Task <bool> OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem) { var messagesToSend = new List <TaskMessage>(); var timerMessages = new List <TaskMessage>(); var orchestratorMessages = new List <TaskMessage>(); var isCompleted = false; var continuedAsNew = false; var isInterrupted = false; // correlation CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext); ExecutionStartedEvent continueAsNewExecutionStarted = null; TaskMessage continuedAsNewMessage = null; IList <HistoryEvent> carryOverEvents = null; string carryOverStatus = null; OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState; runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); OrchestrationRuntimeState originalOrchestrationRuntimeState = runtimeState; OrchestrationState instanceState = null; if (!ReconcileMessagesWithState(workItem)) { // TODO : mark an orchestration as faulted if there is data corruption TraceHelper.TraceSession( TraceEventType.Error, "TaskOrchestrationDispatcher-DeletedOrchestration", runtimeState.OrchestrationInstance?.InstanceId, "Received result for a deleted orchestration"); isCompleted = true; } else { do { continuedAsNew = false; continuedAsNewMessage = null; TraceHelper.TraceInstance( TraceEventType.Verbose, "TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin", runtimeState.OrchestrationInstance, "Executing user orchestration: {0}", DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true)); if (workItem.Cursor == null) { workItem.Cursor = await ExecuteOrchestrationAsync(runtimeState); } else { await ResumeOrchestrationAsync(workItem.Cursor); } IReadOnlyList <OrchestratorAction> decisions = workItem.Cursor.LatestDecisions.ToList(); TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-ExecuteUserOrchestration-End", runtimeState.OrchestrationInstance, "Executed user orchestration. Received {0} orchestrator actions: {1}", decisions.Count, string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); foreach (OrchestratorAction decision in decisions) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-ProcessOrchestratorAction", runtimeState.OrchestrationInstance, "Processing orchestrator action of type {0}", decision.OrchestratorActionType); switch (decision.OrchestratorActionType) { case OrchestratorActionType.ScheduleOrchestrator: messagesToSend.Add( ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState, IncludeParameters)); break; case OrchestratorActionType.CreateTimer: var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; timerMessages.Add(ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState)); break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; orchestratorMessages.Add( ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction, runtimeState, IncludeParameters)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; orchestratorMessages.Add( ProcessSendEventDecision(sendEventAction, runtimeState)); break; case OrchestratorActionType.OrchestrationComplete: OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision; TaskMessage workflowInstanceCompletedMessage = ProcessWorkflowCompletedTaskDecision(completeDecision, runtimeState, IncludeDetails, out continuedAsNew); if (workflowInstanceCompletedMessage != null) { // Send complete message to parent workflow or to itself to start a new execution // Store the event so we can rebuild the state carryOverEvents = null; if (continuedAsNew) { continuedAsNewMessage = workflowInstanceCompletedMessage; continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; if (completeDecision.CarryoverEvents.Any()) { carryOverEvents = completeDecision.CarryoverEvents.ToList(); completeDecision.CarryoverEvents.Clear(); } } else { orchestratorMessages.Add(workflowInstanceCompletedMessage); } } isCompleted = !continuedAsNew; break; default: throw TraceHelper.TraceExceptionInstance( TraceEventType.Error, "TaskOrchestrationDispatcher-UnsupportedDecisionType", runtimeState.OrchestrationInstance, new NotSupportedException("decision type not supported")); } // Underlying orchestration service provider may have a limit of messages per call, to avoid the situation // we keep on asking the provider if message count is ok and stop processing new decisions if not. // // We also put in a fake timer to force next orchestration task for remaining messages int totalMessages = messagesToSend.Count + orchestratorMessages.Count + timerMessages.Count; if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState)) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-MaxMessageCountReached", runtimeState.OrchestrationInstance, "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); if (isCompleted || continuedAsNew) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted", runtimeState.OrchestrationInstance, "Orchestration already completed. Skip adding timer for splitting messages."); break; } var dummyTimer = new CreateTimerOrchestratorAction { Id = FrameworkConstants.FakeTimerIdToSplitDecision, FireAt = DateTime.UtcNow }; timerMessages.Add(ProcessCreateTimerDecision(dummyTimer, runtimeState)); isInterrupted = true; break; } } // finish up processing of the work item if (!continuedAsNew && runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted) { runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); } if (isCompleted) { TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state"); if (runtimeState.ExecutionStartedEvent != null) { instanceState = Utils.BuildOrchestrationState(runtimeState); } } else { if (continuedAsNew) { TraceHelper.TraceSession( TraceEventType.Information, "TaskOrchestrationDispatcher-UpdatingStateForContinuation", workItem.InstanceId, "Updating state for continuation"); runtimeState = new OrchestrationRuntimeState(); runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); runtimeState.AddEvent(continueAsNewExecutionStarted); runtimeState.Status = workItem.OrchestrationRuntimeState.Status ?? carryOverStatus; carryOverStatus = workItem.OrchestrationRuntimeState.Status; if (carryOverEvents != null) { foreach (var historyEvent in carryOverEvents) { runtimeState.AddEvent(historyEvent); } } runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); workItem.OrchestrationRuntimeState = runtimeState; TaskOrchestration newOrchestration = this.objectManager.GetObject( runtimeState.Name, continueAsNewExecutionStarted.Version); workItem.Cursor = new OrchestrationExecutionCursor( runtimeState, newOrchestration, new TaskOrchestrationExecutor( runtimeState, newOrchestration, orchestrationService.EventBehaviourForContinueAsNew), latestDecisions: null); if (workItem.LockedUntilUtc < DateTime.UtcNow.AddMinutes(1)) { await orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem); } } instanceState = Utils.BuildOrchestrationState(runtimeState); } } while (continuedAsNew); } workItem.OrchestrationRuntimeState = originalOrchestrationRuntimeState; runtimeState.Status = runtimeState.Status ?? carryOverStatus; await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( workItem, runtimeState, continuedAsNew?null : messagesToSend, orchestratorMessages, continuedAsNew?null : timerMessages, continuedAsNewMessage, instanceState); workItem.OrchestrationRuntimeState = runtimeState; return(isCompleted || continuedAsNew || isInterrupted); }
protected override async Task OnProcessWorkItem(SessionWorkItem sessionWorkItem) { var messagesToSend = new List <BrokeredMessage>(); var timerMessages = new List <BrokeredMessage>(); var subOrchestrationMessages = new List <BrokeredMessage>(); bool isCompleted = false; bool continuedAsNew = false; BrokeredMessage continuedAsNewMessage = null; ExecutionStartedEvent continueAsNewExecutionStarted = null; MessageSession session = sessionWorkItem.Session; IEnumerable <BrokeredMessage> newMessages = sessionWorkItem.Messages ?? new List <BrokeredMessage>(); long rawSessionStateSize; long newSessionStateSize; bool isEmptySession = false; OrchestrationRuntimeState runtimeState; using (Stream rawSessionStream = await session.GetStateAsync()) using (Stream sessionStream = await Utils.GetDecompressedStreamAsync(rawSessionStream)) { isEmptySession = sessionStream == null; rawSessionStateSize = isEmptySession ? 0 : rawSessionStream.Length; newSessionStateSize = isEmptySession ? 0 : sessionStream.Length; runtimeState = GetOrCreateInstanceState(sessionStream, session.SessionId); } TraceHelper.TraceSession(TraceEventType.Information, session.SessionId, "Size of session state is {0}, compressed {1}", newSessionStateSize, rawSessionStateSize); runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); if (!(await ReconcileMessagesWithStateAsync(session.SessionId, runtimeState, newMessages))) { // TODO : mark an orchestration as faulted if there is data corruption TraceHelper.TraceSession(TraceEventType.Error, session.SessionId, "Received result for a deleted orchestration"); isCompleted = true; } else { TraceHelper.TraceInstance( TraceEventType.Verbose, runtimeState.OrchestrationInstance, "Executing user orchestration: {0}", JsonConvert.SerializeObject(runtimeState.GetOrchestrationRuntimeStateDump(), new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Auto, Formatting = Formatting.Indented })); IEnumerable <OrchestratorAction> decisions = ExecuteOrchestration(runtimeState); TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "Executed user orchestration. Received {0} orchestrator actions: {1}", decisions.Count(), string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); foreach (OrchestratorAction decision in decisions) { TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "Processing orchestrator action of type {0}", decision.OrchestratorActionType); switch (decision.OrchestratorActionType) { case OrchestratorActionType.ScheduleOrchestrator: TaskMessage taskMessage = ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState, IncludeParameters); BrokeredMessage brokeredMessage = Utils.GetBrokeredMessageFromObject( taskMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance, "ScheduleTask"); brokeredMessage.SessionId = session.SessionId; messagesToSend.Add(brokeredMessage); break; case OrchestratorActionType.CreateTimer: var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; TaskMessage timerMessage = ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState); BrokeredMessage brokeredTimerMessage = Utils.GetBrokeredMessageFromObject( timerMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance, "Timer"); brokeredTimerMessage.ScheduledEnqueueTimeUtc = timerOrchestratorAction.FireAt; brokeredTimerMessage.SessionId = session.SessionId; timerMessages.Add(brokeredTimerMessage); break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; TaskMessage createSubOrchestrationInstanceMessage = ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction, runtimeState, IncludeParameters); BrokeredMessage createSubOrchestrationMessage = Utils.GetBrokeredMessageFromObject( createSubOrchestrationInstanceMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance, "Schedule Suborchestration"); createSubOrchestrationMessage.SessionId = createSubOrchestrationInstanceMessage.OrchestrationInstance.InstanceId; subOrchestrationMessages.Add(createSubOrchestrationMessage); break; case OrchestratorActionType.OrchestrationComplete: TaskMessage workflowInstanceCompletedMessage = ProcessWorkflowCompletedTaskDecision((OrchestrationCompleteOrchestratorAction)decision, runtimeState, IncludeDetails, out continuedAsNew); if (workflowInstanceCompletedMessage != null) { // Send complete message to parent workflow or to itself to start a new execution BrokeredMessage workflowCompletedBrokeredMessage = Utils.GetBrokeredMessageFromObject( workflowInstanceCompletedMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance, "Complete Suborchestration"); workflowCompletedBrokeredMessage.SessionId = workflowInstanceCompletedMessage.OrchestrationInstance.InstanceId; // Store the event so we can rebuild the state if (continuedAsNew) { continuedAsNewMessage = workflowCompletedBrokeredMessage; continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; } else { subOrchestrationMessages.Add(workflowCompletedBrokeredMessage); } } isCompleted = !continuedAsNew; break; default: throw TraceHelper.TraceExceptionInstance(TraceEventType.Error, runtimeState.OrchestrationInstance, new NotSupportedException("decision type not supported")); } // We cannot send more than 100 messages within a transaction, to avoid the situation // we keep on checking the message count and stop processing the new decisions. // We also put in a fake timer to force next orchestration task for remaining messages int totalMessages = messagesToSend.Count + subOrchestrationMessages.Count + timerMessages.Count; // Also add tracking messages as they contribute to total messages within transaction if (isTrackingEnabled) { totalMessages += runtimeState.NewEvents.Count; } if (totalMessages > MaxMessageCount) { TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); if (isCompleted || continuedAsNew) { TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "Orchestration already completed. Skip adding timer for splitting messages."); break; } var dummyTimer = new CreateTimerOrchestratorAction { Id = FrameworkConstants.FakeTimerIdToSplitDecision, FireAt = DateTime.UtcNow }; TaskMessage timerMessage = ProcessCreateTimerDecision(dummyTimer, runtimeState); BrokeredMessage brokeredTimerMessage = Utils.GetBrokeredMessageFromObject( timerMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance, "MaxMessageCount Timer"); brokeredTimerMessage.ScheduledEnqueueTimeUtc = dummyTimer.FireAt; brokeredTimerMessage.SessionId = session.SessionId; timerMessages.Add(brokeredTimerMessage); break; } } } // TODO : make async, transactions are a bit tricky using (var ts = new TransactionScope()) { bool isSessionSizeThresholdExceeded = false; if (!continuedAsNew) { runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); } using (var ms = new MemoryStream()) { if (isCompleted) { TraceHelper.TraceSession(TraceEventType.Information, session.SessionId, "Deleting session state"); // if session was already null and we finished the orchestration then no change is required if (!isEmptySession) { session.SetState(null); } runtimeState.Size = 0; runtimeState.CompressedSize = 0; } else { if (ms.Position != 0) { throw TraceHelper.TraceExceptionInstance(TraceEventType.Error, runtimeState.OrchestrationInstance, new ArgumentException("Instance state stream is partially consumed")); } IList <HistoryEvent> newState = runtimeState.Events; if (continuedAsNew) { TraceHelper.TraceSession(TraceEventType.Information, session.SessionId, "Updating state for continuation"); newState = new List <HistoryEvent>(); newState.Add(new OrchestratorStartedEvent(-1)); newState.Add(continueAsNewExecutionStarted); newState.Add(new OrchestratorCompletedEvent(-1)); } string serializedState = JsonConvert.SerializeObject(newState, new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Auto }); var writer = new StreamWriter(ms); writer.Write(serializedState); writer.Flush(); ms.Position = 0; using (Stream compressedState = settings.TaskOrchestrationDispatcherSettings.CompressOrchestrationState ? Utils.GetCompressedStream(ms) : ms) { runtimeState.Size = ms.Length; runtimeState.CompressedSize = compressedState.Length; if (runtimeState.CompressedSize > SessionStreamTerminationThresholdInBytes) { // basic idea is to simply enqueue a terminate message just like how we do it from taskhubclient // it is possible to have other messages in front of the queue and those will get processed before // the terminate message gets processed. but that is ok since in the worst case scenario we will // simply land in this if-block again and end up queuing up another terminate message. // // the interesting scenario is when the second time we *dont* land in this if-block because e.g. // the new messages that we processed caused a new generation to be created. in that case // it is still ok because the worst case scenario is that we will terminate a newly created generation // which shouldn't have been created at all in the first place isSessionSizeThresholdExceeded = true; TraceHelper.TraceSession(TraceEventType.Critical, session.SessionId, "Size of session state " + runtimeState.CompressedSize + " has exceeded termination threshold of " + SessionStreamTerminationThresholdInBytes); string reason = string.Format( "Session state size of {0} exceeded the termination threshold of {1} bytes", runtimeState.CompressedSize, SessionStreamTerminationThresholdInBytes); BrokeredMessage forcedTerminateMessage = CreateForcedTerminateMessage( runtimeState.OrchestrationInstance.InstanceId, reason); orchestratorQueueClient.Send(forcedTerminateMessage); } else { session.SetState(compressedState); } } writer.Close(); } } if (!isSessionSizeThresholdExceeded) { if (runtimeState.CompressedSize > SessionStreamWarningSizeInBytes) { TraceHelper.TraceSession(TraceEventType.Error, session.SessionId, "Size of session state is nearing session size limit of 256KB"); } if (!continuedAsNew) { if (messagesToSend.Count > 0) { messagesToSend.ForEach(m => workerSender.Send(m)); } if (timerMessages.Count > 0) { timerMessages.ForEach(m => orchestratorQueueClient.Send(m)); } } if (subOrchestrationMessages.Count > 0) { subOrchestrationMessages.ForEach(m => orchestratorQueueClient.Send(m)); } if (continuedAsNewMessage != null) { orchestratorQueueClient.Send(continuedAsNewMessage); } if (isTrackingEnabled) { List <BrokeredMessage> trackingMessages = CreateTrackingMessages(runtimeState); TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance, "Created {0} tracking messages", trackingMessages.Count); if (trackingMessages.Count > 0) { trackingMessages.ForEach(m => trackingSender.Send(m)); } } } IEnumerable <Guid> lockTokens = newMessages.Select(m => m.LockToken); session.CompleteBatch(lockTokens); ts.Complete(); } }
/// <summary> /// Method to process a new work item /// </summary> /// <param name="workItem">The work item to process</param> protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem) { var messagesToSend = new List <TaskMessage>(); var timerMessages = new List <TaskMessage>(); var subOrchestrationMessages = new List <TaskMessage>(); bool isCompleted = false; bool continuedAsNew = false; ExecutionStartedEvent continueAsNewExecutionStarted = null; TaskMessage continuedAsNewMessage = null; OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState; runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); if (!ReconcileMessagesWithState(workItem)) { // TODO : mark an orchestration as faulted if there is data corruption TraceHelper.TraceSession( TraceEventType.Error, "TaskOrchestrationDispatcher-DeletedOrchestration", runtimeState.OrchestrationInstance?.InstanceId, "Received result for a deleted orchestration"); isCompleted = true; } else { TraceHelper.TraceInstance( TraceEventType.Verbose, "TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin", runtimeState.OrchestrationInstance, "Executing user orchestration: {0}", DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true)); IList <OrchestratorAction> decisions = (await ExecuteOrchestrationAsync(runtimeState)).ToList(); TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-ExecuteUserOrchestration-End", runtimeState.OrchestrationInstance, "Executed user orchestration. Received {0} orchestrator actions: {1}", decisions.Count(), string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); foreach (OrchestratorAction decision in decisions) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-ProcessOrchestratorAction", runtimeState.OrchestrationInstance, "Processing orchestrator action of type {0}", decision.OrchestratorActionType); switch (decision.OrchestratorActionType) { case OrchestratorActionType.ScheduleOrchestrator: messagesToSend.Add( ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState, IncludeParameters)); break; case OrchestratorActionType.CreateTimer: var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; timerMessages.Add(ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState)); break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; subOrchestrationMessages.Add( ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction, runtimeState, IncludeParameters)); break; case OrchestratorActionType.OrchestrationComplete: TaskMessage workflowInstanceCompletedMessage = ProcessWorkflowCompletedTaskDecision((OrchestrationCompleteOrchestratorAction)decision, runtimeState, IncludeDetails, out continuedAsNew); if (workflowInstanceCompletedMessage != null) { // Send complete message to parent workflow or to itself to start a new execution // Store the event so we can rebuild the state if (continuedAsNew) { continuedAsNewMessage = workflowInstanceCompletedMessage; continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; } else { subOrchestrationMessages.Add(workflowInstanceCompletedMessage); } } isCompleted = !continuedAsNew; break; default: throw TraceHelper.TraceExceptionInstance( TraceEventType.Error, "TaskOrchestrationDispatcher-UnsupportedDecisionType", runtimeState.OrchestrationInstance, new NotSupportedException("decision type not supported")); } // Underlying orchestration service provider may have a limit of messages per call, to avoid the situation // we keep on asking the provider if message count is ok and stop processing new decisions if not. // // We also put in a fake timer to force next orchestration task for remaining messages int totalMessages = messagesToSend.Count + subOrchestrationMessages.Count + timerMessages.Count; if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState)) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-MaxMessageCountReached", runtimeState.OrchestrationInstance, "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); if (isCompleted || continuedAsNew) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted", runtimeState.OrchestrationInstance, "Orchestration already completed. Skip adding timer for splitting messages."); break; } var dummyTimer = new CreateTimerOrchestratorAction { Id = FrameworkConstants.FakeTimerIdToSplitDecision, FireAt = DateTime.UtcNow }; timerMessages.Add(ProcessCreateTimerDecision(dummyTimer, runtimeState)); break; } } } // finish up processing of the work item if (!continuedAsNew) { runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); } OrchestrationRuntimeState newOrchestrationRuntimeState = workItem.OrchestrationRuntimeState; OrchestrationState instanceState = null; if (isCompleted) { TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state"); if (newOrchestrationRuntimeState.ExecutionStartedEvent != null) { instanceState = Utils.BuildOrchestrationState(newOrchestrationRuntimeState); } newOrchestrationRuntimeState = null; } else { instanceState = Utils.BuildOrchestrationState(newOrchestrationRuntimeState); if (continuedAsNew) { TraceHelper.TraceSession( TraceEventType.Information, "TaskOrchestrationDispatcher-UpdatingStateForContinuation", workItem.InstanceId, "Updating state for continuation"); newOrchestrationRuntimeState = new OrchestrationRuntimeState(); newOrchestrationRuntimeState.AddEvent(new OrchestratorStartedEvent(-1)); newOrchestrationRuntimeState.AddEvent(continueAsNewExecutionStarted); newOrchestrationRuntimeState.AddEvent(new OrchestratorCompletedEvent(-1)); } } await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( workItem, newOrchestrationRuntimeState, continuedAsNew?null : messagesToSend, subOrchestrationMessages, continuedAsNew?null : timerMessages, continuedAsNewMessage, instanceState); }