public async Task Release(TaskOrchestrationWorkItem workItem) { var request = new TaskOrchestrationRequest { ReleaseRequest = new ReleaseTaskOrchestrationWorkItemRequest() }; await _stream.RequestStream.WriteAsync(request); var cts = new CancellationTokenSource(_releaseResponseTimeout); if (!await _stream.ResponseStream.MoveNext(cts.Token)) { throw new Exception("Session aborted"); } if (_stream.ResponseStream.Current.MessageCase != TaskOrchestrationResponse.MessageOneofCase.ReleaseResponse) { throw new Exception("Unexpected response"); } await _stream.RequestStream.CompleteAsync(); // Last read to close stream await _stream.ResponseStream.MoveNext(cts.Token); _stream.Dispose(); }
public async Task Renew(TaskOrchestrationWorkItem workItem) { var request = new TaskOrchestrationRequest { RenewRequest = new RenewTaskOrchestrationWorkItemLockRequest() }; await _stream.RequestStream.WriteAsync(request); var cts = new CancellationTokenSource(_renewResponseTimeout); if (!await _stream.ResponseStream.MoveNext(cts.Token)) { throw new Exception("Session aborted"); } if (_stream.ResponseStream.Current.MessageCase != TaskOrchestrationResponse.MessageOneofCase.RenewResponse) { throw new Exception("Unexpected response"); } var renewResponse = _stream.ResponseStream.Current.RenewResponse; workItem.LockedUntilUtc = renewResponse.LockedUntilUtc.ToDateTime(); }
public async Task <IList <TaskMessage> > FetchNewOrchestrationMessagesAsync( TaskOrchestrationWorkItem workItem) { var request = new TaskOrchestrationRequest { FetchRequest = new FetchNewOrchestrationMessagesRequest() }; await _stream.RequestStream.WriteAsync(request); var cts = new CancellationTokenSource(_fetchResponseTimeout); if (!await _stream.ResponseStream.MoveNext(cts.Token)) { throw new Exception("Session aborted"); } if (_stream.ResponseStream.Current.MessageCase != TaskOrchestrationResponse.MessageOneofCase.FetchResponse) { throw new Exception("Unexpected response"); } var fetchResponse = _stream.ResponseStream.Current.FetchResponse; if (fetchResponse.NewMessages == null) { return(null); } return(fetchResponse.NewMessages.Messages .Select(x => _options.DataConverter.Deserialize <TaskMessage>(x)) .ToArray()); }
Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem) { OrchestrationExecutionCursor cursor = workItem.Cursor; var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance); dispatchContext.SetProperty(cursor.TaskOrchestration); dispatchContext.SetProperty(cursor.RuntimeState); dispatchContext.SetProperty(workItem); cursor.LatestDecisions = Enumerable.Empty <OrchestratorAction>(); return(this.dispatchPipeline.RunAsync(dispatchContext, _ => { cursor.LatestDecisions = cursor.OrchestrationExecutor.ExecuteNewEvents(); return CompletedTask; })); }
public async Task Complete( TaskOrchestrationWorkItem workItem, OrchestrationRuntimeState newOrchestrationRuntimeState, IList <TaskMessage> outboundMessages, IList <TaskMessage> orchestratorMessages, IList <TaskMessage> timerMessages, TaskMessage continuedAsNewMessage, OrchestrationState orchestrationState) { var request = new TaskOrchestrationRequest { CompleteRequest = new CompleteTaskOrchestrationWorkItemRequest { NewEvents = { workItem.OrchestrationRuntimeState.NewEvents.Select(_options.DataConverter.Serialize) }, NewStatus = workItem.OrchestrationRuntimeState.Status, NewOrchestrationEvents = { newOrchestrationRuntimeState == workItem.OrchestrationRuntimeState ? Enumerable.Empty <string>() : newOrchestrationRuntimeState.NewEvents.Select(_options.DataConverter.Serialize) }, NewOrchestrationStatus = newOrchestrationRuntimeState == workItem.OrchestrationRuntimeState ? null : newOrchestrationRuntimeState.Status, OutboundMessages = { outboundMessages.Select(_options.DataConverter.Serialize) }, OrchestratorMessages = { orchestratorMessages.Select(_options.DataConverter.Serialize) }, TimerMessages = { timerMessages.Select(_options.DataConverter.Serialize) }, ContinuedAsNewMessage = continuedAsNewMessage == null ? string.Empty : _options.DataConverter.Serialize(continuedAsNewMessage), OrchestrationState = _options.DataConverter.Serialize(orchestrationState) } }; await _stream.RequestStream.WriteAsync(request); var cts = new CancellationTokenSource(_completeResponseTimeout); if (!await _stream.ResponseStream.MoveNext(cts.Token)) { throw new Exception("Session aborted"); } if (_stream.ResponseStream.Current.MessageCase != TaskOrchestrationResponse.MessageOneofCase.CompleteResponse) { throw new Exception("Unexpected response"); } }
public async Task CompleteTaskOrchestrationWorkItemAsync( TaskOrchestrationWorkItem workItem, OrchestrationRuntimeState newOrchestrationRuntimeState, IList <TaskMessage> outboundMessages, IList <TaskMessage> orchestratorMessages, IList <TaskMessage> timerMessages, TaskMessage continuedAsNewMessage, OrchestrationState orchestrationState) { await(workItem.Session as GrpcClientOrchestrationSession).Complete( workItem, newOrchestrationRuntimeState, outboundMessages, orchestratorMessages, timerMessages, continuedAsNewMessage, orchestrationState); }
public async Task Abandon(TaskOrchestrationWorkItem workItem) { var request = new TaskOrchestrationRequest { AbandonRequest = new AbandonTaskOrchestrationWorkItemLockRequest() }; await _stream.RequestStream.WriteAsync(request); var cts = new CancellationTokenSource(_abandonResponseTimeout); if (!await _stream.ResponseStream.MoveNext(cts.Token)) { throw new Exception("Session aborted"); } if (_stream.ResponseStream.Current.MessageCase != TaskOrchestrationResponse.MessageOneofCase.AbandonResponse) { throw new Exception("Unexpected response"); } }
async Task ResumeOrchestrationAsync(TaskOrchestrationWorkItem workItem) { OrchestrationExecutionCursor cursor = workItem.Cursor; var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(cursor.RuntimeState.OrchestrationInstance); dispatchContext.SetProperty(cursor.TaskOrchestration); dispatchContext.SetProperty(cursor.RuntimeState); dispatchContext.SetProperty(workItem); cursor.LatestDecisions = Enumerable.Empty <OrchestratorAction>(); await this.dispatchPipeline.RunAsync(dispatchContext, _ => { OrchestratorExecutionResult result = cursor.OrchestrationExecutor.ExecuteNewEvents(); dispatchContext.SetProperty(result); return(CompletedTask); }); var result = dispatchContext.GetProperty <OrchestratorExecutionResult>(); cursor.LatestDecisions = result?.Actions ?? Enumerable.Empty <OrchestratorAction>(); cursor.RuntimeState.Status = result?.CustomStatus; }
async Task <OrchestrationExecutionCursor> ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem) { TaskOrchestration taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version); if (taskOrchestration == null) { throw TraceHelper.TraceExceptionInstance( TraceEventType.Error, "TaskOrchestrationDispatcher-TypeMissing", runtimeState.OrchestrationInstance, new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})")); } var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(runtimeState.OrchestrationInstance); dispatchContext.SetProperty(taskOrchestration); dispatchContext.SetProperty(runtimeState); dispatchContext.SetProperty(workItem); var executor = new TaskOrchestrationExecutor(runtimeState, taskOrchestration, this.orchestrationService.EventBehaviourForContinueAsNew); IEnumerable <OrchestratorAction> decisions = Enumerable.Empty <OrchestratorAction>(); await this.dispatchPipeline.RunAsync(dispatchContext, _ => { decisions = executor.Execute(); return(CompletedTask); }); return(new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions)); }
async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem) { try { if (workItem.Session == null) { // Legacy behavior await this.OnProcessWorkItemAsync(workItem); return; } var isExtendedSession = false; CorrelationTraceClient.Propagate( () => { // Check if it is extended session. isExtendedSession = this.concurrentSessionLock.Acquire(); this.concurrentSessionLock.Release(); workItem.IsExtendedSession = isExtendedSession; }); var processCount = 0; try { while (true) { // If the provider provided work items, execute them. if (workItem.NewMessages?.Count > 0) { bool isCompletedOrInterrupted = await this.OnProcessWorkItemAsync(workItem); if (isCompletedOrInterrupted) { break; } processCount++; } // Fetches beyond the first require getting an extended session lock, used to prevent starvation. if (processCount > 0 && !isExtendedSession) { isExtendedSession = this.concurrentSessionLock.Acquire(); if (!isExtendedSession) { TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-MaxOperations", "Failed to acquire concurrent session lock."); break; } } TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-StartFetch", "Starting fetch of existing session."); Stopwatch timer = Stopwatch.StartNew(); // Wait for new messages to arrive for the session. This call is expected to block (asynchronously) // until either new messages are available or until a provider-specific timeout has expired. workItem.NewMessages = await workItem.Session.FetchNewOrchestrationMessagesAsync(workItem); if (workItem.NewMessages == null) { break; } TraceHelper.Trace( TraceEventType.Verbose, "OnProcessWorkItemSession-EndFetch", $"Fetched {workItem.NewMessages.Count} new message(s) after {timer.ElapsedMilliseconds} ms from existing session."); workItem.OrchestrationRuntimeState.NewEvents.Clear(); } } finally { if (isExtendedSession) { TraceHelper.Trace( TraceEventType.Verbose, "OnProcessWorkItemSession-Release", $"Releasing extended session after {processCount} batch(es)."); this.concurrentSessionLock.Release(); } } } catch (SessionAbortedException e) { // Either the orchestration or the orchestration service explicitly abandoned the session. OrchestrationInstance instance = workItem.OrchestrationRuntimeState?.OrchestrationInstance ?? new OrchestrationInstance { InstanceId = workItem.InstanceId }; this.logHelper.OrchestrationAborted(instance, e.Message); TraceHelper.TraceInstance(TraceEventType.Warning, "TaskOrchestrationDispatcher-ExecutionAborted", instance, "{0}", e.Message); await this.orchestrationService.AbandonTaskOrchestrationWorkItemAsync(workItem); } }
public async Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { await(workItem.Session as GrpcClientOrchestrationSession).Abandon(workItem); }
public async Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { await(workItem.Session as GrpcClientOrchestrationSession).Release(workItem); }
static bool ReconcileMessagesWithState(TaskOrchestrationWorkItem workItem) { foreach (TaskMessage message in workItem.NewMessages) { OrchestrationInstance orchestrationInstance = message.OrchestrationInstance; if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { throw TraceHelper.TraceException( TraceEventType.Error, "TaskOrchestrationDispatcher-OrchestrationInstanceMissing", new InvalidOperationException("Message does not contain any OrchestrationInstance information")); } if (workItem.OrchestrationRuntimeState.Events.Count == 1 && message.Event.EventType != EventType.ExecutionStarted) { // we get here because of: // i) responses for scheduled tasks after the orchestrations have been completed // ii) responses for explicitly deleted orchestrations return(false); } TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-ProcessEvent", orchestrationInstance, "Processing new event with Id {0} and type {1}", message.Event.EventId, message.Event.EventType); if (message.Event.EventType == EventType.ExecutionStarted) { if (workItem.OrchestrationRuntimeState.Events.Count > 1) { // this was caused due to a dupe execution started event, swallow this one TraceHelper.TraceInstance( TraceEventType.Warning, "TaskOrchestrationDispatcher-DuplicateStartEvent", orchestrationInstance, "Duplicate start event. Ignoring event with Id {0} and type {1} ", message.Event.EventId, message.Event.EventType); continue; } } else if (!string.IsNullOrWhiteSpace(orchestrationInstance.ExecutionId) && !string.Equals(orchestrationInstance.ExecutionId, workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId)) { // eat up any events for previous executions TraceHelper.TraceInstance( TraceEventType.Warning, "TaskOrchestrationDispatcher-ExecutionIdMismatch", orchestrationInstance, "ExecutionId of event does not match current executionId. Ignoring event with Id {0} and type {1} ", message.Event.EventId, message.Event.EventType); continue; } workItem.OrchestrationRuntimeState.AddEvent(message.Event); } return(true); }
/// <summary> /// Method to process a new work item /// </summary> /// <param name="workItem">The work item to process</param> protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem) { var messagesToSend = new List <TaskMessage>(); var timerMessages = new List <TaskMessage>(); var subOrchestrationMessages = new List <TaskMessage>(); bool isCompleted = false; bool continuedAsNew = false; ExecutionStartedEvent continueAsNewExecutionStarted = null; TaskMessage continuedAsNewMessage = null; OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState; runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); if (!ReconcileMessagesWithState(workItem)) { // TODO : mark an orchestration as faulted if there is data corruption TraceHelper.TraceSession( TraceEventType.Error, "TaskOrchestrationDispatcher-DeletedOrchestration", runtimeState.OrchestrationInstance?.InstanceId, "Received result for a deleted orchestration"); isCompleted = true; } else { TraceHelper.TraceInstance( TraceEventType.Verbose, "TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin", runtimeState.OrchestrationInstance, "Executing user orchestration: {0}", DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true)); IList <OrchestratorAction> decisions = (await ExecuteOrchestrationAsync(runtimeState)).ToList(); TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-ExecuteUserOrchestration-End", runtimeState.OrchestrationInstance, "Executed user orchestration. Received {0} orchestrator actions: {1}", decisions.Count(), string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); foreach (OrchestratorAction decision in decisions) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-ProcessOrchestratorAction", runtimeState.OrchestrationInstance, "Processing orchestrator action of type {0}", decision.OrchestratorActionType); switch (decision.OrchestratorActionType) { case OrchestratorActionType.ScheduleOrchestrator: messagesToSend.Add( ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState, IncludeParameters)); break; case OrchestratorActionType.CreateTimer: var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; timerMessages.Add(ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState)); break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; subOrchestrationMessages.Add( ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction, runtimeState, IncludeParameters)); break; case OrchestratorActionType.OrchestrationComplete: TaskMessage workflowInstanceCompletedMessage = ProcessWorkflowCompletedTaskDecision((OrchestrationCompleteOrchestratorAction)decision, runtimeState, IncludeDetails, out continuedAsNew); if (workflowInstanceCompletedMessage != null) { // Send complete message to parent workflow or to itself to start a new execution // Store the event so we can rebuild the state if (continuedAsNew) { continuedAsNewMessage = workflowInstanceCompletedMessage; continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; } else { subOrchestrationMessages.Add(workflowInstanceCompletedMessage); } } isCompleted = !continuedAsNew; break; default: throw TraceHelper.TraceExceptionInstance( TraceEventType.Error, "TaskOrchestrationDispatcher-UnsupportedDecisionType", runtimeState.OrchestrationInstance, new NotSupportedException("decision type not supported")); } // Underlying orchestration service provider may have a limit of messages per call, to avoid the situation // we keep on asking the provider if message count is ok and stop processing new decisions if not. // // We also put in a fake timer to force next orchestration task for remaining messages int totalMessages = messagesToSend.Count + subOrchestrationMessages.Count + timerMessages.Count; if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState)) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-MaxMessageCountReached", runtimeState.OrchestrationInstance, "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); if (isCompleted || continuedAsNew) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted", runtimeState.OrchestrationInstance, "Orchestration already completed. Skip adding timer for splitting messages."); break; } var dummyTimer = new CreateTimerOrchestratorAction { Id = FrameworkConstants.FakeTimerIdToSplitDecision, FireAt = DateTime.UtcNow }; timerMessages.Add(ProcessCreateTimerDecision(dummyTimer, runtimeState)); break; } } } // finish up processing of the work item if (!continuedAsNew) { runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); } OrchestrationRuntimeState newOrchestrationRuntimeState = workItem.OrchestrationRuntimeState; OrchestrationState instanceState = null; if (isCompleted) { TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state"); if (newOrchestrationRuntimeState.ExecutionStartedEvent != null) { instanceState = Utils.BuildOrchestrationState(newOrchestrationRuntimeState); } newOrchestrationRuntimeState = null; } else { instanceState = Utils.BuildOrchestrationState(newOrchestrationRuntimeState); if (continuedAsNew) { TraceHelper.TraceSession( TraceEventType.Information, "TaskOrchestrationDispatcher-UpdatingStateForContinuation", workItem.InstanceId, "Updating state for continuation"); newOrchestrationRuntimeState = new OrchestrationRuntimeState(); newOrchestrationRuntimeState.AddEvent(new OrchestratorStartedEvent(-1)); newOrchestrationRuntimeState.AddEvent(continueAsNewExecutionStarted); newOrchestrationRuntimeState.AddEvent(new OrchestratorCompletedEvent(-1)); } } await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( workItem, newOrchestrationRuntimeState, continuedAsNew?null : messagesToSend, subOrchestrationMessages, continuedAsNew?null : timerMessages, continuedAsNewMessage, instanceState); }
async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem) { if (workItem.Session == null) { // Legacy behavior await OnProcessWorkItemAsync(workItem); return; } var isExtendedSession = false; var processCount = 0; try { while (true) { // If the provider provided work items, execute them. if (workItem.NewMessages?.Count > 0) { bool isCompletedOrInterrupted = await OnProcessWorkItemAsync(workItem); if (isCompletedOrInterrupted) { break; } processCount++; } // Fetches beyond the first require getting an extended session lock, used to prevent starvation. if (processCount > 0 && !isExtendedSession) { isExtendedSession = this.concurrentSessionLock.Acquire(); if (!isExtendedSession) { TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-MaxOperations", "Failed to acquire concurrent session lock."); break; } } TraceHelper.Trace(TraceEventType.Verbose, "OnProcessWorkItemSession-StartFetch", "Starting fetch of existing session."); Stopwatch timer = Stopwatch.StartNew(); // Wait for new messages to arrive for the session. This call is expected to block (asynchronously) // until either new messages are available or until a provider-specific timeout has expired. workItem.NewMessages = await workItem.Session.FetchNewOrchestrationMessagesAsync(workItem); if (workItem.NewMessages == null) { break; } TraceHelper.Trace( TraceEventType.Verbose, "OnProcessWorkItemSession-EndFetch", $"Fetched {workItem.NewMessages.Count} new message(s) after {timer.ElapsedMilliseconds} ms from existing session."); workItem.OrchestrationRuntimeState.NewEvents.Clear(); } } finally { if (isExtendedSession) { TraceHelper.Trace( TraceEventType.Verbose, "OnProcessWorkItemSession-Release", $"Releasing extended session after {processCount} batch(es)."); this.concurrentSessionLock.Release(); } } }
/// <summary> /// Method to process a new work item /// </summary> /// <param name="workItem">The work item to process</param> protected async Task <bool> OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem) { var messagesToSend = new List <TaskMessage>(); var timerMessages = new List <TaskMessage>(); var orchestratorMessages = new List <TaskMessage>(); var isCompleted = false; var continuedAsNew = false; var isInterrupted = false; // correlation CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext); ExecutionStartedEvent continueAsNewExecutionStarted = null; TaskMessage continuedAsNewMessage = null; IList <HistoryEvent> carryOverEvents = null; string carryOverStatus = null; OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState; runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); OrchestrationRuntimeState originalOrchestrationRuntimeState = runtimeState; OrchestrationState instanceState = null; if (!ReconcileMessagesWithState(workItem)) { // TODO : mark an orchestration as faulted if there is data corruption TraceHelper.TraceSession( TraceEventType.Error, "TaskOrchestrationDispatcher-DeletedOrchestration", runtimeState.OrchestrationInstance?.InstanceId, "Received result for a deleted orchestration"); isCompleted = true; } else { do { continuedAsNew = false; continuedAsNewMessage = null; TraceHelper.TraceInstance( TraceEventType.Verbose, "TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin", runtimeState.OrchestrationInstance, "Executing user orchestration: {0}", DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true)); if (workItem.Cursor == null) { workItem.Cursor = await ExecuteOrchestrationAsync(runtimeState); } else { await ResumeOrchestrationAsync(workItem.Cursor); } IReadOnlyList <OrchestratorAction> decisions = workItem.Cursor.LatestDecisions.ToList(); TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-ExecuteUserOrchestration-End", runtimeState.OrchestrationInstance, "Executed user orchestration. Received {0} orchestrator actions: {1}", decisions.Count, string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType))); foreach (OrchestratorAction decision in decisions) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-ProcessOrchestratorAction", runtimeState.OrchestrationInstance, "Processing orchestrator action of type {0}", decision.OrchestratorActionType); switch (decision.OrchestratorActionType) { case OrchestratorActionType.ScheduleOrchestrator: messagesToSend.Add( ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState, IncludeParameters)); break; case OrchestratorActionType.CreateTimer: var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision; timerMessages.Add(ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState)); break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; orchestratorMessages.Add( ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction, runtimeState, IncludeParameters)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; orchestratorMessages.Add( ProcessSendEventDecision(sendEventAction, runtimeState)); break; case OrchestratorActionType.OrchestrationComplete: OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision; TaskMessage workflowInstanceCompletedMessage = ProcessWorkflowCompletedTaskDecision(completeDecision, runtimeState, IncludeDetails, out continuedAsNew); if (workflowInstanceCompletedMessage != null) { // Send complete message to parent workflow or to itself to start a new execution // Store the event so we can rebuild the state carryOverEvents = null; if (continuedAsNew) { continuedAsNewMessage = workflowInstanceCompletedMessage; continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent; if (completeDecision.CarryoverEvents.Any()) { carryOverEvents = completeDecision.CarryoverEvents.ToList(); completeDecision.CarryoverEvents.Clear(); } } else { orchestratorMessages.Add(workflowInstanceCompletedMessage); } } isCompleted = !continuedAsNew; break; default: throw TraceHelper.TraceExceptionInstance( TraceEventType.Error, "TaskOrchestrationDispatcher-UnsupportedDecisionType", runtimeState.OrchestrationInstance, new NotSupportedException("decision type not supported")); } // Underlying orchestration service provider may have a limit of messages per call, to avoid the situation // we keep on asking the provider if message count is ok and stop processing new decisions if not. // // We also put in a fake timer to force next orchestration task for remaining messages int totalMessages = messagesToSend.Count + orchestratorMessages.Count + timerMessages.Count; if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState)) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-MaxMessageCountReached", runtimeState.OrchestrationInstance, "MaxMessageCount reached. Adding timer to process remaining events in next attempt."); if (isCompleted || continuedAsNew) { TraceHelper.TraceInstance( TraceEventType.Information, "TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted", runtimeState.OrchestrationInstance, "Orchestration already completed. Skip adding timer for splitting messages."); break; } var dummyTimer = new CreateTimerOrchestratorAction { Id = FrameworkConstants.FakeTimerIdToSplitDecision, FireAt = DateTime.UtcNow }; timerMessages.Add(ProcessCreateTimerDecision(dummyTimer, runtimeState)); isInterrupted = true; break; } } // finish up processing of the work item if (!continuedAsNew && runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted) { runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); } if (isCompleted) { TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state"); if (runtimeState.ExecutionStartedEvent != null) { instanceState = Utils.BuildOrchestrationState(runtimeState); } } else { if (continuedAsNew) { TraceHelper.TraceSession( TraceEventType.Information, "TaskOrchestrationDispatcher-UpdatingStateForContinuation", workItem.InstanceId, "Updating state for continuation"); runtimeState = new OrchestrationRuntimeState(); runtimeState.AddEvent(new OrchestratorStartedEvent(-1)); runtimeState.AddEvent(continueAsNewExecutionStarted); runtimeState.Status = workItem.OrchestrationRuntimeState.Status ?? carryOverStatus; carryOverStatus = workItem.OrchestrationRuntimeState.Status; if (carryOverEvents != null) { foreach (var historyEvent in carryOverEvents) { runtimeState.AddEvent(historyEvent); } } runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); workItem.OrchestrationRuntimeState = runtimeState; TaskOrchestration newOrchestration = this.objectManager.GetObject( runtimeState.Name, continueAsNewExecutionStarted.Version); workItem.Cursor = new OrchestrationExecutionCursor( runtimeState, newOrchestration, new TaskOrchestrationExecutor( runtimeState, newOrchestration, orchestrationService.EventBehaviourForContinueAsNew), latestDecisions: null); if (workItem.LockedUntilUtc < DateTime.UtcNow.AddMinutes(1)) { await orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem); } } instanceState = Utils.BuildOrchestrationState(runtimeState); } } while (continuedAsNew); } workItem.OrchestrationRuntimeState = originalOrchestrationRuntimeState; runtimeState.Status = runtimeState.Status ?? carryOverStatus; await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( workItem, runtimeState, continuedAsNew?null : messagesToSend, orchestratorMessages, continuedAsNew?null : timerMessages, continuedAsNewMessage, instanceState); workItem.OrchestrationRuntimeState = runtimeState; return(isCompleted || continuedAsNew || isInterrupted); }
public async Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem) { await(workItem.Session as GrpcClientOrchestrationSession).Renew(workItem); }
public override async Task LockNextTaskOrchestrationWorkItem(IAsyncStreamReader <TaskOrchestrationRequest> requestStream, IServerStreamWriter <TaskOrchestrationResponse> responseStream, ServerCallContext context) { try { TaskOrchestrationWorkItem workItem = null; // Receive and reply each message await foreach (var message in requestStream.ReadAllAsync(context.CancellationToken)) { switch (message.MessageCase) { case TaskOrchestrationRequest.MessageOneofCase.LockRequest: var lockRequest = message.LockRequest; var orchestrations = lockRequest.Orchestrations.Select(x => new NameVersion(x.Name, x.Version)).ToArray(); workItem = await(lockRequest.AllOrchestrations ? _orchestrationService .LockNextTaskOrchestrationWorkItemAsync(lockRequest.ReceiveTimeout.ToTimeSpan(), context.CancellationToken) : (_extendedOrchestrationService ?? throw DistributedWorkersNotSupported()) .LockNextTaskOrchestrationWorkItemAsync(lockRequest.ReceiveTimeout.ToTimeSpan(), orchestrations, context.CancellationToken) ); var lockResponse = new TaskOrchestrationResponse { LockResponse = new LockNextTaskOrchestrationWorkItemResponse { WorkItem = workItem == null ? null : new DurableTaskGrpc.TaskOrchestrationWorkItem { InstanceId = workItem.InstanceId, LockedUntilUtc = Timestamp.FromDateTime(workItem.LockedUntilUtc), Events = { workItem.OrchestrationRuntimeState.Events.Select(_options.DataConverter.Serialize) }, NewMessages = { workItem.NewMessages.Select(_options.DataConverter.Serialize) } } } }; context.CancellationToken.ThrowIfCancellationRequested(); await responseStream.WriteAsync(lockResponse); break; case TaskOrchestrationRequest.MessageOneofCase.RenewRequest: var renewRequest = message.RenewRequest; await _orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem); var renewResponse = new TaskOrchestrationResponse { RenewResponse = new RenewTaskOrchestrationWorkItemLockResponse { LockedUntilUtc = Timestamp.FromDateTime(workItem.LockedUntilUtc) } }; context.CancellationToken.ThrowIfCancellationRequested(); await responseStream.WriteAsync(renewResponse); break; case TaskOrchestrationRequest.MessageOneofCase.CompleteRequest: var completeRequest = message.CompleteRequest; var outboundMessages = completeRequest.OutboundMessages.Select(x => _options.DataConverter.Deserialize <TaskMessage>(x)).ToArray(); var timerMessages = completeRequest.TimerMessages.Select(x => _options.DataConverter.Deserialize <TaskMessage>(x)).ToArray(); var orchestratorMessages = completeRequest.OrchestratorMessages.Select(x => _options.DataConverter.Deserialize <TaskMessage>(x)).ToArray(); var continuedAsNewMessage = string.IsNullOrEmpty(completeRequest.ContinuedAsNewMessage) ? null : _options.DataConverter.Deserialize <TaskMessage>(completeRequest.ContinuedAsNewMessage); var newEvents = completeRequest.NewEvents.Select(x => _options.DataConverter.Deserialize <HistoryEvent>(x)).ToArray(); workItem.OrchestrationRuntimeState ??= new OrchestrationRuntimeState(); foreach (var newEvent in newEvents) { workItem.OrchestrationRuntimeState.AddEvent(newEvent); } workItem.OrchestrationRuntimeState.Status = completeRequest.NewStatus; var newOrchestrationRuntimeState = workItem.OrchestrationRuntimeState; var newOrchestrationRuntimeStateEvents = completeRequest.NewOrchestrationEvents.Select(x => _options.DataConverter.Deserialize <HistoryEvent>(x)).ToArray(); if (newOrchestrationRuntimeStateEvents.Length > 0) { newOrchestrationRuntimeState = new OrchestrationRuntimeState(); foreach (var newEvent in newOrchestrationRuntimeStateEvents) { newOrchestrationRuntimeState.AddEvent(newEvent); } newOrchestrationRuntimeState.Status = completeRequest.NewOrchestrationStatus; } var orchestrationState = Utils.BuildOrchestrationState(newOrchestrationRuntimeState); await _orchestrationService.CompleteTaskOrchestrationWorkItemAsync( workItem, newOrchestrationRuntimeState, outboundMessages, orchestratorMessages, timerMessages, continuedAsNewMessage, orchestrationState); newOrchestrationRuntimeState.NewEvents.Clear(); workItem.OrchestrationRuntimeState = newOrchestrationRuntimeState; context.CancellationToken.ThrowIfCancellationRequested(); await responseStream.WriteAsync(new TaskOrchestrationResponse { CompleteResponse = new CompleteTaskOrchestrationWorkItemResponse() }); break; case TaskOrchestrationRequest.MessageOneofCase.FetchRequest: var fetchRequest = message.FetchRequest; if (workItem.Session == null) { var fetchResponse = new TaskOrchestrationResponse { FetchResponse = new FetchNewOrchestrationMessagesResponse { NewMessages = null } }; context.CancellationToken.ThrowIfCancellationRequested(); await responseStream.WriteAsync(fetchResponse); } else { var newMessages = await workItem.Session.FetchNewOrchestrationMessagesAsync(workItem); var fetchResponse = new TaskOrchestrationResponse { FetchResponse = new FetchNewOrchestrationMessagesResponse { NewMessages = newMessages == null ? null : new OrchestrationMessages { Messages = { newMessages.Select(_options.DataConverter.Serialize) } } } }; context.CancellationToken.ThrowIfCancellationRequested(); await responseStream.WriteAsync(fetchResponse); } break; case TaskOrchestrationRequest.MessageOneofCase.ReleaseRequest: var releaseRequest = message.ReleaseRequest; await _orchestrationService.ReleaseTaskOrchestrationWorkItemAsync(workItem); context.CancellationToken.ThrowIfCancellationRequested(); await responseStream.WriteAsync(new TaskOrchestrationResponse { ReleaseResponse = new ReleaseTaskOrchestrationWorkItemResponse() }); break; case TaskOrchestrationRequest.MessageOneofCase.AbandonRequest: var abandonRequest = message.AbandonRequest; await _orchestrationService.AbandonTaskOrchestrationWorkItemAsync(workItem); context.CancellationToken.ThrowIfCancellationRequested(); await responseStream.WriteAsync(new TaskOrchestrationResponse { AbandonResponse = new AbandonTaskOrchestrationWorkItemLockResponse() }); break; } } } catch (OperationCanceledException) when(context.CancellationToken.IsCancellationRequested) { // Avoid exceptions when clients cancel request } }
async Task <OrchestrationExecutionCursor> ExecuteOrchestrationAsync(OrchestrationRuntimeState runtimeState, TaskOrchestrationWorkItem workItem) { // Get the TaskOrchestration implementation. If it's not found, it either means that the developer never // registered it (which is an error, and we'll throw for this further down) or it could be that some custom // middleware (e.g. out-of-process execution middleware) is intended to implement the orchestration logic. TaskOrchestration?taskOrchestration = this.objectManager.GetObject(runtimeState.Name, runtimeState.Version !); var dispatchContext = new DispatchMiddlewareContext(); dispatchContext.SetProperty(runtimeState.OrchestrationInstance); dispatchContext.SetProperty(taskOrchestration); dispatchContext.SetProperty(runtimeState); dispatchContext.SetProperty(workItem); TaskOrchestrationExecutor?executor = null; await this.dispatchPipeline.RunAsync(dispatchContext, _ => { // Check to see if the custom middleware intercepted and substituted the orchestration execution // with its own execution behavior, providing us with the end results. If so, we can terminate // the dispatch pipeline here. var resultFromMiddleware = dispatchContext.GetProperty <OrchestratorExecutionResult>(); if (resultFromMiddleware != null) { return(CompletedTask); } if (taskOrchestration == null) { throw TraceHelper.TraceExceptionInstance( TraceEventType.Error, "TaskOrchestrationDispatcher-TypeMissing", runtimeState.OrchestrationInstance, new TypeMissingException($"Orchestration not found: ({runtimeState.Name}, {runtimeState.Version})")); } executor = new TaskOrchestrationExecutor( runtimeState, taskOrchestration, this.orchestrationService.EventBehaviourForContinueAsNew, this.errorPropagationMode); OrchestratorExecutionResult resultFromOrchestrator = executor.Execute(); dispatchContext.SetProperty(resultFromOrchestrator); return(CompletedTask); }); var result = dispatchContext.GetProperty <OrchestratorExecutionResult>(); IEnumerable <OrchestratorAction> decisions = result?.Actions ?? Enumerable.Empty <OrchestratorAction>(); runtimeState.Status = result?.CustomStatus; return(new OrchestrationExecutionCursor(runtimeState, taskOrchestration, executor, decisions)); }