/// <summary> /// Logs that a work item message is being dropped and includes a reason. /// </summary> /// <param name="workItem">The work item that this message belonged to.</param> /// <param name="message">The message being dropped.</param> /// <param name="reason">The reason for dropping the message.</param> internal void DroppingOrchestrationMessage(TaskOrchestrationWorkItem workItem, TaskMessage message, string reason) { if (this.IsStructuredLoggingEnabled) { this.WriteStructuredLog(new LogEvents.DiscardingMessage(workItem, message, reason)); } }
// Note: Do not rely on cancellationToken parameter to this method because the top layer does not yet implement any cancellation. public async Task <TaskOrchestrationWorkItem> LockNextTaskOrchestrationWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken) { var currentSession = await this.orchestrationProvider.AcceptSessionAsync(receiveTimeout); if (currentSession == null) { return(null); } List <Message <Guid, TaskMessageItem> > newMessages; try { newMessages = await this.orchestrationProvider.ReceiveSessionMessagesAsync(currentSession); var currentRuntimeState = new OrchestrationRuntimeState(currentSession.SessionState); var workItem = new TaskOrchestrationWorkItem() { NewMessages = newMessages.Select(m => m.Value.TaskMessage).ToList(), InstanceId = currentSession.SessionId.InstanceId, OrchestrationRuntimeState = currentRuntimeState }; if (newMessages.Count == 0) { if (currentRuntimeState.ExecutionStartedEvent == null) { ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition($"Orchestration with no execution started event found: {currentSession.SessionId}"); return(null); } bool isComplete = this.IsOrchestrationComplete(currentRuntimeState.OrchestrationStatus); if (isComplete) { await this.HandleCompletedOrchestration(workItem); } this.orchestrationProvider.TryUnlockSession(currentSession.SessionId, isComplete: isComplete); return(null); } var sessionInfo = new SessionInformation() { Instance = currentSession.SessionId, LockTokens = newMessages.Select(m => m.Key).ToList() }; if (!this.sessionInfos.TryAdd(workItem.InstanceId, sessionInfo)) { ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition($"{nameof(FabricOrchestrationService)}.{nameof(LockNextTaskOrchestrationWorkItemAsync)} : Multiple receivers processing the same session : {currentSession.SessionId.InstanceId}?"); } return(workItem); } catch (Exception) { this.orchestrationProvider.TryUnlockSession(currentSession.SessionId, abandon: true); throw; } }
/// <summary> /// Helper method for logging the dropping of all messages associated with the specified work item. /// </summary> /// <param name="workItem">The work item being dropped.</param> /// <param name="reason">The reason for dropping this work item.</param> internal void DroppingOrchestrationWorkItem(TaskOrchestrationWorkItem workItem, string reason) { foreach (TaskMessage message in workItem.NewMessages) { this.DroppingOrchestrationMessage(workItem, message, reason); } }
/// <summary> /// Logs that an orchestration work item message is being processed. /// </summary> /// <param name="workItem">The orchestration work item.</param> /// <param name="message">The message being processed.</param> internal void ProcessingOrchestrationMessage(TaskOrchestrationWorkItem workItem, TaskMessage message) { if (this.IsStructuredLoggingEnabled) { this.WriteStructuredLog(new LogEvents.ProcessingOrchestrationMessage(workItem, message)); } }
// Called by the DTFx dispatcher thread public async Task <IList <TaskMessage> > FetchNewOrchestrationMessagesAsync( TaskOrchestrationWorkItem workItem) { if (!await this.messagesAvailableEvent.WaitAsync(this.idleTimeout)) { return(null); // timed-out } this.StartNewLogicalTraceScope(); lock (this.nextMessageBatch) { this.CurrentMessageBatch = this.nextMessageBatch.ToList(); this.nextMessageBatch.Clear(); } var messages = new List <TaskMessage>(this.CurrentMessageBatch.Count); foreach (MessageData msg in this.CurrentMessageBatch) { this.TraceProcessingMessage(msg, isExtendedSession: true); messages.Add(msg.TaskMessage); } return(messages); }
public abstract Task CompleteTaskOrchestrationWorkItemAsync( TaskOrchestrationWorkItem workItem, OrchestrationRuntimeState newOrchestrationRuntimeState, IList <TaskMessage> outboundMessages, IList <TaskMessage> orchestratorMessages, IList <TaskMessage> timerMessages, TaskMessage continuedAsNewMessage, OrchestrationState orchestrationState);
public Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { if (OrchestrationsServiceScopes.TryRemove(workItem.InstanceId, out var serviceScope)) { serviceScope.Dispose(); } return(_innerOrchestrationService.ReleaseTaskOrchestrationWorkItemAsync(workItem)); }
/// <inheritdoc /> public async Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { if (this.partitionOrchestrationManager == null || this.redisConnection == null) { await StartAsync(); } // TODO: Handle differently then just releasing? await ReleaseTaskOrchestrationWorkItemAsync(workItem); }
/// <inheritdoc /> public async Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { if (this.partitionOrchestrationManager == null || this.redisConnection == null) { await StartAsync(); } await this.logger.LogAsync($"Releasing lock on orchestration {workItem.InstanceId}"); this.partitionOrchestrationManager.ReleaseOrchestration(workItem.InstanceId); }
public Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { bool isComplete = this.IsOrchestrationComplete(workItem.OrchestrationRuntimeState.OrchestrationStatus); SessionInformation sessionInfo = TryRemoveSessionInfo(workItem.InstanceId); if (sessionInfo != null) { this.orchestrationProvider.TryUnlockSession(sessionInfo.Instance, isComplete: isComplete); } return(Task.CompletedTask); }
public Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { SessionInformation sessionInfo = TryRemoveSessionInfo(workItem.InstanceId); if (sessionInfo == null) { ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition($"{nameof(AbandonTaskOrchestrationWorkItemAsync)} : Could not get a session info object while trying to abandon session {workItem.InstanceId}"); } else { this.orchestrationProvider.TryUnlockSession(sessionInfo.Instance, abandon: true); } return(Task.CompletedTask); }
public async Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem) { using (var dbContext = _dbContextFactory()) { var session = workItem.Session as EFCoreOrchestrationSession; var lockedUntilUTC = DateTime.UtcNow.Add(_options.OrchestrationLockTimeout); dbContext.Instances.Attach(session.Instance); session.Instance.LockedUntil = DateTime.UtcNow.Add(_options.OrchestrationLockTimeout); await dbContext.SaveChangesAsync(); workItem.LockedUntilUtc = lockedUntilUTC; } }
public override async Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem) { using SqlConnection connection = await this.GetAndOpenConnectionAsync(); using SqlCommand command = this.GetSprocCommand(connection, "dt._RenewOrchestrationLocks"); DateTime lockExpiration = DateTime.UtcNow.Add(this.settings.WorkItemLockTimeout); command.Parameters.Add("@InstanceID", SqlDbType.VarChar, size: 100).Value = workItem.InstanceId; command.Parameters.Add("@LockExpiration", SqlDbType.DateTime2).Value = lockExpiration; await SqlUtils.ExecuteNonQueryAsync(command, this.traceHelper, workItem.InstanceId); workItem.LockedUntilUtc = lockExpiration; }
public async Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { using (var dbContext = _dbContextFactory()) { var session = workItem.Session as EFCoreOrchestrationSession; if (!session.Released) { dbContext.Instances.Attach(session.Instance); session.Instance.LockId = null; session.Instance.LockedUntil = DateTime.UtcNow; await dbContext.SaveChangesAsync(); session.Released = true; } } }
public async Task <IList <TaskMessage> > FetchNewOrchestrationMessagesAsync( TaskOrchestrationWorkItem workItem) { return(await BackoffPollingHelper.PollAsync(async() => { using (var dbContext = _dbContextFactory()) { var messages = await FetchNewMessagesAsync(dbContext); await dbContext.SaveChangesAsync(); return messages; } }, x => x.Count > 0, _options.FetchNewMessagesPollingTimeout, _options.PollingInterval, _stopCancellationToken)); }
/// <inheritdoc/> public Task CompleteTaskOrchestrationWorkItemAsync( TaskOrchestrationWorkItem workItem, OrchestrationRuntimeState newOrchestrationRuntimeState, IList <TaskMessage> outboundMessages, IList <TaskMessage> orchestratorMessages, IList <TaskMessage> timerMessages, TaskMessage continuedAsNewMessage, OrchestrationState orchestrationState) { return(this.GetOrchestrationService().CompleteTaskOrchestrationWorkItemAsync( workItem, newOrchestrationRuntimeState, outboundMessages, orchestratorMessages, timerMessages, continuedAsNewMessage, orchestrationState)); }
public async Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { using (var dbContext = _dbContextFactory()) { var session = workItem.Session as EFCoreOrchestrationSession; if (session.Released) { throw new InvalidOperationException("Session was already released"); } dbContext.Instances.Attach(session.Instance); session.Instance.LockId = null; // TODO: Exponential backoff session.Instance.LockedUntil = DateTime.UtcNow.AddMinutes(1); await dbContext.SaveChangesAsync(); session.Released = true; } }
/// <inheritdoc /> public async Task <TaskOrchestrationWorkItem> LockNextTaskOrchestrationWorkItemAsync( TimeSpan receiveTimeout, CancellationToken cancellationToken) { TaskSession taskSession = await this._OrchestratorQueue.AcceptSessionAsync(receiveTimeout, CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, this._CancellationTokenSource.Token).Token); if (taskSession == null) { return(null); } var wi = new TaskOrchestrationWorkItem { NewMessages = taskSession.Messages.ToList(), InstanceId = taskSession.Id, LockedUntilUtc = DateTime.UtcNow.AddMinutes(5), OrchestrationRuntimeState = DeserializeOrchestrationRuntimeState(taskSession.SessionState) ?? new OrchestrationRuntimeState(), }; return(wi); }
// Caller should ensure the workItem has reached terminal state. private async Task HandleCompletedOrchestration(TaskOrchestrationWorkItem workItem) { await RetryHelper.ExecuteWithRetryOnTransient(async() => { using (var txn = this.stateManager.CreateTransaction()) { await this.instanceStore.WriteEntitiesAsync(txn, new InstanceEntityBase[] { new OrchestrationStateInstanceEntity() { State = Utils.BuildOrchestrationState(workItem.OrchestrationRuntimeState) } }); // DropSession does 2 things (like mentioned in the comments above) - remove the row from sessions dictionary // and delete the session messages dictionary. The second step is in a background thread and not part of transaction. // However even if this transaction failed but we ended up deleting session messages dictionary, that's ok - at // that time, it should be an empty dictionary and we would have updated the runtime session state to full completed // state in the transaction from Complete method. So the subsequent attempt would be able to complete the session. await this.orchestrationProvider.DropSession(txn, workItem.OrchestrationRuntimeState.OrchestrationInstance); await txn.CommitAsync(); } }, uniqueActionIdentifier : $"OrchestrationId = '{workItem.InstanceId}', Action = '{nameof(HandleCompletedOrchestration)}'"); this.instanceStore.OnOrchestrationCompleted(workItem.OrchestrationRuntimeState.OrchestrationInstance); string message = string.Format("Orchestration with instanceId : '{0}' and executionId : '{1}' Finished with the status {2} and result {3} in {4} seconds.", workItem.InstanceId, workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId, workItem.OrchestrationRuntimeState.OrchestrationStatus.ToString(), workItem.OrchestrationRuntimeState.Output, (workItem.OrchestrationRuntimeState.CompletedTime - workItem.OrchestrationRuntimeState.CreatedTime).TotalSeconds); ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId, workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId, message); }
public async Task <IList <TaskMessage> > FetchNewOrchestrationMessagesAsync( TaskOrchestrationWorkItem workItem) { if (!await this.messagesAvailableEvent.WaitAsync(this.idleTimeout)) { return(null); } this.CurrentMessageBatch = this.fetchMessagesCallback.Invoke(this.Instance); if (this.CurrentMessageBatch == null) { return(null); } var messages = new List <TaskMessage>(this.CurrentMessageBatch.Count); foreach (MessageData msg in this.CurrentMessageBatch) { this.TraceProcessingMessage(msg, isExtendedSession: true); messages.Add(msg.TaskMessage); } return(messages); }
public async Task CompleteTaskOrchestrationWorkItemAsync( TaskOrchestrationWorkItem workItem, OrchestrationRuntimeState newOrchestrationRuntimeState, IList <TaskMessage> outboundMessages, IList <TaskMessage> orchestratorMessages, IList <TaskMessage> timerMessages, TaskMessage continuedAsNewMessage, OrchestrationState orchestrationState) { SessionInformation sessionInfo = GetSessionInfo(workItem.InstanceId); ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId, workItem.OrchestrationRuntimeState.OrchestrationInstance?.ExecutionId, $"Current orchestration status: {workItem.OrchestrationRuntimeState.OrchestrationStatus}"); bool isComplete = this.IsOrchestrationComplete(workItem.OrchestrationRuntimeState.OrchestrationStatus); IList <OrchestrationInstance> sessionsToEnqueue = null; List <Message <Guid, TaskMessageItem> > scheduledMessages = null; List <Message <string, TaskMessageItem> > activityMessages = null; await RetryHelper.ExecuteWithRetryOnTransient(async() => { bool retryOnException; do { try { retryOnException = false; sessionsToEnqueue = null; scheduledMessages = null; activityMessages = null; using (var txn = this.stateManager.CreateTransaction()) { if (outboundMessages?.Count > 0) { activityMessages = outboundMessages.Select(m => new Message <string, TaskMessageItem>(Guid.NewGuid().ToString(), new TaskMessageItem(m))).ToList(); await this.activitiesProvider.SendBatchBeginAsync(txn, activityMessages); } if (timerMessages?.Count > 0) { scheduledMessages = timerMessages.Select(m => new Message <Guid, TaskMessageItem>(Guid.NewGuid(), new TaskMessageItem(m))).ToList(); await this.scheduledMessagesProvider.SendBatchBeginAsync(txn, scheduledMessages); } if (orchestratorMessages?.Count > 0) { if (workItem.OrchestrationRuntimeState?.ParentInstance != null) { sessionsToEnqueue = await this.orchestrationProvider.TryAppendMessageBatchAsync(txn, orchestratorMessages.Select(tm => new TaskMessageItem(tm))); } else { await this.orchestrationProvider.AppendMessageBatchAsync(txn, orchestratorMessages.Select(tm => new TaskMessageItem(tm))); sessionsToEnqueue = orchestratorMessages.Select(m => m.OrchestrationInstance).ToList(); } } if (continuedAsNewMessage != null) { await this.orchestrationProvider.AppendMessageAsync(txn, new TaskMessageItem(continuedAsNewMessage)); sessionsToEnqueue = new List <OrchestrationInstance>() { continuedAsNewMessage.OrchestrationInstance }; } await this.orchestrationProvider.CompleteMessages(txn, sessionInfo.Instance, sessionInfo.LockTokens); if (workItem.OrchestrationRuntimeState.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew) { await HandleCompletedOrchestration(workItem); } // When an orchestration is completed, we need to drop the session which involves 2 steps (1) Removing the row from sessions // (2) Dropping the session messages dictionary. The second step is done in background thread for performance so is not // part of transaction. Since it will happen outside the trasanction, if this transaction fails for some reason and we dropped // the session as part of this transaction, we wouldn't have updated the session state but would have lost the messages // in the session messages dictionary which are needed for state to reach complete state (when the orchestration is picked up again in next fetch). // So we don't want to drop session as part of this transaction. // Instead, we drop the session as part of a subsequent different transaction. // However, framework passes us 'null' value for 'newOrchestrationRuntimeState' when orchestration is completed and // if we updated the session state to null and this transaction succeded, and a node failures occurs and we // never call the subsequent transaction, we will lose the runtime state of orchestration and never will be able to // mark it as complete even if it is. So we use the work item's runtime state when 'newOrchestrationRuntimeState' is null // so that the latest state is what is stored for the session. // As part of next transaction, we are going to remove the row anyway for the session and it doesn't matter to update it to 'null'. await this.orchestrationProvider.UpdateSessionState(txn, newOrchestrationRuntimeState.OrchestrationInstance, newOrchestrationRuntimeState ?? workItem.OrchestrationRuntimeState); // We skip writing to instanceStore when orchestration reached terminal state to avoid a minor timing issue that // wait for an orchestration completes but another orchestration with the same name cannot be started immediately // because the session is still in store. We update the instance store on orchestration completion and drop the // session as part of the next atomic transaction. if (this.instanceStore != null && orchestrationState != null && !isComplete) { await this.instanceStore.WriteEntitiesAsync(txn, new InstanceEntityBase[] { new OrchestrationStateInstanceEntity() { State = orchestrationState } }); } await txn.CommitAsync(); } } catch (FabricReplicationOperationTooLargeException ex) { ServiceFabricProviderEventSource.Tracing.ExceptionInReliableCollectionOperations($"OrchestrationInstance = {sessionInfo.Instance}, Action = {nameof(CompleteTaskOrchestrationWorkItemAsync)}", ex.ToString()); retryOnException = true; newOrchestrationRuntimeState = null; outboundMessages = null; timerMessages = null; orchestratorMessages = null; if (orchestrationState != null) { orchestrationState.OrchestrationStatus = OrchestrationStatus.Failed; orchestrationState.Output = $"Fabric exception when trying to process orchestration: {ex}. Investigate and consider reducing the serialization size of orchestration inputs/outputs/overall length to avoid the issue."; } } } while (retryOnException); }, uniqueActionIdentifier : $"OrchestrationId = '{workItem.InstanceId}', Action = '{nameof(CompleteTaskOrchestrationWorkItemAsync)}'"); if (activityMessages != null) { this.activitiesProvider.SendBatchComplete(activityMessages); } if (scheduledMessages != null) { this.scheduledMessagesProvider.SendBatchComplete(scheduledMessages); } if (sessionsToEnqueue != null) { foreach (var instance in sessionsToEnqueue) { this.orchestrationProvider.TryEnqueueSession(instance); } } if (isComplete) { await HandleCompletedOrchestration(workItem); } }
public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem) { return(Task.CompletedTask); }
/// <inheritdoc /> public async Task CompleteTaskOrchestrationWorkItemAsync( TaskOrchestrationWorkItem workItem, OrchestrationRuntimeState newOrchestrationRuntimeState, IList <TaskMessage> outboundMessages, IList <TaskMessage> orchestratorMessages, IList <TaskMessage> timerMessages, TaskMessage continuedAsNewMessage, OrchestrationState orchestrationState) { if (this.partitionOrchestrationManager == null || this.redisConnection == null) { await StartAsync(); } string orchestrationId = workItem.InstanceId; RedisTransactionBuilder transaction = this.partitionOrchestrationManager.CreateExistingOrchestrationTransaction(orchestrationId); List <string> events = newOrchestrationRuntimeState.Events.Select(histEvent => histEvent as TaskCompletedEvent) .Where(taskCompletedEvent => taskCompletedEvent != null) .OrderBy(task => task.TaskScheduledId) .Select(TaskCompletedEvent => $"{{\"id\": {TaskCompletedEvent.TaskScheduledId}, \"Result\": {TaskCompletedEvent.Result}}}") .ToList(); string logMessage = "Current events processed: " + string.Join(",", events); await this.logger.LogAsync(logMessage); transaction.SetOrchestrationRuntimeState(orchestrationId, newOrchestrationRuntimeState); foreach (TaskMessage outboundMessage in outboundMessages) { transaction.SendActivityMessage(outboundMessage); } foreach (TaskMessage message in orchestratorMessages) { transaction.SendControlQueueMessage(message); } if (continuedAsNewMessage != null) { transaction.SendControlQueueMessage(continuedAsNewMessage); } // TODO send timer messages in transaction transaction.AddOrchestrationStateToHistory(orchestrationId: orchestrationId, state: orchestrationState); transaction.RemoveItemsFromOrchestrationQueue(orchestrationId, workItem.NewMessages.Count); bool transactionSucceeded = await transaction.CommitTransactionAsync(); if (transactionSucceeded) { logMessage = $"Succeeded in transaction of finishing task orchestration item: execution id={workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId}, numMessagesProcessed: {workItem.NewMessages.Count}, numOutBoundMessages: {outboundMessages.Count}, numOrchMessages: {orchestratorMessages.Count}"; } else { logMessage = $"Failed in transaction of finishing task orchestration item: execution id={workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId}, numMessagesProcessed: {workItem.NewMessages.Count}, numOutBoundMessages: {outboundMessages.Count}, numOrchMessages: {orchestratorMessages.Count}"; } await this.logger.LogAsync(logMessage); }
/// <inheritdoc /> public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem) { // No need to renew, as current lock implementation lasts as long as process holds it. return(Task.CompletedTask); }
/// <inheritdoc /> public Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { return(Task.FromResult <object>(null)); }
/// <inheritdoc /> public Task CompleteTaskOrchestrationWorkItemAsync( TaskOrchestrationWorkItem workItem, OrchestrationRuntimeState newOrchestrationRuntimeState, IList <TaskMessage> outboundMessages, IList <TaskMessage> orchestratorMessages, IList <TaskMessage> workItemTimerMessages, TaskMessage continuedAsNewMessage, OrchestrationState state) { lock (this.thisLock) { this.orchestratorQueue.CompleteSession( workItem.InstanceId, newOrchestrationRuntimeState != null ? this.SerializeOrchestrationRuntimeState(newOrchestrationRuntimeState) : null, orchestratorMessages, continuedAsNewMessage ); if (outboundMessages != null) { foreach (TaskMessage m in outboundMessages) { // AFFANDAR : TODO : make async this.workerQueue.SendMessageAsync(m); } } if (workItemTimerMessages != null) { foreach (TaskMessage m in workItemTimerMessages) { this.timerMessages.Add(m); } } if (state != null) { Dictionary <string, OrchestrationState> ed; if (!this.instanceStore.TryGetValue(workItem.InstanceId, out ed)) { ed = new Dictionary <string, OrchestrationState>(); this.instanceStore[workItem.InstanceId] = ed; } ed[workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId] = state; // signal any waiters waiting on instanceid_executionid or just the latest instanceid_ TaskCompletionSource <OrchestrationState> tcs = null; TaskCompletionSource <OrchestrationState> tcs1 = null; if (state.OrchestrationStatus == OrchestrationStatus.Running || state.OrchestrationStatus == OrchestrationStatus.Pending) { return(Task.FromResult(0)); } string key = workItem.OrchestrationRuntimeState.OrchestrationInstance.InstanceId + "_" + workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId; string key1 = workItem.OrchestrationRuntimeState.OrchestrationInstance.InstanceId + "_"; var tasks = new List <Task>(); if (this.orchestrationWaiters.TryGetValue(key, out tcs)) { tasks.Add(Task.Run(() => tcs.TrySetResult(state))); } // for instanceid level waiters, we will not consider continueasnew as a terminal state because // the high level orch is still ongoing if (state.OrchestrationStatus != OrchestrationStatus.ContinuedAsNew && this.orchestrationWaiters.TryGetValue(key1, out tcs1)) { tasks.Add(Task.Run(() => tcs1.TrySetResult(state))); } if (tasks.Count > 0) { Task.WaitAll(tasks.ToArray()); } } } return(Task.FromResult(0)); }
/// <inheritdoc /> public Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) { this.orchestratorQueue.AbandonSession(workItem.InstanceId); return(Task.FromResult <object>(null)); }
public override async Task CompleteTaskOrchestrationWorkItemAsync( TaskOrchestrationWorkItem workItem, OrchestrationRuntimeState newRuntimeState, IList <TaskMessage> outboundMessages, IList <TaskMessage> orchestratorMessages, IList <TaskMessage> timerMessages, TaskMessage continuedAsNewMessage, OrchestrationState orchestrationState) { ExtendedOrchestrationWorkItem currentWorkItem = (ExtendedOrchestrationWorkItem)workItem; this.traceHelper.CheckpointStarting(orchestrationState); Stopwatch sw = Stopwatch.StartNew(); using SqlConnection connection = await this.GetAndOpenConnectionAsync(); using SqlCommand command = this.GetSprocCommand(connection, "dt._CheckpointOrchestration"); OrchestrationInstance instance = newRuntimeState.OrchestrationInstance !; IList <HistoryEvent> newEvents = newRuntimeState.NewEvents; IList <HistoryEvent> allEvents = newRuntimeState.Events; int nextSequenceNumber = allEvents.Count - newEvents.Count; command.Parameters.Add("@InstanceID", SqlDbType.VarChar, size: 100).Value = instance.InstanceId; command.Parameters.Add("@ExecutionID", SqlDbType.VarChar, size: 50).Value = instance.ExecutionId; command.Parameters.Add("@RuntimeStatus", SqlDbType.VarChar, size: 30).Value = orchestrationState.OrchestrationStatus.ToString(); command.Parameters.Add("@CustomStatusPayload", SqlDbType.VarChar).Value = orchestrationState.Status ?? SqlString.Null; currentWorkItem.EventPayloadMappings.Add(outboundMessages); currentWorkItem.EventPayloadMappings.Add(orchestratorMessages); command.Parameters.AddMessageIdParameter("@DeletedEvents", workItem.NewMessages); command.Parameters.AddOrchestrationEventsParameter( "@NewOrchestrationEvents", orchestratorMessages, timerMessages, continuedAsNewMessage, currentWorkItem.EventPayloadMappings); command.Parameters.AddTaskEventsParameter( "@NewTaskEvents", outboundMessages, currentWorkItem.EventPayloadMappings); command.Parameters.AddHistoryEventsParameter( "@NewHistoryEvents", newEvents, instance, nextSequenceNumber, currentWorkItem.EventPayloadMappings); try { await SqlUtils.ExecuteNonQueryAsync(command, this.traceHelper, instance.InstanceId); } catch (SqlException e) when(SqlUtils.IsUniqueKeyViolation(e)) { this.traceHelper.DuplicateExecutionDetected(instance, orchestrationState.Name); return; } // notify pollers that new messages may be available if (outboundMessages.Count > 0) { this.activityBackoffHelper.Reset(); } if (orchestratorMessages.Count > 0 || timerMessages.Count > 0) { this.orchestrationBackoffHelper.Reset(); } this.traceHelper.CheckpointCompleted(orchestrationState, sw); }
/// <inheritdoc /> public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem) { workItem.LockedUntilUtc = workItem.LockedUntilUtc.AddMinutes(5); return(Task.FromResult(0)); }
// We abandon work items by just letting their locks expire. The benefit of this "lazy" approach is that it // removes the need for a DB access and also ensures that a work-item can't spam the error logs in a tight loop. public override Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) => Task.CompletedTask;