/// <summary>
/// Logs that a work item message is being dropped and includes a reason.
/// </summary>
/// <param name="workItem">The work item that this message belonged to.</param>
/// <param name="message">The message being dropped.</param>
/// <param name="reason">The reason for dropping the message.</param>
internal void DroppingOrchestrationMessage(TaskOrchestrationWorkItem workItem, TaskMessage message, string reason)
{
if (this.IsStructuredLoggingEnabled)
{
this.WriteStructuredLog(new LogEvents.DiscardingMessage(workItem, message, reason));
}
}
// Note: Do not rely on cancellationToken parameter to this method because the top layer does not yet implement any cancellation.
public async Task <TaskOrchestrationWorkItem> LockNextTaskOrchestrationWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken)
{
var currentSession = await this.orchestrationProvider.AcceptSessionAsync(receiveTimeout);
if (currentSession == null)
{
return(null);
}
List <Message <Guid, TaskMessageItem> > newMessages;
try
{
newMessages = await this.orchestrationProvider.ReceiveSessionMessagesAsync(currentSession);
var currentRuntimeState = new OrchestrationRuntimeState(currentSession.SessionState);
var workItem = new TaskOrchestrationWorkItem()
{
NewMessages = newMessages.Select(m => m.Value.TaskMessage).ToList(),
InstanceId = currentSession.SessionId.InstanceId,
OrchestrationRuntimeState = currentRuntimeState
};
if (newMessages.Count == 0)
{
if (currentRuntimeState.ExecutionStartedEvent == null)
{
ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition($"Orchestration with no execution started event found: {currentSession.SessionId}");
return(null);
}
bool isComplete = this.IsOrchestrationComplete(currentRuntimeState.OrchestrationStatus);
if (isComplete)
{
await this.HandleCompletedOrchestration(workItem);
}
this.orchestrationProvider.TryUnlockSession(currentSession.SessionId, isComplete: isComplete);
return(null);
}
var sessionInfo = new SessionInformation()
{
Instance = currentSession.SessionId,
LockTokens = newMessages.Select(m => m.Key).ToList()
};
if (!this.sessionInfos.TryAdd(workItem.InstanceId, sessionInfo))
{
ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition($"{nameof(FabricOrchestrationService)}.{nameof(LockNextTaskOrchestrationWorkItemAsync)} : Multiple receivers processing the same session : {currentSession.SessionId.InstanceId}?");
}
return(workItem);
}
catch (Exception)
{
this.orchestrationProvider.TryUnlockSession(currentSession.SessionId, abandon: true);
throw;
}
}
/// <summary>
/// Helper method for logging the dropping of all messages associated with the specified work item.
/// </summary>
/// <param name="workItem">The work item being dropped.</param>
/// <param name="reason">The reason for dropping this work item.</param>
internal void DroppingOrchestrationWorkItem(TaskOrchestrationWorkItem workItem, string reason)
{
foreach (TaskMessage message in workItem.NewMessages)
{
this.DroppingOrchestrationMessage(workItem, message, reason);
}
}
/// <summary>
/// Logs that an orchestration work item message is being processed.
/// </summary>
/// <param name="workItem">The orchestration work item.</param>
/// <param name="message">The message being processed.</param>
internal void ProcessingOrchestrationMessage(TaskOrchestrationWorkItem workItem, TaskMessage message)
{
if (this.IsStructuredLoggingEnabled)
{
this.WriteStructuredLog(new LogEvents.ProcessingOrchestrationMessage(workItem, message));
}
}
// Called by the DTFx dispatcher thread
public async Task <IList <TaskMessage> > FetchNewOrchestrationMessagesAsync(
TaskOrchestrationWorkItem workItem)
{
if (!await this.messagesAvailableEvent.WaitAsync(this.idleTimeout))
{
return(null); // timed-out
}
this.StartNewLogicalTraceScope();
lock (this.nextMessageBatch)
{
this.CurrentMessageBatch = this.nextMessageBatch.ToList();
this.nextMessageBatch.Clear();
}
var messages = new List <TaskMessage>(this.CurrentMessageBatch.Count);
foreach (MessageData msg in this.CurrentMessageBatch)
{
this.TraceProcessingMessage(msg, isExtendedSession: true);
messages.Add(msg.TaskMessage);
}
return(messages);
}
public abstract Task CompleteTaskOrchestrationWorkItemAsync(
TaskOrchestrationWorkItem workItem,
OrchestrationRuntimeState newOrchestrationRuntimeState,
IList <TaskMessage> outboundMessages,
IList <TaskMessage> orchestratorMessages,
IList <TaskMessage> timerMessages,
TaskMessage continuedAsNewMessage,
OrchestrationState orchestrationState);
public Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
if (OrchestrationsServiceScopes.TryRemove(workItem.InstanceId, out var serviceScope))
{
serviceScope.Dispose();
}
return(_innerOrchestrationService.ReleaseTaskOrchestrationWorkItemAsync(workItem));
}
/// <inheritdoc />
public async Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
if (this.partitionOrchestrationManager == null || this.redisConnection == null)
{
await StartAsync();
}
// TODO: Handle differently then just releasing?
await ReleaseTaskOrchestrationWorkItemAsync(workItem);
}
/// <inheritdoc />
public async Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
if (this.partitionOrchestrationManager == null || this.redisConnection == null)
{
await StartAsync();
}
await this.logger.LogAsync($"Releasing lock on orchestration {workItem.InstanceId}");
this.partitionOrchestrationManager.ReleaseOrchestration(workItem.InstanceId);
}
public Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
bool isComplete = this.IsOrchestrationComplete(workItem.OrchestrationRuntimeState.OrchestrationStatus);
SessionInformation sessionInfo = TryRemoveSessionInfo(workItem.InstanceId);
if (sessionInfo != null)
{
this.orchestrationProvider.TryUnlockSession(sessionInfo.Instance, isComplete: isComplete);
}
return(Task.CompletedTask);
}
public Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
SessionInformation sessionInfo = TryRemoveSessionInfo(workItem.InstanceId);
if (sessionInfo == null)
{
ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition($"{nameof(AbandonTaskOrchestrationWorkItemAsync)} : Could not get a session info object while trying to abandon session {workItem.InstanceId}");
}
else
{
this.orchestrationProvider.TryUnlockSession(sessionInfo.Instance, abandon: true);
}
return(Task.CompletedTask);
}
public async Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem)
{
using (var dbContext = _dbContextFactory())
{
var session = workItem.Session as EFCoreOrchestrationSession;
var lockedUntilUTC = DateTime.UtcNow.Add(_options.OrchestrationLockTimeout);
dbContext.Instances.Attach(session.Instance);
session.Instance.LockedUntil = DateTime.UtcNow.Add(_options.OrchestrationLockTimeout);
await dbContext.SaveChangesAsync();
workItem.LockedUntilUtc = lockedUntilUTC;
}
}
public override async Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem)
{
using SqlConnection connection = await this.GetAndOpenConnectionAsync();
using SqlCommand command = this.GetSprocCommand(connection, "dt._RenewOrchestrationLocks");
DateTime lockExpiration = DateTime.UtcNow.Add(this.settings.WorkItemLockTimeout);
command.Parameters.Add("@InstanceID", SqlDbType.VarChar, size: 100).Value = workItem.InstanceId;
command.Parameters.Add("@LockExpiration", SqlDbType.DateTime2).Value = lockExpiration;
await SqlUtils.ExecuteNonQueryAsync(command, this.traceHelper, workItem.InstanceId);
workItem.LockedUntilUtc = lockExpiration;
}
public async Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
using (var dbContext = _dbContextFactory())
{
var session = workItem.Session as EFCoreOrchestrationSession;
if (!session.Released)
{
dbContext.Instances.Attach(session.Instance);
session.Instance.LockId = null;
session.Instance.LockedUntil = DateTime.UtcNow;
await dbContext.SaveChangesAsync();
session.Released = true;
}
}
}
public async Task <IList <TaskMessage> > FetchNewOrchestrationMessagesAsync(
TaskOrchestrationWorkItem workItem)
{
return(await BackoffPollingHelper.PollAsync(async() =>
{
using (var dbContext = _dbContextFactory())
{
var messages = await FetchNewMessagesAsync(dbContext);
await dbContext.SaveChangesAsync();
return messages;
}
},
x => x.Count > 0,
_options.FetchNewMessagesPollingTimeout,
_options.PollingInterval,
_stopCancellationToken));
}
/// <inheritdoc/>
public Task CompleteTaskOrchestrationWorkItemAsync(
TaskOrchestrationWorkItem workItem,
OrchestrationRuntimeState newOrchestrationRuntimeState,
IList <TaskMessage> outboundMessages,
IList <TaskMessage> orchestratorMessages,
IList <TaskMessage> timerMessages,
TaskMessage continuedAsNewMessage,
OrchestrationState orchestrationState)
{
return(this.GetOrchestrationService().CompleteTaskOrchestrationWorkItemAsync(
workItem,
newOrchestrationRuntimeState,
outboundMessages,
orchestratorMessages,
timerMessages,
continuedAsNewMessage,
orchestrationState));
}
public async Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
using (var dbContext = _dbContextFactory())
{
var session = workItem.Session as EFCoreOrchestrationSession;
if (session.Released)
{
throw new InvalidOperationException("Session was already released");
}
dbContext.Instances.Attach(session.Instance);
session.Instance.LockId = null;
// TODO: Exponential backoff
session.Instance.LockedUntil = DateTime.UtcNow.AddMinutes(1);
await dbContext.SaveChangesAsync();
session.Released = true;
}
}
/// <inheritdoc />
public async Task <TaskOrchestrationWorkItem> LockNextTaskOrchestrationWorkItemAsync(
TimeSpan receiveTimeout,
CancellationToken cancellationToken)
{
TaskSession taskSession = await this._OrchestratorQueue.AcceptSessionAsync(receiveTimeout,
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, this._CancellationTokenSource.Token).Token);
if (taskSession == null)
{
return(null);
}
var wi = new TaskOrchestrationWorkItem {
NewMessages = taskSession.Messages.ToList(),
InstanceId = taskSession.Id,
LockedUntilUtc = DateTime.UtcNow.AddMinutes(5),
OrchestrationRuntimeState =
DeserializeOrchestrationRuntimeState(taskSession.SessionState) ??
new OrchestrationRuntimeState(),
};
return(wi);
}
// Caller should ensure the workItem has reached terminal state.
private async Task HandleCompletedOrchestration(TaskOrchestrationWorkItem workItem)
{
await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var txn = this.stateManager.CreateTransaction())
{
await this.instanceStore.WriteEntitiesAsync(txn, new InstanceEntityBase[]
{
new OrchestrationStateInstanceEntity()
{
State = Utils.BuildOrchestrationState(workItem.OrchestrationRuntimeState)
}
});
// DropSession does 2 things (like mentioned in the comments above) - remove the row from sessions dictionary
// and delete the session messages dictionary. The second step is in a background thread and not part of transaction.
// However even if this transaction failed but we ended up deleting session messages dictionary, that's ok - at
// that time, it should be an empty dictionary and we would have updated the runtime session state to full completed
// state in the transaction from Complete method. So the subsequent attempt would be able to complete the session.
await this.orchestrationProvider.DropSession(txn, workItem.OrchestrationRuntimeState.OrchestrationInstance);
await txn.CommitAsync();
}
}, uniqueActionIdentifier : $"OrchestrationId = '{workItem.InstanceId}', Action = '{nameof(HandleCompletedOrchestration)}'");
this.instanceStore.OnOrchestrationCompleted(workItem.OrchestrationRuntimeState.OrchestrationInstance);
string message = string.Format("Orchestration with instanceId : '{0}' and executionId : '{1}' Finished with the status {2} and result {3} in {4} seconds.",
workItem.InstanceId,
workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId,
workItem.OrchestrationRuntimeState.OrchestrationStatus.ToString(),
workItem.OrchestrationRuntimeState.Output,
(workItem.OrchestrationRuntimeState.CompletedTime - workItem.OrchestrationRuntimeState.CreatedTime).TotalSeconds);
ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId,
workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId,
message);
}
public async Task <IList <TaskMessage> > FetchNewOrchestrationMessagesAsync(
TaskOrchestrationWorkItem workItem)
{
if (!await this.messagesAvailableEvent.WaitAsync(this.idleTimeout))
{
return(null);
}
this.CurrentMessageBatch = this.fetchMessagesCallback.Invoke(this.Instance);
if (this.CurrentMessageBatch == null)
{
return(null);
}
var messages = new List <TaskMessage>(this.CurrentMessageBatch.Count);
foreach (MessageData msg in this.CurrentMessageBatch)
{
this.TraceProcessingMessage(msg, isExtendedSession: true);
messages.Add(msg.TaskMessage);
}
return(messages);
}
public async Task CompleteTaskOrchestrationWorkItemAsync(
TaskOrchestrationWorkItem workItem,
OrchestrationRuntimeState newOrchestrationRuntimeState,
IList <TaskMessage> outboundMessages,
IList <TaskMessage> orchestratorMessages,
IList <TaskMessage> timerMessages,
TaskMessage continuedAsNewMessage,
OrchestrationState orchestrationState)
{
SessionInformation sessionInfo = GetSessionInfo(workItem.InstanceId);
ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId,
workItem.OrchestrationRuntimeState.OrchestrationInstance?.ExecutionId,
$"Current orchestration status: {workItem.OrchestrationRuntimeState.OrchestrationStatus}");
bool isComplete = this.IsOrchestrationComplete(workItem.OrchestrationRuntimeState.OrchestrationStatus);
IList <OrchestrationInstance> sessionsToEnqueue = null;
List <Message <Guid, TaskMessageItem> > scheduledMessages = null;
List <Message <string, TaskMessageItem> > activityMessages = null;
await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
bool retryOnException;
do
{
try
{
retryOnException = false;
sessionsToEnqueue = null;
scheduledMessages = null;
activityMessages = null;
using (var txn = this.stateManager.CreateTransaction())
{
if (outboundMessages?.Count > 0)
{
activityMessages = outboundMessages.Select(m => new Message <string, TaskMessageItem>(Guid.NewGuid().ToString(), new TaskMessageItem(m))).ToList();
await this.activitiesProvider.SendBatchBeginAsync(txn, activityMessages);
}
if (timerMessages?.Count > 0)
{
scheduledMessages = timerMessages.Select(m => new Message <Guid, TaskMessageItem>(Guid.NewGuid(), new TaskMessageItem(m))).ToList();
await this.scheduledMessagesProvider.SendBatchBeginAsync(txn, scheduledMessages);
}
if (orchestratorMessages?.Count > 0)
{
if (workItem.OrchestrationRuntimeState?.ParentInstance != null)
{
sessionsToEnqueue = await this.orchestrationProvider.TryAppendMessageBatchAsync(txn, orchestratorMessages.Select(tm => new TaskMessageItem(tm)));
}
else
{
await this.orchestrationProvider.AppendMessageBatchAsync(txn, orchestratorMessages.Select(tm => new TaskMessageItem(tm)));
sessionsToEnqueue = orchestratorMessages.Select(m => m.OrchestrationInstance).ToList();
}
}
if (continuedAsNewMessage != null)
{
await this.orchestrationProvider.AppendMessageAsync(txn, new TaskMessageItem(continuedAsNewMessage));
sessionsToEnqueue = new List <OrchestrationInstance>()
{
continuedAsNewMessage.OrchestrationInstance
};
}
await this.orchestrationProvider.CompleteMessages(txn, sessionInfo.Instance, sessionInfo.LockTokens);
if (workItem.OrchestrationRuntimeState.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
{
await HandleCompletedOrchestration(workItem);
}
// When an orchestration is completed, we need to drop the session which involves 2 steps (1) Removing the row from sessions
// (2) Dropping the session messages dictionary. The second step is done in background thread for performance so is not
// part of transaction. Since it will happen outside the trasanction, if this transaction fails for some reason and we dropped
// the session as part of this transaction, we wouldn't have updated the session state but would have lost the messages
// in the session messages dictionary which are needed for state to reach complete state (when the orchestration is picked up again in next fetch).
// So we don't want to drop session as part of this transaction.
// Instead, we drop the session as part of a subsequent different transaction.
// However, framework passes us 'null' value for 'newOrchestrationRuntimeState' when orchestration is completed and
// if we updated the session state to null and this transaction succeded, and a node failures occurs and we
// never call the subsequent transaction, we will lose the runtime state of orchestration and never will be able to
// mark it as complete even if it is. So we use the work item's runtime state when 'newOrchestrationRuntimeState' is null
// so that the latest state is what is stored for the session.
// As part of next transaction, we are going to remove the row anyway for the session and it doesn't matter to update it to 'null'.
await this.orchestrationProvider.UpdateSessionState(txn, newOrchestrationRuntimeState.OrchestrationInstance, newOrchestrationRuntimeState ?? workItem.OrchestrationRuntimeState);
// We skip writing to instanceStore when orchestration reached terminal state to avoid a minor timing issue that
// wait for an orchestration completes but another orchestration with the same name cannot be started immediately
// because the session is still in store. We update the instance store on orchestration completion and drop the
// session as part of the next atomic transaction.
if (this.instanceStore != null && orchestrationState != null && !isComplete)
{
await this.instanceStore.WriteEntitiesAsync(txn, new InstanceEntityBase[]
{
new OrchestrationStateInstanceEntity()
{
State = orchestrationState
}
});
}
await txn.CommitAsync();
}
}
catch (FabricReplicationOperationTooLargeException ex)
{
ServiceFabricProviderEventSource.Tracing.ExceptionInReliableCollectionOperations($"OrchestrationInstance = {sessionInfo.Instance}, Action = {nameof(CompleteTaskOrchestrationWorkItemAsync)}", ex.ToString());
retryOnException = true;
newOrchestrationRuntimeState = null;
outboundMessages = null;
timerMessages = null;
orchestratorMessages = null;
if (orchestrationState != null)
{
orchestrationState.OrchestrationStatus = OrchestrationStatus.Failed;
orchestrationState.Output = $"Fabric exception when trying to process orchestration: {ex}. Investigate and consider reducing the serialization size of orchestration inputs/outputs/overall length to avoid the issue.";
}
}
} while (retryOnException);
}, uniqueActionIdentifier : $"OrchestrationId = '{workItem.InstanceId}', Action = '{nameof(CompleteTaskOrchestrationWorkItemAsync)}'");
if (activityMessages != null)
{
this.activitiesProvider.SendBatchComplete(activityMessages);
}
if (scheduledMessages != null)
{
this.scheduledMessagesProvider.SendBatchComplete(scheduledMessages);
}
if (sessionsToEnqueue != null)
{
foreach (var instance in sessionsToEnqueue)
{
this.orchestrationProvider.TryEnqueueSession(instance);
}
}
if (isComplete)
{
await HandleCompletedOrchestration(workItem);
}
}
public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem)
{
return(Task.CompletedTask);
}
/// <inheritdoc />
public async Task CompleteTaskOrchestrationWorkItemAsync(
TaskOrchestrationWorkItem workItem,
OrchestrationRuntimeState newOrchestrationRuntimeState,
IList <TaskMessage> outboundMessages,
IList <TaskMessage> orchestratorMessages,
IList <TaskMessage> timerMessages,
TaskMessage continuedAsNewMessage,
OrchestrationState orchestrationState)
{
if (this.partitionOrchestrationManager == null || this.redisConnection == null)
{
await StartAsync();
}
string orchestrationId = workItem.InstanceId;
RedisTransactionBuilder transaction = this.partitionOrchestrationManager.CreateExistingOrchestrationTransaction(orchestrationId);
List <string> events = newOrchestrationRuntimeState.Events.Select(histEvent => histEvent as TaskCompletedEvent)
.Where(taskCompletedEvent => taskCompletedEvent != null)
.OrderBy(task => task.TaskScheduledId)
.Select(TaskCompletedEvent => $"{{\"id\": {TaskCompletedEvent.TaskScheduledId}, \"Result\": {TaskCompletedEvent.Result}}}")
.ToList();
string logMessage = "Current events processed: " + string.Join(",", events);
await this.logger.LogAsync(logMessage);
transaction.SetOrchestrationRuntimeState(orchestrationId, newOrchestrationRuntimeState);
foreach (TaskMessage outboundMessage in outboundMessages)
{
transaction.SendActivityMessage(outboundMessage);
}
foreach (TaskMessage message in orchestratorMessages)
{
transaction.SendControlQueueMessage(message);
}
if (continuedAsNewMessage != null)
{
transaction.SendControlQueueMessage(continuedAsNewMessage);
}
// TODO send timer messages in transaction
transaction.AddOrchestrationStateToHistory(orchestrationId: orchestrationId,
state: orchestrationState);
transaction.RemoveItemsFromOrchestrationQueue(orchestrationId, workItem.NewMessages.Count);
bool transactionSucceeded = await transaction.CommitTransactionAsync();
if (transactionSucceeded)
{
logMessage = $"Succeeded in transaction of finishing task orchestration item: execution id={workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId}, numMessagesProcessed: {workItem.NewMessages.Count}, numOutBoundMessages: {outboundMessages.Count}, numOrchMessages: {orchestratorMessages.Count}";
}
else
{
logMessage = $"Failed in transaction of finishing task orchestration item: execution id={workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId}, numMessagesProcessed: {workItem.NewMessages.Count}, numOutBoundMessages: {outboundMessages.Count}, numOrchMessages: {orchestratorMessages.Count}";
}
await this.logger.LogAsync(logMessage);
}
/// <inheritdoc />
public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem)
{
// No need to renew, as current lock implementation lasts as long as process holds it.
return(Task.CompletedTask);
}
/// <inheritdoc />
public Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
return(Task.FromResult <object>(null));
}
/// <inheritdoc />
public Task CompleteTaskOrchestrationWorkItemAsync(
TaskOrchestrationWorkItem workItem,
OrchestrationRuntimeState newOrchestrationRuntimeState,
IList <TaskMessage> outboundMessages,
IList <TaskMessage> orchestratorMessages,
IList <TaskMessage> workItemTimerMessages,
TaskMessage continuedAsNewMessage,
OrchestrationState state)
{
lock (this.thisLock)
{
this.orchestratorQueue.CompleteSession(
workItem.InstanceId,
newOrchestrationRuntimeState != null ?
this.SerializeOrchestrationRuntimeState(newOrchestrationRuntimeState) : null,
orchestratorMessages,
continuedAsNewMessage
);
if (outboundMessages != null)
{
foreach (TaskMessage m in outboundMessages)
{
// AFFANDAR : TODO : make async
this.workerQueue.SendMessageAsync(m);
}
}
if (workItemTimerMessages != null)
{
foreach (TaskMessage m in workItemTimerMessages)
{
this.timerMessages.Add(m);
}
}
if (state != null)
{
Dictionary <string, OrchestrationState> ed;
if (!this.instanceStore.TryGetValue(workItem.InstanceId, out ed))
{
ed = new Dictionary <string, OrchestrationState>();
this.instanceStore[workItem.InstanceId] = ed;
}
ed[workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId] = state;
// signal any waiters waiting on instanceid_executionid or just the latest instanceid_
TaskCompletionSource <OrchestrationState> tcs = null;
TaskCompletionSource <OrchestrationState> tcs1 = null;
if (state.OrchestrationStatus == OrchestrationStatus.Running ||
state.OrchestrationStatus == OrchestrationStatus.Pending)
{
return(Task.FromResult(0));
}
string key = workItem.OrchestrationRuntimeState.OrchestrationInstance.InstanceId + "_" +
workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId;
string key1 = workItem.OrchestrationRuntimeState.OrchestrationInstance.InstanceId + "_";
var tasks = new List <Task>();
if (this.orchestrationWaiters.TryGetValue(key, out tcs))
{
tasks.Add(Task.Run(() => tcs.TrySetResult(state)));
}
// for instanceid level waiters, we will not consider continueasnew as a terminal state because
// the high level orch is still ongoing
if (state.OrchestrationStatus != OrchestrationStatus.ContinuedAsNew &&
this.orchestrationWaiters.TryGetValue(key1, out tcs1))
{
tasks.Add(Task.Run(() => tcs1.TrySetResult(state)));
}
if (tasks.Count > 0)
{
Task.WaitAll(tasks.ToArray());
}
}
}
return(Task.FromResult(0));
}
/// <inheritdoc />
public Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
this.orchestratorQueue.AbandonSession(workItem.InstanceId);
return(Task.FromResult <object>(null));
}
public override async Task CompleteTaskOrchestrationWorkItemAsync(
TaskOrchestrationWorkItem workItem,
OrchestrationRuntimeState newRuntimeState,
IList <TaskMessage> outboundMessages,
IList <TaskMessage> orchestratorMessages,
IList <TaskMessage> timerMessages,
TaskMessage continuedAsNewMessage,
OrchestrationState orchestrationState)
{
ExtendedOrchestrationWorkItem currentWorkItem = (ExtendedOrchestrationWorkItem)workItem;
this.traceHelper.CheckpointStarting(orchestrationState);
Stopwatch sw = Stopwatch.StartNew();
using SqlConnection connection = await this.GetAndOpenConnectionAsync();
using SqlCommand command = this.GetSprocCommand(connection, "dt._CheckpointOrchestration");
OrchestrationInstance instance = newRuntimeState.OrchestrationInstance !;
IList <HistoryEvent> newEvents = newRuntimeState.NewEvents;
IList <HistoryEvent> allEvents = newRuntimeState.Events;
int nextSequenceNumber = allEvents.Count - newEvents.Count;
command.Parameters.Add("@InstanceID", SqlDbType.VarChar, size: 100).Value = instance.InstanceId;
command.Parameters.Add("@ExecutionID", SqlDbType.VarChar, size: 50).Value = instance.ExecutionId;
command.Parameters.Add("@RuntimeStatus", SqlDbType.VarChar, size: 30).Value = orchestrationState.OrchestrationStatus.ToString();
command.Parameters.Add("@CustomStatusPayload", SqlDbType.VarChar).Value = orchestrationState.Status ?? SqlString.Null;
currentWorkItem.EventPayloadMappings.Add(outboundMessages);
currentWorkItem.EventPayloadMappings.Add(orchestratorMessages);
command.Parameters.AddMessageIdParameter("@DeletedEvents", workItem.NewMessages);
command.Parameters.AddOrchestrationEventsParameter(
"@NewOrchestrationEvents",
orchestratorMessages,
timerMessages,
continuedAsNewMessage,
currentWorkItem.EventPayloadMappings);
command.Parameters.AddTaskEventsParameter(
"@NewTaskEvents",
outboundMessages,
currentWorkItem.EventPayloadMappings);
command.Parameters.AddHistoryEventsParameter(
"@NewHistoryEvents",
newEvents,
instance,
nextSequenceNumber,
currentWorkItem.EventPayloadMappings);
try
{
await SqlUtils.ExecuteNonQueryAsync(command, this.traceHelper, instance.InstanceId);
}
catch (SqlException e) when(SqlUtils.IsUniqueKeyViolation(e))
{
this.traceHelper.DuplicateExecutionDetected(instance, orchestrationState.Name);
return;
}
// notify pollers that new messages may be available
if (outboundMessages.Count > 0)
{
this.activityBackoffHelper.Reset();
}
if (orchestratorMessages.Count > 0 || timerMessages.Count > 0)
{
this.orchestrationBackoffHelper.Reset();
}
this.traceHelper.CheckpointCompleted(orchestrationState, sw);
}
/// <inheritdoc />
public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem)
{
workItem.LockedUntilUtc = workItem.LockedUntilUtc.AddMinutes(5);
return(Task.FromResult(0));
}
// We abandon work items by just letting their locks expire. The benefit of this "lazy" approach is that it // removes the need for a DB access and also ensures that a work-item can't spam the error logs in a tight loop. public override Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) => Task.CompletedTask;
