public bool TryGetExistingSession(string instanceId, out OrchestrationSession session)
{
lock (this.messageAndSessionLock)
{
return(this.activeOrchestrationSessions.TryGetValue(instanceId, out session));
}
}
public bool TryReleaseSession(string instanceId, CancellationToken cancellationToken, out OrchestrationSession session)
{
// Taking this lock ensures we don't add new messages to a session we're about to release.
lock (this.messageAndSessionLock)
{
// Release is local/in-memory only because instances are affinitized to queues and this
// node already holds the lease for the target control queue.
if (this.activeOrchestrationSessions.TryGetValue(instanceId, out session) &&
this.activeOrchestrationSessions.Remove(instanceId))
{
// Put any unprocessed messages back into the pending buffer.
this.AddMessageToPendingOrchestration(
session.ControlQueue,
session.PendingMessages.Concat(session.DeferredMessages),
session.TraceActivityId,
cancellationToken);
return(true);
}
else
{
this.settings.Logger.AssertFailure(
this.storageAccountName,
this.settings.TaskHubName,
$"{nameof(TryReleaseSession)}: Session for instance {instanceId} was not found!");
return(false);
}
}
}
public async Task <OrchestrationSession> GetNextSessionAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
// This call will block until:
// 1) a batch of messages has been received for a particular instance and
// 2) the history for that instance has been fetched
LinkedListNode <PendingMessageBatch> node = await this.readyForProcessingQueue.DequeueAsync(cancellationToken);
lock (this.messageAndSessionLock)
{
PendingMessageBatch nextBatch = node.Value;
this.pendingOrchestrationMessageBatches.Remove(node);
if (!this.activeOrchestrationSessions.TryGetValue(nextBatch.OrchestrationInstanceId, out var existingSession))
{
OrchestrationInstance instance = nextBatch.OrchestrationState.OrchestrationInstance ??
new OrchestrationInstance
{
InstanceId = nextBatch.OrchestrationInstanceId,
ExecutionId = nextBatch.OrchestrationExecutionId,
};
Guid traceActivityId = AzureStorageOrchestrationService.StartNewLogicalTraceScope(useExisting: true);
OrchestrationSession session = new OrchestrationSession(
this.settings,
this.storageAccountName,
instance,
nextBatch.ControlQueue,
nextBatch.Messages,
nextBatch.OrchestrationState,
nextBatch.ETag,
nextBatch.LastCheckpointTime,
this.settings.ExtendedSessionIdleTimeout,
traceActivityId);
this.activeOrchestrationSessions.Add(instance.InstanceId, session);
return(session);
}
else if (nextBatch.OrchestrationExecutionId == existingSession.Instance.ExecutionId)
{
// there is already an active session with the same execution id.
// The session might be waiting for more messages. If it is, signal them.
existingSession.AddOrReplaceMessages(node.Value.Messages);
}
else
{
// A message arrived for a different generation of an existing orchestration instance.
// Put it back into the ready queue so that it can be processed once the current generation
// is done executing.
if (this.readyForProcessingQueue.Count == 0)
{
// To avoid a tight dequeue loop, delay for a bit before putting this node back into the queue.
// This is only necessary when the queue is empty. The main dequeue thread must not be blocked
// by this delay, which is why we use Task.Delay(...).ContinueWith(...) instead of await.
Task.Delay(millisecondsDelay: 200).ContinueWith(_ =>
{
lock (this.messageAndSessionLock)
{
this.pendingOrchestrationMessageBatches.AddLast(node);
this.readyForProcessingQueue.Enqueue(node);
}
});
}
else
{
this.pendingOrchestrationMessageBatches.AddLast(node);
this.readyForProcessingQueue.Enqueue(node);
}
}
}
}
return(null);
}
internal void AddMessageToPendingOrchestration(
ControlQueue controlQueue,
IEnumerable <MessageData> queueMessages,
Guid traceActivityId,
CancellationToken cancellationToken)
{
// Conditions to consider:
// 1. Do we need to create a new orchestration session or does one already exist?
// 2. Do we already have a copy of this message?
// 3. Do we need to add messages to a currently executing orchestration?
lock (this.messageAndSessionLock)
{
LinkedListNode <PendingMessageBatch> node;
var existingSessionMessages = new Dictionary <OrchestrationSession, List <MessageData> >();
foreach (MessageData data in queueMessages)
{
string instanceId = data.TaskMessage.OrchestrationInstance.InstanceId;
string executionId = data.TaskMessage.OrchestrationInstance.ExecutionId;
if (this.activeOrchestrationSessions.TryGetValue(instanceId, out OrchestrationSession session))
{
// If the target orchestration is already running we add the message to the session
// directly rather than adding it to the linked list. A null executionId value
// means that this is a management operation, like RaiseEvent or Terminate, which
// should be delivered to the current session.
if (executionId == null || session.Instance.ExecutionId == executionId)
{
List <MessageData> pendingMessages;
if (!existingSessionMessages.TryGetValue(session, out pendingMessages))
{
pendingMessages = new List <MessageData>();
existingSessionMessages.Add(session, pendingMessages);
}
pendingMessages.Add(data);
}
else if (data.TaskMessage.Event.Timestamp < session.RuntimeState.CreatedTime)
{
// This message was created for a previous generation of this instance.
// This is common for canceled timer fired events in ContinueAsNew scenarios.
session.DiscardMessage(data);
}
else
{
// Most likely this message was created for a new generation of the current
// instance. This can happen if a ContinueAsNew message arrives before the current
// session finished unloading. Defer the message so that it can be processed later.
session.DeferMessage(data);
}
continue;
}
// Walk backwards through the list of batches until we find one with a matching Instance ID.
// This is assumed to be more efficient than walking forward if most messages arrive in the queue in groups.
PendingMessageBatch targetBatch = null;
node = this.pendingOrchestrationMessageBatches.Last;
while (node != null)
{
PendingMessageBatch batch = node.Value;
if (batch.OrchestrationInstanceId == instanceId)
{
if (executionId == null || batch.OrchestrationExecutionId == executionId)
{
targetBatch = batch;
break;
}
else if (batch.OrchestrationExecutionId == null)
{
targetBatch = batch;
batch.OrchestrationExecutionId = executionId;
break;
}
}
node = node.Previous;
}
if (targetBatch == null)
{
targetBatch = new PendingMessageBatch(controlQueue, instanceId, executionId);
node = this.pendingOrchestrationMessageBatches.AddLast(targetBatch);
// Before the batch of messages can be processed, we need to download the latest execution state.
// This is done beforehand in the background as a performance optimization.
this.ScheduleOrchestrationStatePrefetch(node, traceActivityId, cancellationToken);
}
// New messages are added; duplicate messages are replaced
targetBatch.Messages.AddOrReplace(data);
}
// The session might be waiting for more messages. If it is, signal them.
foreach (var pair in existingSessionMessages)
{
OrchestrationSession session = pair.Key;
List <MessageData> newMessages = pair.Value;
// New messages are added; duplicate messages are replaced
session.AddOrReplaceMessages(newMessages);
}
}
}
void AddMessageToPendingOrchestration(
IEnumerable <MessageData> queueMessages,
Guid traceActivityId,
CancellationToken cancellationToken)
{
// Conditions to consider:
// 1. Do we need to create a new orchestration session or does one already exist?
// 2. Do we already have a copy of this message?
// 3. Do we need to add messages to a currently executing orchestration?
lock (this.messageAndSessionLock)
{
LinkedListNode <PendingMessageBatch> node;
var existingSessionMessages = new Dictionary <OrchestrationSession, List <MessageData> >();
foreach (MessageData data in queueMessages)
{
string instanceId = data.TaskMessage.OrchestrationInstance.InstanceId;
string executionId = data.TaskMessage.OrchestrationInstance.ExecutionId;
// If the target orchestration already running we add the message to the session
// directly rather than adding it to the linked list. A null executionId value
// means that this is a management operation, like RaiseEvent or Terminate, which
// should be delivered to the current session.
if (this.activeOrchestrationSessions.TryGetValue(instanceId, out OrchestrationSession session) &&
(executionId == null || session.Instance.ExecutionId == executionId))
{
List <MessageData> pendingMessages;
if (!existingSessionMessages.TryGetValue(session, out pendingMessages))
{
pendingMessages = new List <MessageData>();
existingSessionMessages.Add(session, pendingMessages);
}
pendingMessages.Add(data);
continue;
}
// Walk backwards through the list of batches until we find one with a matching Instance ID.
// This is assumed to be more efficient than walking forward if most messages arrive in the queue in groups.
PendingMessageBatch targetBatch = null;
node = this.pendingOrchestrationMessageBatches.Last;
while (node != null)
{
PendingMessageBatch batch = node.Value;
if (batch.OrchestrationInstanceId == instanceId &&
(executionId == null || batch.OrchestrationExecutionId == executionId))
{
targetBatch = batch;
break;
}
node = node.Previous;
}
if (targetBatch == null)
{
targetBatch = new PendingMessageBatch(instanceId, executionId);
node = this.pendingOrchestrationMessageBatches.AddLast(targetBatch);
// Before the batch of messages can be processed, we need to download the latest execution state.
// This is done on another background thread so that it won't slow down dequeuing.
this.ScheduleOrchestrationStatePrefetch(node, traceActivityId, cancellationToken);
}
if (targetBatch.Messages.AddOrReplace(data))
{
// Added. This is the normal path.
this.stats.PendingOrchestratorMessages.Increment();
}
else
{
// Replaced. This happens if the visibility timeout of a message expires while it
// is still sitting here in memory.
AnalyticsEventSource.Log.DuplicateMessageDetected(
this.storageAccountName,
this.settings.TaskHubName,
data.OriginalQueueMessage.Id,
data.OriginalQueueMessage.DequeueCount,
Utils.ExtensionVersion);
}
}
// The session might be waiting for more messages. If it is, signal them.
foreach (var pair in existingSessionMessages)
{
OrchestrationSession session = pair.Key;
List <MessageData> newMessages = pair.Value;
IEnumerable <MessageData> replacements = session.AddOrReplaceMessages(newMessages);
foreach (MessageData replacementMessage in replacements)
{
AnalyticsEventSource.Log.DuplicateMessageDetected(
this.storageAccountName,
this.settings.TaskHubName,
replacementMessage.OriginalQueueMessage.Id,
replacementMessage.OriginalQueueMessage.DequeueCount,
Utils.ExtensionVersion);
}
}
}
}
internal void AddMessageToPendingOrchestration(
ControlQueue controlQueue,
IEnumerable <MessageData> queueMessages,
Guid traceActivityId,
CancellationToken cancellationToken)
{
// Conditions to consider:
// 1. Do we need to create a new orchestration session or does one already exist?
// 2. Do we already have a copy of this message?
// 3. Do we need to add messages to a currently executing orchestration?
lock (this.messageAndSessionLock)
{
var existingSessionMessages = new Dictionary <OrchestrationSession, List <MessageData> >();
foreach (MessageData data in queueMessages)
{
// The instanceID identifies the orchestration across replays and ContinueAsNew generations.
// The executionID identifies a generation of an orchestration instance, doesn't change across replays.
string instanceId = data.TaskMessage.OrchestrationInstance.InstanceId;
string executionId = data.TaskMessage.OrchestrationInstance.ExecutionId;
// If the target orchestration is already in memory, we can potentially add the message to the session directly
// rather than adding it to the pending list. This behavior applies primarily when extended sessions are enabled.
// We can't do this for ExecutionStarted messages - those must *always* go to the pending list since they are for
// creating entirely new orchestration instances.
if (data.TaskMessage.Event.EventType != EventType.ExecutionStarted &&
this.activeOrchestrationSessions.TryGetValue(instanceId, out OrchestrationSession session))
{
// A null executionId value means that this is a management operation, like RaiseEvent or Terminate, which
// should be delivered to the current session.
if (executionId == null || session.Instance.ExecutionId == executionId)
{
List <MessageData> pendingMessages;
if (!existingSessionMessages.TryGetValue(session, out pendingMessages))
{
pendingMessages = new List <MessageData>();
existingSessionMessages.Add(session, pendingMessages);
}
pendingMessages.Add(data);
continue;
}
// Looks like this message is for another generation of the active orchestration. Let it fall
// into the pending list below. If it's a message for an older generation, it will be eventually
// discarded after we discover that we have no state associated with its execution ID. This is
// most common in scenarios involving durable timers and ContinueAsNew. Otherwise, this message
// will be processed after the current session unloads.
}
PendingMessageBatch?targetBatch = null; // batch for the current instanceID-executionID pair
// Unless the message is an ExecutionStarted event, we attempt to assign the current message to an
// existing batch by walking backwards through the list of batches until we find one with a matching InstanceID.
// This is assumed to be more efficient than walking forward if most messages arrive in the queue in groups.
LinkedListNode <PendingMessageBatch> node = this.pendingOrchestrationMessageBatches.Last;
while (node != null && data.TaskMessage.Event.EventType != EventType.ExecutionStarted)
{
PendingMessageBatch batch = node.Value;
if (batch.OrchestrationInstanceId == instanceId)
{
if (executionId == null || batch.OrchestrationExecutionId == executionId)
{
targetBatch = batch;
break;
}
else if (batch.OrchestrationExecutionId == null)
{
targetBatch = batch;
batch.OrchestrationExecutionId = executionId;
break;
}
}
node = node.Previous;
}
// If there is no batch for this instanceID-executionID pair, create one
if (targetBatch == null)
{
targetBatch = new PendingMessageBatch(controlQueue, instanceId, executionId);
node = this.pendingOrchestrationMessageBatches.AddLast(targetBatch);
// Before the batch of messages can be processed, we need to download the latest execution state.
// This is done beforehand in the background as a performance optimization.
Task.Run(() => this.ScheduleOrchestrationStatePrefetch(node, traceActivityId, cancellationToken));
}
// New messages are added; duplicate messages are replaced
targetBatch.Messages.AddOrReplace(data);
}
// The session might be waiting for more messages. If it is, signal them.
foreach (var pair in existingSessionMessages)
{
OrchestrationSession session = pair.Key;
List <MessageData> newMessages = pair.Value;
// New messages are added; duplicate messages are replaced
session.AddOrReplaceMessages(newMessages);
}
}
}
