public async Task <PersistentSession> AcceptSessionAsync(TimeSpan receiveTimeout)
{
if (!IsStopped())
{
bool newItemsBeforeTimeout = true;
while (newItemsBeforeTimeout)
{
if (this.fetchQueue.TryDequeue(out string returnInstanceId))
{
try
{
return(await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var txn = this.StateManager.CreateTransaction())
{
var existingValue = await this.Store.TryGetValueAsync(txn, returnInstanceId);
if (existingValue.HasValue)
{
if (this.lockedSessions.TryUpdate(returnInstanceId, newValue: LockState.Locked, comparisonValue: LockState.InFetchQueue))
{
ServiceFabricProviderEventSource.Tracing.TraceMessage(returnInstanceId, "Session Locked Accepted");
return existingValue.Value;
}
else
{
var errorMessage = $"Internal Server Error : Unexpected to dequeue the session {returnInstanceId} which was already locked before";
ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition(errorMessage);
throw new Exception(errorMessage);
}
}
else
{
var errorMessage = $"Internal Server Error: Did not find the session object in reliable dictionary while having the session {returnInstanceId} in memory";
ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition(errorMessage);
throw new Exception(errorMessage);
}
}
}, uniqueActionIdentifier : $"{nameof(SessionProvider)}.{nameof(AcceptSessionAsync)}, SessionId : {returnInstanceId}"));
}
catch (Exception)
{
this.fetchQueue.Enqueue(returnInstanceId);
throw;
}
}
newItemsBeforeTimeout = await WaitForItemsAsync(receiveTimeout);
}
}
return(null);
}
public async Task AppendMessageAsync(TaskMessageItem newMessage)
{
await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var txn = this.StateManager.CreateTransaction())
{
await this.AppendMessageAsync(txn, newMessage);
await txn.CommitAsync();
}
}, uniqueActionIdentifier : $"Orchestration = '{newMessage.TaskMessage.OrchestrationInstance}', Action = '{nameof(SessionProvider)}.{nameof(AppendMessageAsync)}'");
this.TryEnqueueSession(newMessage.TaskMessage.OrchestrationInstance);
}
public async Task DropSession(ITransaction txn, OrchestrationInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException(nameof(instance));
}
await this.Store.TryRemoveAsync(txn, instance.InstanceId);
this.sessionMessageProviders.TryRemove(instance, out SessionMessageProvider _);
var noWait = RetryHelper.ExecuteWithRetryOnTransient(() => this.StateManager.RemoveAsync(GetSessionMessagesDictionaryName(instance)),
uniqueActionIdentifier: $"Orchestration = '{instance}', Action = 'DropSessionMessagesDictionaryBackgroundTask'");
}
public async Task <PersistentSession> GetSession(string instanceId)
{
await EnsureStoreInitialized();
return(await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var txn = this.StateManager.CreateTransaction())
{
var value = await this.Store.TryGetValueAsync(txn, instanceId);
if (value.HasValue)
{
return value.Value;
}
}
return null;
}, uniqueActionIdentifier : $"Orchestration InstanceId = {instanceId}, Action = {nameof(SessionProvider)}.{nameof(GetSession)}"));
}
protected Task <Message <TKey, TValue> > GetValueAsync(TKey key)
{
return(RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var tx = this.StateManager.CreateTransaction())
{
var result = await this.Store.TryGetValueAsync(tx, key);
if (result.HasValue)
{
return new Message <TKey, TValue>(key, result.Value);
}
var errorMessage = $"Internal Server Error: Did not find an item in reliable dictionary while having the item key {key} in memory";
ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition(errorMessage);
throw new Exception(errorMessage);
}
}, uniqueActionIdentifier: $"Key = {key}, Action = MessageProviderBase.GetValueAsync, StoreName : {this.storeName}"));
}
public async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage)
{
if (creationMessage.Event is ExecutionStartedEvent executionStarted && executionStarted.ScheduledStartTime.HasValue)
{
throw new NotSupportedException("Service Fabric storage provider for Durable Tasks currently does not support scheduled starts");
}
creationMessage.OrchestrationInstance.InstanceId.EnsureValidInstanceId();
ExecutionStartedEvent startEvent = creationMessage.Event as ExecutionStartedEvent;
if (startEvent == null)
{
await this.SendTaskOrchestrationMessageAsync(creationMessage);
return;
}
var instance = creationMessage.OrchestrationInstance;
var added = await RetryHelper.ExecuteWithRetryOnTransient <bool>(async() =>
{
using (var tx = this.stateManager.CreateTransaction())
{
if (await this.orchestrationProvider.TryAddSession(tx, new TaskMessageItem(creationMessage)))
{
await WriteExecutionStartedEventToInstanceStore(tx, startEvent);
await tx.CommitAsync();
return(true);
}
return(false);
}
}, uniqueActionIdentifier : $"Orchestration = '{instance}', Action = '{nameof(CreateTaskOrchestrationAsync)}'");
if (added)
{
string message = string.Format("Orchestration with instanceId : '{0}' and executionId : '{1}' is Created.", instance.InstanceId, instance.ExecutionId);
ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(instance.InstanceId, instance.ExecutionId, message);
this.orchestrationProvider.TryEnqueueSession(creationMessage.OrchestrationInstance);
}
else
{
throw new OrchestrationAlreadyExistsException($"An orchestration with id '{creationMessage.OrchestrationInstance.InstanceId}' is already running.");
}
}
public async Task <List <string> > GetExecutionIds(string instanceId)
{
List <string> executionIds = null;
await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var tx = this.stateManager.CreateTransaction())
{
var executionIdsValue = await this.executionIdStore.TryGetValueAsync(tx, instanceId);
if (executionIdsValue.HasValue)
{
executionIds = executionIdsValue.Value;
}
}
}, uniqueActionIdentifier : $"Orchestration Instance Id = {instanceId}, Action = {nameof(FabricOrchestrationInstanceStore)}.{nameof(GetExecutionIds)}");
return(executionIds);
}
protected Task EnumerateItems(Action <KeyValuePair <TKey, TValue> > itemAction)
{
return(RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var tx = this.StateManager.CreateTransaction())
{
var count = await this.Store.GetCountAsync(tx);
if (count > 0)
{
var enumerable = await this.Store.CreateEnumerableAsync(tx, EnumerationMode.Unordered);
using (var enumerator = enumerable.GetAsyncEnumerator())
{
while (await enumerator.MoveNextAsync(this.CancellationToken))
{
itemAction(enumerator.Current);
}
}
}
}
}, uniqueActionIdentifier: $"Action = MessageProviderBase.EnumerateItems, StoreName : {this.storeName}"));
}
public async Task CompleteTaskActivityWorkItemAsync(TaskActivityWorkItem workItem, TaskMessage responseMessage)
{
bool added = false;
await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
bool retryOnException;
do
{
try
{
added = false;
retryOnException = false;
using (var txn = this.stateManager.CreateTransaction())
{
await this.activitiesProvider.CompleteAsync(txn, workItem.Id);
added = await this.orchestrationProvider.TryAppendMessageAsync(txn, new TaskMessageItem(responseMessage));
await txn.CommitAsync();
}
}
catch (FabricReplicationOperationTooLargeException ex)
{
ServiceFabricProviderEventSource.Tracing.ExceptionInReliableCollectionOperations($"OrchestrationInstance = {responseMessage.OrchestrationInstance}, ActivityId = {workItem.Id}, Action = {nameof(CompleteTaskActivityWorkItemAsync)}", ex.ToString());
retryOnException = true;
var originalEvent = responseMessage.Event;
int taskScheduledId = GetTaskScheduledId(originalEvent);
string details = $"Fabric exception when trying to save activity result: {ex}. Consider reducing the serialization size of activity result to avoid the issue.";
responseMessage.Event = new TaskFailedEvent(originalEvent.EventId, taskScheduledId, ex.Message, details);
}
} while (retryOnException);
}, uniqueActionIdentifier : $"Orchestration = '{responseMessage.OrchestrationInstance}', ActivityId = '{workItem.Id}', Action = '{nameof(CompleteTaskActivityWorkItemAsync)}'");
if (added)
{
this.orchestrationProvider.TryEnqueueSession(responseMessage.OrchestrationInstance);
}
}
// Caller should ensure the workItem has reached terminal state.
private async Task HandleCompletedOrchestration(TaskOrchestrationWorkItem workItem)
{
await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var txn = this.stateManager.CreateTransaction())
{
await this.instanceStore.WriteEntitiesAsync(txn, new InstanceEntityBase[]
{
new OrchestrationStateInstanceEntity()
{
State = Utils.BuildOrchestrationState(workItem.OrchestrationRuntimeState)
}
});
// DropSession does 2 things (like mentioned in the comments above) - remove the row from sessions dictionary
// and delete the session messages dictionary. The second step is in a background thread and not part of transaction.
// However even if this transaction failed but we ended up deleting session messages dictionary, that's ok - at
// that time, it should be an empty dictionary and we would have updated the runtime session state to full completed
// state in the transaction from Complete method. So the subsequent attempt would be able to complete the session.
await this.orchestrationProvider.DropSession(txn, workItem.OrchestrationRuntimeState.OrchestrationInstance);
await txn.CommitAsync();
}
}, uniqueActionIdentifier : $"OrchestrationId = '{workItem.InstanceId}', Action = '{nameof(HandleCompletedOrchestration)}'");
this.instanceStore.OnOrchestrationCompleted(workItem.OrchestrationRuntimeState.OrchestrationInstance);
string message = string.Format("Orchestration with instanceId : '{0}' and executionId : '{1}' Finished with the status {2} and result {3} in {4} seconds.",
workItem.InstanceId,
workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId,
workItem.OrchestrationRuntimeState.OrchestrationStatus.ToString(),
workItem.OrchestrationRuntimeState.Output,
(workItem.OrchestrationRuntimeState.CompletedTime - workItem.OrchestrationRuntimeState.CreatedTime).TotalSeconds);
ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId,
workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId,
message);
}
public async Task CompleteTaskOrchestrationWorkItemAsync(
TaskOrchestrationWorkItem workItem,
OrchestrationRuntimeState newOrchestrationRuntimeState,
IList <TaskMessage> outboundMessages,
IList <TaskMessage> orchestratorMessages,
IList <TaskMessage> timerMessages,
TaskMessage continuedAsNewMessage,
OrchestrationState orchestrationState)
{
SessionInformation sessionInfo = GetSessionInfo(workItem.InstanceId);
ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId,
workItem.OrchestrationRuntimeState.OrchestrationInstance?.ExecutionId,
$"Current orchestration status: {workItem.OrchestrationRuntimeState.OrchestrationStatus}");
bool isComplete = this.IsOrchestrationComplete(workItem.OrchestrationRuntimeState.OrchestrationStatus);
IList <OrchestrationInstance> sessionsToEnqueue = null;
List <Message <Guid, TaskMessageItem> > scheduledMessages = null;
List <Message <string, TaskMessageItem> > activityMessages = null;
await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
bool retryOnException;
do
{
try
{
retryOnException = false;
sessionsToEnqueue = null;
scheduledMessages = null;
activityMessages = null;
using (var txn = this.stateManager.CreateTransaction())
{
if (outboundMessages?.Count > 0)
{
activityMessages = outboundMessages.Select(m => new Message <string, TaskMessageItem>(Guid.NewGuid().ToString(), new TaskMessageItem(m))).ToList();
await this.activitiesProvider.SendBatchBeginAsync(txn, activityMessages);
}
if (timerMessages?.Count > 0)
{
scheduledMessages = timerMessages.Select(m => new Message <Guid, TaskMessageItem>(Guid.NewGuid(), new TaskMessageItem(m))).ToList();
await this.scheduledMessagesProvider.SendBatchBeginAsync(txn, scheduledMessages);
}
if (orchestratorMessages?.Count > 0)
{
if (workItem.OrchestrationRuntimeState?.ParentInstance != null)
{
sessionsToEnqueue = await this.orchestrationProvider.TryAppendMessageBatchAsync(txn, orchestratorMessages.Select(tm => new TaskMessageItem(tm)));
}
else
{
await this.orchestrationProvider.AppendMessageBatchAsync(txn, orchestratorMessages.Select(tm => new TaskMessageItem(tm)));
sessionsToEnqueue = orchestratorMessages.Select(m => m.OrchestrationInstance).ToList();
}
}
if (continuedAsNewMessage != null)
{
await this.orchestrationProvider.AppendMessageAsync(txn, new TaskMessageItem(continuedAsNewMessage));
sessionsToEnqueue = new List <OrchestrationInstance>()
{
continuedAsNewMessage.OrchestrationInstance
};
}
await this.orchestrationProvider.CompleteMessages(txn, sessionInfo.Instance, sessionInfo.LockTokens);
if (workItem.OrchestrationRuntimeState.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
{
await HandleCompletedOrchestration(workItem);
}
// When an orchestration is completed, we need to drop the session which involves 2 steps (1) Removing the row from sessions
// (2) Dropping the session messages dictionary. The second step is done in background thread for performance so is not
// part of transaction. Since it will happen outside the trasanction, if this transaction fails for some reason and we dropped
// the session as part of this transaction, we wouldn't have updated the session state but would have lost the messages
// in the session messages dictionary which are needed for state to reach complete state (when the orchestration is picked up again in next fetch).
// So we don't want to drop session as part of this transaction.
// Instead, we drop the session as part of a subsequent different transaction.
// However, framework passes us 'null' value for 'newOrchestrationRuntimeState' when orchestration is completed and
// if we updated the session state to null and this transaction succeded, and a node failures occurs and we
// never call the subsequent transaction, we will lose the runtime state of orchestration and never will be able to
// mark it as complete even if it is. So we use the work item's runtime state when 'newOrchestrationRuntimeState' is null
// so that the latest state is what is stored for the session.
// As part of next transaction, we are going to remove the row anyway for the session and it doesn't matter to update it to 'null'.
await this.orchestrationProvider.UpdateSessionState(txn, newOrchestrationRuntimeState.OrchestrationInstance, newOrchestrationRuntimeState ?? workItem.OrchestrationRuntimeState);
// We skip writing to instanceStore when orchestration reached terminal state to avoid a minor timing issue that
// wait for an orchestration completes but another orchestration with the same name cannot be started immediately
// because the session is still in store. We update the instance store on orchestration completion and drop the
// session as part of the next atomic transaction.
if (this.instanceStore != null && orchestrationState != null && !isComplete)
{
await this.instanceStore.WriteEntitiesAsync(txn, new InstanceEntityBase[]
{
new OrchestrationStateInstanceEntity()
{
State = orchestrationState
}
});
}
await txn.CommitAsync();
}
}
catch (FabricReplicationOperationTooLargeException ex)
{
ServiceFabricProviderEventSource.Tracing.ExceptionInReliableCollectionOperations($"OrchestrationInstance = {sessionInfo.Instance}, Action = {nameof(CompleteTaskOrchestrationWorkItemAsync)}", ex.ToString());
retryOnException = true;
newOrchestrationRuntimeState = null;
outboundMessages = null;
timerMessages = null;
orchestratorMessages = null;
if (orchestrationState != null)
{
orchestrationState.OrchestrationStatus = OrchestrationStatus.Failed;
orchestrationState.Output = $"Fabric exception when trying to process orchestration: {ex}. Investigate and consider reducing the serialization size of orchestration inputs/outputs/overall length to avoid the issue.";
}
}
} while (retryOnException);
}, uniqueActionIdentifier : $"OrchestrationId = '{workItem.InstanceId}', Action = '{nameof(CompleteTaskOrchestrationWorkItemAsync)}'");
if (activityMessages != null)
{
this.activitiesProvider.SendBatchComplete(activityMessages);
}
if (scheduledMessages != null)
{
this.scheduledMessagesProvider.SendBatchComplete(scheduledMessages);
}
if (sessionsToEnqueue != null)
{
foreach (var instance in sessionsToEnqueue)
{
this.orchestrationProvider.TryEnqueueSession(instance);
}
}
if (isComplete)
{
await HandleCompletedOrchestration(workItem);
}
}
// Since this method is started as part of StartAsync, the other stores maynot be immediately initialized
// by the time this invokes operations on those stores. But that would be a transient error and the next
// iteration of processing should take care of making things right.
async Task ProcessScheduledMessages()
{
while (!IsStopped())
{
try
{
var currentTime = DateTime.UtcNow;
var nextCheck = currentTime + TimeSpan.FromSeconds(1);
var builder = this.inMemorySet.ToBuilder();
List <Message <Guid, TaskMessageItem> > activatedMessages = new List <Message <Guid, TaskMessageItem> >();
while (builder.Count > 0)
{
var firstPendingMessage = builder.Min;
var timerEvent = firstPendingMessage.Value.TaskMessage.Event as TimerFiredEvent;
if (timerEvent == null)
{
throw new Exception("Internal Server Error : Ended up adding non TimerFiredEvent TaskMessage as scheduled message");
}
if (timerEvent.FireAt <= currentTime)
{
activatedMessages.Add(firstPendingMessage);
builder.Remove(firstPendingMessage);
}
else
{
nextCheck = timerEvent.FireAt;
break;
}
}
if (IsStopped())
{
break;
}
if (activatedMessages.Count > 0)
{
var keys = activatedMessages.Select(m => m.Key);
var values = activatedMessages.Select(m => m.Value).ToList();
IList <OrchestrationInstance> modifiedSessions = null;
await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var tx = this.StateManager.CreateTransaction())
{
modifiedSessions = await this.sessionProvider.TryAppendMessageBatchAsync(tx, values);
await this.CompleteBatchAsync(tx, keys);
await tx.CommitAsync();
}
}, uniqueActionIdentifier : $"Action = '{nameof(ScheduledMessageProvider)}.{nameof(ProcessScheduledMessages)}'");
lock (@lock)
{
this.inMemorySet = this.inMemorySet.Except(activatedMessages);
}
if (modifiedSessions != null)
{
foreach (var sessionId in modifiedSessions)
{
this.sessionProvider.TryEnqueueSession(sessionId);
}
}
}
this.nextActivationCheck = nextCheck;
await WaitForItemsAsync(this.nextActivationCheck - DateTime.UtcNow);
}
catch (Exception e)
{
ServiceFabricProviderEventSource.Tracing.ExceptionWhileRunningBackgroundJob($"{nameof(ScheduledMessageProvider)}.{nameof(ProcessScheduledMessages)}", e.ToString());
await Task.Delay(TimeSpan.FromMilliseconds(100));
}
}
}
public async Task <OrchestrationStateInstanceEntity> GetOrchestrationStateAsync(string instanceId, string executionId)
{
await EnsureStoreInitializedAsync();
var queryKey = this.GetKey(instanceId, executionId);
var result = await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var txn = this.stateManager.CreateTransaction())
{
var state = await this.instanceStore.TryGetValueAsync(txn, queryKey);
if (state.HasValue)
{
return(new OrchestrationStateInstanceEntity()
{
State = state.Value,
});
}
}
return(null);
}, uniqueActionIdentifier : $"Orchestration Instance Id = {instanceId}, ExecutionId = {executionId}, Action = {nameof(FabricOrchestrationInstanceStore)}.{nameof(GetOrchestrationStateAsync)}:QueryInstanceStore");
if (result != null)
{
return(result);
}
// If querying for orchestration which completed an hour ago, we won't return the results.
var now = DateTime.UtcNow;
for (int i = 0; i < 2; i++)
{
var backupDictionaryName = GetDictionaryKeyFromTimeFormat(now - TimeSpan.FromHours(i));
var backupDictionary = await this.stateManager.TryGetAsync <IReliableDictionary <string, OrchestrationState> >(backupDictionaryName);
if (backupDictionary.HasValue)
{
result = await RetryHelper.ExecuteWithRetryOnTransient(async() =>
{
using (var txn = this.stateManager.CreateTransaction())
{
var state = await backupDictionary.Value.TryGetValueAsync(txn, queryKey);
if (state.HasValue)
{
return(new OrchestrationStateInstanceEntity()
{
State = state.Value,
});
}
}
return(null);
}, uniqueActionIdentifier : $"Orchestration Instance Id = {instanceId}, ExecutionId = {executionId}, Action = {nameof(FabricOrchestrationInstanceStore)}.{nameof(GetOrchestrationStateAsync)}:QueryBackupInstanceStore {backupDictionaryName}");
if (result != null)
{
return(result);
}
}
}
return(null);
}