public async Task <PersistentSession> AcceptSessionAsync(TimeSpan receiveTimeout)
        {
            if (!IsStopped())
            {
                bool newItemsBeforeTimeout = true;
                while (newItemsBeforeTimeout)
                {
                    if (this.fetchQueue.TryDequeue(out string returnInstanceId))
                    {
                        try
                        {
                            return(await RetryHelper.ExecuteWithRetryOnTransient(async() =>
                            {
                                using (var txn = this.StateManager.CreateTransaction())
                                {
                                    var existingValue = await this.Store.TryGetValueAsync(txn, returnInstanceId);

                                    if (existingValue.HasValue)
                                    {
                                        if (this.lockedSessions.TryUpdate(returnInstanceId, newValue: LockState.Locked, comparisonValue: LockState.InFetchQueue))
                                        {
                                            ServiceFabricProviderEventSource.Tracing.TraceMessage(returnInstanceId, "Session Locked Accepted");
                                            return existingValue.Value;
                                        }
                                        else
                                        {
                                            var errorMessage = $"Internal Server Error : Unexpected to dequeue the session {returnInstanceId} which was already locked before";
                                            ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition(errorMessage);
                                            throw new Exception(errorMessage);
                                        }
                                    }
                                    else
                                    {
                                        var errorMessage = $"Internal Server Error: Did not find the session object in reliable dictionary while having the session {returnInstanceId} in memory";
                                        ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition(errorMessage);
                                        throw new Exception(errorMessage);
                                    }
                                }
                            }, uniqueActionIdentifier : $"{nameof(SessionProvider)}.{nameof(AcceptSessionAsync)}, SessionId : {returnInstanceId}"));
                        }
                        catch (Exception)
                        {
                            this.fetchQueue.Enqueue(returnInstanceId);
                            throw;
                        }
                    }

                    newItemsBeforeTimeout = await WaitForItemsAsync(receiveTimeout);
                }
            }
            return(null);
        }
        public async Task AppendMessageAsync(TaskMessageItem newMessage)
        {
            await RetryHelper.ExecuteWithRetryOnTransient(async() =>
            {
                using (var txn = this.StateManager.CreateTransaction())
                {
                    await this.AppendMessageAsync(txn, newMessage);
                    await txn.CommitAsync();
                }
            }, uniqueActionIdentifier : $"Orchestration = '{newMessage.TaskMessage.OrchestrationInstance}', Action = '{nameof(SessionProvider)}.{nameof(AppendMessageAsync)}'");

            this.TryEnqueueSession(newMessage.TaskMessage.OrchestrationInstance);
        }
        public async Task DropSession(ITransaction txn, OrchestrationInstance instance)
        {
            if (instance == null)
            {
                throw new ArgumentNullException(nameof(instance));
            }

            await this.Store.TryRemoveAsync(txn, instance.InstanceId);

            this.sessionMessageProviders.TryRemove(instance, out SessionMessageProvider _);

            var noWait = RetryHelper.ExecuteWithRetryOnTransient(() => this.StateManager.RemoveAsync(GetSessionMessagesDictionaryName(instance)),
                                                                 uniqueActionIdentifier: $"Orchestration = '{instance}', Action = 'DropSessionMessagesDictionaryBackgroundTask'");
        }
        public async Task <PersistentSession> GetSession(string instanceId)
        {
            await EnsureStoreInitialized();

            return(await RetryHelper.ExecuteWithRetryOnTransient(async() =>
            {
                using (var txn = this.StateManager.CreateTransaction())
                {
                    var value = await this.Store.TryGetValueAsync(txn, instanceId);
                    if (value.HasValue)
                    {
                        return value.Value;
                    }
                }
                return null;
            }, uniqueActionIdentifier : $"Orchestration InstanceId = {instanceId}, Action = {nameof(SessionProvider)}.{nameof(GetSession)}"));
        }
예제 #5
0
 protected Task <Message <TKey, TValue> > GetValueAsync(TKey key)
 {
     return(RetryHelper.ExecuteWithRetryOnTransient(async() =>
     {
         using (var tx = this.StateManager.CreateTransaction())
         {
             var result = await this.Store.TryGetValueAsync(tx, key);
             if (result.HasValue)
             {
                 return new Message <TKey, TValue>(key, result.Value);
             }
             var errorMessage = $"Internal Server Error: Did not find an item in reliable dictionary while having the item key {key} in memory";
             ServiceFabricProviderEventSource.Tracing.UnexpectedCodeCondition(errorMessage);
             throw new Exception(errorMessage);
         }
     }, uniqueActionIdentifier: $"Key = {key}, Action = MessageProviderBase.GetValueAsync, StoreName : {this.storeName}"));
 }
        public async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage)
        {
            if (creationMessage.Event is ExecutionStartedEvent executionStarted && executionStarted.ScheduledStartTime.HasValue)
            {
                throw new NotSupportedException("Service Fabric storage provider for Durable Tasks currently does not support scheduled starts");
            }

            creationMessage.OrchestrationInstance.InstanceId.EnsureValidInstanceId();
            ExecutionStartedEvent startEvent = creationMessage.Event as ExecutionStartedEvent;

            if (startEvent == null)
            {
                await this.SendTaskOrchestrationMessageAsync(creationMessage);

                return;
            }

            var instance = creationMessage.OrchestrationInstance;

            var added = await RetryHelper.ExecuteWithRetryOnTransient <bool>(async() =>
            {
                using (var tx = this.stateManager.CreateTransaction())
                {
                    if (await this.orchestrationProvider.TryAddSession(tx, new TaskMessageItem(creationMessage)))
                    {
                        await WriteExecutionStartedEventToInstanceStore(tx, startEvent);
                        await tx.CommitAsync();
                        return(true);
                    }

                    return(false);
                }
            }, uniqueActionIdentifier : $"Orchestration = '{instance}', Action = '{nameof(CreateTaskOrchestrationAsync)}'");

            if (added)
            {
                string message = string.Format("Orchestration with instanceId : '{0}' and executionId : '{1}' is Created.", instance.InstanceId, instance.ExecutionId);
                ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(instance.InstanceId, instance.ExecutionId, message);
                this.orchestrationProvider.TryEnqueueSession(creationMessage.OrchestrationInstance);
            }
            else
            {
                throw new OrchestrationAlreadyExistsException($"An orchestration with id '{creationMessage.OrchestrationInstance.InstanceId}' is already running.");
            }
        }
        public async Task <List <string> > GetExecutionIds(string instanceId)
        {
            List <string> executionIds = null;

            await RetryHelper.ExecuteWithRetryOnTransient(async() =>
            {
                using (var tx = this.stateManager.CreateTransaction())
                {
                    var executionIdsValue = await this.executionIdStore.TryGetValueAsync(tx, instanceId);
                    if (executionIdsValue.HasValue)
                    {
                        executionIds = executionIdsValue.Value;
                    }
                }
            }, uniqueActionIdentifier : $"Orchestration Instance Id = {instanceId}, Action = {nameof(FabricOrchestrationInstanceStore)}.{nameof(GetExecutionIds)}");

            return(executionIds);
        }
예제 #8
0
        protected Task EnumerateItems(Action <KeyValuePair <TKey, TValue> > itemAction)
        {
            return(RetryHelper.ExecuteWithRetryOnTransient(async() =>
            {
                using (var tx = this.StateManager.CreateTransaction())
                {
                    var count = await this.Store.GetCountAsync(tx);

                    if (count > 0)
                    {
                        var enumerable = await this.Store.CreateEnumerableAsync(tx, EnumerationMode.Unordered);
                        using (var enumerator = enumerable.GetAsyncEnumerator())
                        {
                            while (await enumerator.MoveNextAsync(this.CancellationToken))
                            {
                                itemAction(enumerator.Current);
                            }
                        }
                    }
                }
            }, uniqueActionIdentifier: $"Action = MessageProviderBase.EnumerateItems, StoreName : {this.storeName}"));
        }
        public async Task CompleteTaskActivityWorkItemAsync(TaskActivityWorkItem workItem, TaskMessage responseMessage)
        {
            bool added = false;

            await RetryHelper.ExecuteWithRetryOnTransient(async() =>
            {
                bool retryOnException;
                do
                {
                    try
                    {
                        added            = false;
                        retryOnException = false;

                        using (var txn = this.stateManager.CreateTransaction())
                        {
                            await this.activitiesProvider.CompleteAsync(txn, workItem.Id);
                            added = await this.orchestrationProvider.TryAppendMessageAsync(txn, new TaskMessageItem(responseMessage));
                            await txn.CommitAsync();
                        }
                    }
                    catch (FabricReplicationOperationTooLargeException ex)
                    {
                        ServiceFabricProviderEventSource.Tracing.ExceptionInReliableCollectionOperations($"OrchestrationInstance = {responseMessage.OrchestrationInstance}, ActivityId = {workItem.Id}, Action = {nameof(CompleteTaskActivityWorkItemAsync)}", ex.ToString());
                        retryOnException      = true;
                        var originalEvent     = responseMessage.Event;
                        int taskScheduledId   = GetTaskScheduledId(originalEvent);
                        string details        = $"Fabric exception when trying to save activity result: {ex}. Consider reducing the serialization size of activity result to avoid the issue.";
                        responseMessage.Event = new TaskFailedEvent(originalEvent.EventId, taskScheduledId, ex.Message, details);
                    }
                } while (retryOnException);
            }, uniqueActionIdentifier : $"Orchestration = '{responseMessage.OrchestrationInstance}', ActivityId = '{workItem.Id}', Action = '{nameof(CompleteTaskActivityWorkItemAsync)}'");

            if (added)
            {
                this.orchestrationProvider.TryEnqueueSession(responseMessage.OrchestrationInstance);
            }
        }
        // Caller should ensure the workItem has reached terminal state.
        private async Task HandleCompletedOrchestration(TaskOrchestrationWorkItem workItem)
        {
            await RetryHelper.ExecuteWithRetryOnTransient(async() =>
            {
                using (var txn = this.stateManager.CreateTransaction())
                {
                    await this.instanceStore.WriteEntitiesAsync(txn, new InstanceEntityBase[]
                    {
                        new OrchestrationStateInstanceEntity()
                        {
                            State = Utils.BuildOrchestrationState(workItem.OrchestrationRuntimeState)
                        }
                    });
                    // DropSession does 2 things (like mentioned in the comments above) - remove the row from sessions dictionary
                    // and delete the session messages dictionary. The second step is in a background thread and not part of transaction.
                    // However even if this transaction failed but we ended up deleting session messages dictionary, that's ok - at
                    // that time, it should be an empty dictionary and we would have updated the runtime session state to full completed
                    // state in the transaction from Complete method. So the subsequent attempt would be able to complete the session.
                    await this.orchestrationProvider.DropSession(txn, workItem.OrchestrationRuntimeState.OrchestrationInstance);
                    await txn.CommitAsync();
                }
            }, uniqueActionIdentifier : $"OrchestrationId = '{workItem.InstanceId}', Action = '{nameof(HandleCompletedOrchestration)}'");

            this.instanceStore.OnOrchestrationCompleted(workItem.OrchestrationRuntimeState.OrchestrationInstance);

            string message = string.Format("Orchestration with instanceId : '{0}' and executionId : '{1}' Finished with the status {2} and result {3} in {4} seconds.",
                                           workItem.InstanceId,
                                           workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId,
                                           workItem.OrchestrationRuntimeState.OrchestrationStatus.ToString(),
                                           workItem.OrchestrationRuntimeState.Output,
                                           (workItem.OrchestrationRuntimeState.CompletedTime - workItem.OrchestrationRuntimeState.CreatedTime).TotalSeconds);

            ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId,
                                                                                 workItem.OrchestrationRuntimeState.OrchestrationInstance.ExecutionId,
                                                                                 message);
        }
        public async Task CompleteTaskOrchestrationWorkItemAsync(
            TaskOrchestrationWorkItem workItem,
            OrchestrationRuntimeState newOrchestrationRuntimeState,
            IList <TaskMessage> outboundMessages,
            IList <TaskMessage> orchestratorMessages,
            IList <TaskMessage> timerMessages,
            TaskMessage continuedAsNewMessage,
            OrchestrationState orchestrationState)
        {
            SessionInformation sessionInfo = GetSessionInfo(workItem.InstanceId);

            ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(workItem.InstanceId,
                                                                                 workItem.OrchestrationRuntimeState.OrchestrationInstance?.ExecutionId,
                                                                                 $"Current orchestration status: {workItem.OrchestrationRuntimeState.OrchestrationStatus}");
            bool isComplete = this.IsOrchestrationComplete(workItem.OrchestrationRuntimeState.OrchestrationStatus);

            IList <OrchestrationInstance>             sessionsToEnqueue = null;
            List <Message <Guid, TaskMessageItem> >   scheduledMessages = null;
            List <Message <string, TaskMessageItem> > activityMessages  = null;

            await RetryHelper.ExecuteWithRetryOnTransient(async() =>
            {
                bool retryOnException;
                do
                {
                    try
                    {
                        retryOnException  = false;
                        sessionsToEnqueue = null;
                        scheduledMessages = null;
                        activityMessages  = null;

                        using (var txn = this.stateManager.CreateTransaction())
                        {
                            if (outboundMessages?.Count > 0)
                            {
                                activityMessages = outboundMessages.Select(m => new Message <string, TaskMessageItem>(Guid.NewGuid().ToString(), new TaskMessageItem(m))).ToList();
                                await this.activitiesProvider.SendBatchBeginAsync(txn, activityMessages);
                            }

                            if (timerMessages?.Count > 0)
                            {
                                scheduledMessages = timerMessages.Select(m => new Message <Guid, TaskMessageItem>(Guid.NewGuid(), new TaskMessageItem(m))).ToList();
                                await this.scheduledMessagesProvider.SendBatchBeginAsync(txn, scheduledMessages);
                            }

                            if (orchestratorMessages?.Count > 0)
                            {
                                if (workItem.OrchestrationRuntimeState?.ParentInstance != null)
                                {
                                    sessionsToEnqueue = await this.orchestrationProvider.TryAppendMessageBatchAsync(txn, orchestratorMessages.Select(tm => new TaskMessageItem(tm)));
                                }
                                else
                                {
                                    await this.orchestrationProvider.AppendMessageBatchAsync(txn, orchestratorMessages.Select(tm => new TaskMessageItem(tm)));
                                    sessionsToEnqueue = orchestratorMessages.Select(m => m.OrchestrationInstance).ToList();
                                }
                            }

                            if (continuedAsNewMessage != null)
                            {
                                await this.orchestrationProvider.AppendMessageAsync(txn, new TaskMessageItem(continuedAsNewMessage));
                                sessionsToEnqueue = new List <OrchestrationInstance>()
                                {
                                    continuedAsNewMessage.OrchestrationInstance
                                };
                            }

                            await this.orchestrationProvider.CompleteMessages(txn, sessionInfo.Instance, sessionInfo.LockTokens);

                            if (workItem.OrchestrationRuntimeState.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
                            {
                                await HandleCompletedOrchestration(workItem);
                            }

                            // When an orchestration is completed, we need to drop the session which involves 2 steps (1) Removing the row from sessions
                            // (2) Dropping the session messages dictionary. The second step is done in background thread for performance so is not
                            // part of transaction. Since it will happen outside the trasanction, if this transaction fails for some reason and we dropped
                            // the session as part of this transaction, we wouldn't have updated the session state but would have lost the messages
                            // in the session messages dictionary which are needed for state to reach complete state (when the orchestration is picked up again in next fetch).
                            // So we don't want to drop session as part of this transaction.
                            // Instead, we drop the session as part of a subsequent different transaction.
                            // However, framework passes us 'null' value for 'newOrchestrationRuntimeState' when orchestration is completed and
                            // if we updated the session state to null and this transaction succeded, and a node failures occurs and we
                            // never call the subsequent transaction, we will lose the runtime state of orchestration and never will be able to
                            // mark it as complete even if it is. So we use the work item's runtime state when 'newOrchestrationRuntimeState' is null
                            // so that the latest state is what is stored for the session.
                            // As part of next transaction, we are going to remove the row anyway for the session and it doesn't matter to update it to 'null'.

                            await this.orchestrationProvider.UpdateSessionState(txn, newOrchestrationRuntimeState.OrchestrationInstance, newOrchestrationRuntimeState ?? workItem.OrchestrationRuntimeState);

                            // We skip writing to instanceStore when orchestration reached terminal state to avoid a minor timing issue that
                            // wait for an orchestration completes but another orchestration with the same name cannot be started immediately
                            // because the session is still in store. We update the instance store on orchestration completion and drop the
                            // session as part of the next atomic transaction.
                            if (this.instanceStore != null && orchestrationState != null && !isComplete)
                            {
                                await this.instanceStore.WriteEntitiesAsync(txn, new InstanceEntityBase[]
                                {
                                    new OrchestrationStateInstanceEntity()
                                    {
                                        State = orchestrationState
                                    }
                                });
                            }
                            await txn.CommitAsync();
                        }
                    }
                    catch (FabricReplicationOperationTooLargeException ex)
                    {
                        ServiceFabricProviderEventSource.Tracing.ExceptionInReliableCollectionOperations($"OrchestrationInstance = {sessionInfo.Instance}, Action = {nameof(CompleteTaskOrchestrationWorkItemAsync)}", ex.ToString());
                        retryOnException             = true;
                        newOrchestrationRuntimeState = null;
                        outboundMessages             = null;
                        timerMessages        = null;
                        orchestratorMessages = null;
                        if (orchestrationState != null)
                        {
                            orchestrationState.OrchestrationStatus = OrchestrationStatus.Failed;
                            orchestrationState.Output = $"Fabric exception when trying to process orchestration: {ex}. Investigate and consider reducing the serialization size of orchestration inputs/outputs/overall length to avoid the issue.";
                        }
                    }
                } while (retryOnException);
            }, uniqueActionIdentifier : $"OrchestrationId = '{workItem.InstanceId}', Action = '{nameof(CompleteTaskOrchestrationWorkItemAsync)}'");

            if (activityMessages != null)
            {
                this.activitiesProvider.SendBatchComplete(activityMessages);
            }
            if (scheduledMessages != null)
            {
                this.scheduledMessagesProvider.SendBatchComplete(scheduledMessages);
            }
            if (sessionsToEnqueue != null)
            {
                foreach (var instance in sessionsToEnqueue)
                {
                    this.orchestrationProvider.TryEnqueueSession(instance);
                }
            }

            if (isComplete)
            {
                await HandleCompletedOrchestration(workItem);
            }
        }
예제 #12
0
        // Since this method is started as part of StartAsync, the other stores maynot be immediately initialized
        // by the time this invokes operations on those stores. But that would be a transient error and the next
        // iteration of processing should take care of making things right.
        async Task ProcessScheduledMessages()
        {
            while (!IsStopped())
            {
                try
                {
                    var currentTime = DateTime.UtcNow;
                    var nextCheck   = currentTime + TimeSpan.FromSeconds(1);

                    var builder = this.inMemorySet.ToBuilder();
                    List <Message <Guid, TaskMessageItem> > activatedMessages = new List <Message <Guid, TaskMessageItem> >();

                    while (builder.Count > 0)
                    {
                        var firstPendingMessage = builder.Min;
                        var timerEvent          = firstPendingMessage.Value.TaskMessage.Event as TimerFiredEvent;

                        if (timerEvent == null)
                        {
                            throw new Exception("Internal Server Error : Ended up adding non TimerFiredEvent TaskMessage as scheduled message");
                        }

                        if (timerEvent.FireAt <= currentTime)
                        {
                            activatedMessages.Add(firstPendingMessage);
                            builder.Remove(firstPendingMessage);
                        }
                        else
                        {
                            nextCheck = timerEvent.FireAt;
                            break;
                        }
                    }

                    if (IsStopped())
                    {
                        break;
                    }

                    if (activatedMessages.Count > 0)
                    {
                        var keys   = activatedMessages.Select(m => m.Key);
                        var values = activatedMessages.Select(m => m.Value).ToList();

                        IList <OrchestrationInstance> modifiedSessions = null;

                        await RetryHelper.ExecuteWithRetryOnTransient(async() =>
                        {
                            using (var tx = this.StateManager.CreateTransaction())
                            {
                                modifiedSessions = await this.sessionProvider.TryAppendMessageBatchAsync(tx, values);
                                await this.CompleteBatchAsync(tx, keys);
                                await tx.CommitAsync();
                            }
                        }, uniqueActionIdentifier : $"Action = '{nameof(ScheduledMessageProvider)}.{nameof(ProcessScheduledMessages)}'");

                        lock (@lock)
                        {
                            this.inMemorySet = this.inMemorySet.Except(activatedMessages);
                        }

                        if (modifiedSessions != null)
                        {
                            foreach (var sessionId in modifiedSessions)
                            {
                                this.sessionProvider.TryEnqueueSession(sessionId);
                            }
                        }
                    }

                    this.nextActivationCheck = nextCheck;
                    await WaitForItemsAsync(this.nextActivationCheck - DateTime.UtcNow);
                }
                catch (Exception e)
                {
                    ServiceFabricProviderEventSource.Tracing.ExceptionWhileRunningBackgroundJob($"{nameof(ScheduledMessageProvider)}.{nameof(ProcessScheduledMessages)}", e.ToString());
                    await Task.Delay(TimeSpan.FromMilliseconds(100));
                }
            }
        }
        public async Task <OrchestrationStateInstanceEntity> GetOrchestrationStateAsync(string instanceId, string executionId)
        {
            await EnsureStoreInitializedAsync();

            var queryKey = this.GetKey(instanceId, executionId);
            var result   = await RetryHelper.ExecuteWithRetryOnTransient(async() =>
            {
                using (var txn = this.stateManager.CreateTransaction())
                {
                    var state = await this.instanceStore.TryGetValueAsync(txn, queryKey);
                    if (state.HasValue)
                    {
                        return(new OrchestrationStateInstanceEntity()
                        {
                            State = state.Value,
                        });
                    }
                }
                return(null);
            }, uniqueActionIdentifier : $"Orchestration Instance Id = {instanceId}, ExecutionId = {executionId}, Action = {nameof(FabricOrchestrationInstanceStore)}.{nameof(GetOrchestrationStateAsync)}:QueryInstanceStore");

            if (result != null)
            {
                return(result);
            }

            // If querying for orchestration which completed an hour ago, we won't return the results.
            var now = DateTime.UtcNow;

            for (int i = 0; i < 2; i++)
            {
                var backupDictionaryName = GetDictionaryKeyFromTimeFormat(now - TimeSpan.FromHours(i));
                var backupDictionary     = await this.stateManager.TryGetAsync <IReliableDictionary <string, OrchestrationState> >(backupDictionaryName);

                if (backupDictionary.HasValue)
                {
                    result = await RetryHelper.ExecuteWithRetryOnTransient(async() =>
                    {
                        using (var txn = this.stateManager.CreateTransaction())
                        {
                            var state = await backupDictionary.Value.TryGetValueAsync(txn, queryKey);
                            if (state.HasValue)
                            {
                                return(new OrchestrationStateInstanceEntity()
                                {
                                    State = state.Value,
                                });
                            }
                        }

                        return(null);
                    }, uniqueActionIdentifier : $"Orchestration Instance Id = {instanceId}, ExecutionId = {executionId}, Action = {nameof(FabricOrchestrationInstanceStore)}.{nameof(GetOrchestrationStateAsync)}:QueryBackupInstanceStore {backupDictionaryName}");

                    if (result != null)
                    {
                        return(result);
                    }
                }
            }

            return(null);
        }