private async Task ProcessQueueItemsAsync() { await this.controlQueueLock.WaitAsync(); IDatabase database = this.redisConnection.GetDatabase(); RedisValue[] messagesJson = await database.ListRangeAsync(this.partitionControlQueueKey); foreach (string messageJson in messagesJson) { TaskMessage taskMessage = RedisSerializer.DeserializeObject <TaskMessage>(messageJson); string instanceId = taskMessage.OrchestrationInstance.InstanceId; string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition); string orchestrationInstanceQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, instanceId); // Enter empty runtime state if it is an execution start event if (taskMessage.Event is ExecutionStartedEvent startedEvent) { OrchestrationRuntimeState emptyState = GetEmptyState(taskMessage); await database.HashSetAsync(orchestrationStateKey, instanceId, RedisSerializer.SerializeObject(emptyState.Events), When.NotExists); } await database.ListRightPopLeftPushAsync(this.partitionControlQueueKey, orchestrationInstanceQueueKey); this.activeOrchestrationLocks.TryAdd(instanceId, new SemaphoreSlim(1, 1)); } //Release lock so another notification can be handled this.controlQueueLock.Release(); }
public async Task CreateTaskOrchestration(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) { if (!(creationMessage.Event is ExecutionStartedEvent executionStartEvent)) { throw new InvalidOperationException("Invalid creation task message"); } string orchestrationId = creationMessage.OrchestrationInstance.InstanceId; // Lock so that two clients can't create the same orchestrationId at the same time. await this.orchestrationCurrentStateLock.WaitAsync(); IDatabase redisDB = this.redisConnection.GetDatabase(); string orchestrationHistoryKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId); string currentStateJson = await redisDB.StringGetAsync(orchestrationHistoryKey); if (currentStateJson != null) { OrchestrationState currentState = RedisSerializer.DeserializeObject <OrchestrationState>(currentStateJson); if (dedupeStatuses == null || dedupeStatuses.Contains(currentState.OrchestrationStatus)) { // An orchestration with same instance id is already running throw new OrchestrationAlreadyExistsException($"An orchestration with id '{creationMessage.OrchestrationInstance.InstanceId}' already exists. It is in state {currentState.OrchestrationStatus}"); } } RedisTransactionBuilder transactionBuilder = this.CreateNonExistingOrchestrationTransaction(orchestrationId); var state = new OrchestrationState { OrchestrationInstance = new OrchestrationInstance { InstanceId = creationMessage.OrchestrationInstance.InstanceId, ExecutionId = creationMessage.OrchestrationInstance.ExecutionId, }, CreatedTime = DateTime.UtcNow, OrchestrationStatus = OrchestrationStatus.Pending, Version = executionStartEvent.Version, Name = executionStartEvent.Name, Input = executionStartEvent.Input, }; transactionBuilder.AddOrchestrationToSet(orchestrationId); transactionBuilder.AddOrchestrationStateToHistory(orchestrationId, state); transactionBuilder.SendControlQueueMessage(creationMessage); bool transactionSucceeded = await transactionBuilder.CommitTransactionAsync(); string logMessage; if (transactionSucceeded) { logMessage = "Succeeded in transaction of creating task orchestration"; } else { logMessage = "Failed in transaction of creating task orchestration"; } await this.logger.LogAsync(logMessage); this.orchestrationCurrentStateLock.Release(); }
private async Task <OrchestrationRuntimeState> GetOrchestrationRuntimeState(string orchestrationId) { IDatabase redisDatabase = this.redisConnection.GetDatabase(); string orchestrationRuntimeState = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition); string eventListJson = await redisDatabase.HashGetAsync(orchestrationRuntimeState, orchestrationId); return(RedisSerializer.DeserializeRuntimeState(eventListJson)); }
public RedisTransactionBuilder RemoveTaskMessageFromActivityQueue(TaskMessage message, string workerId) { string activityProcessingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.taskHub, workerId); string messageJson = RedisSerializer.SerializeObject(message); transaction.ListRemoveAsync(activityProcessingQueueKey, messageJson); return(this); }
public RedisTransactionBuilder SendActivityMessage(TaskMessage message) { string activityIncomingQueueKey = RedisKeyNameResolver.GetTaskActivityIncomingQueueKey(this.taskHub); string messageJson = RedisSerializer.SerializeObject(message); transaction.ListLeftPushAsync(activityIncomingQueueKey, messageJson); return(this); }
private async Task <List <TaskMessage> > GetOrchestrationControlQueueMessages(string orchestrationId) { IDatabase redisDatabase = this.redisConnection.GetDatabase(); string orchestrationControlQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, orchestrationId); return((await redisDatabase.ListRangeAsync(orchestrationControlQueueKey)) .Select(json => RedisSerializer.DeserializeObject <TaskMessage>(json)) .ToList()); }
/// <inheritdoc /> public async Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem) { if (this.activityTaskManager == null) { await StartAsync(); } IDatabase redisDb = this.redisConnection.GetDatabase(); string activityProcessingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.settings.TaskHubName, this.workerGuid); string messageJson = RedisSerializer.SerializeObject(workItem.TaskMessage); await redisDb.ListRemoveAsync(activityProcessingQueueKey, messageJson); }
public async Task <OrchestrationState> GetOrchestrationState(string orchestrationId) { IDatabase redisDatabase = this.redisConnection.GetDatabase(); string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId); string orchestrationStateJson = await redisDatabase.StringGetAsync(orchestrationStateKey); if (orchestrationStateJson == null) { return(null); } return(RedisSerializer.DeserializeObject <OrchestrationState>(orchestrationStateJson)); }
public RedisTransactionBuilder AddOrchestrationStateToHistory(string orchestrationId, OrchestrationState state) { if (this.partition == null) { throw new ArgumentNullException($"Cannot call {nameof(AddOrchestrationStateToHistory)} without a partition set."); } string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId); string stateJson = RedisSerializer.SerializeObject(state); // Add to front of list so history in latest to oldest order transaction.StringSetAsync(orchestrationStateKey, stateJson); return(this); }
public RedisTransactionBuilder SendControlQueueMessage(TaskMessage message) { if (this.partition == null) { throw new ArgumentNullException($"Cannot call {nameof(SendControlQueueMessage)} without a partition set."); } string controlQueueKey = RedisKeyNameResolver.GetPartitionControlQueueKey(this.taskHub, this.partition); string controlQueueNotificationChannelKey = RedisKeyNameResolver.GetPartitionControlNotificationChannelKey(this.taskHub, this.partition); string messageJson = RedisSerializer.SerializeObject(message); this.transaction.ListLeftPushAsync(controlQueueKey, messageJson); this.transaction.PublishAsync(new RedisChannel(controlQueueNotificationChannelKey, RedisChannel.PatternMode.Literal), messageJson); return(this); }
public RedisTransactionBuilder SetOrchestrationRuntimeState(string orchestrationId, OrchestrationRuntimeState state) { if (this.partition == null) { throw new ArgumentNullException($"Cannot call {nameof(SetOrchestrationRuntimeState)} without a partition set."); } string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition); // Do not want to serialize new events; state.NewEvents.Clear(); string stateJson = RedisSerializer.SerializeObject(state.Events); transaction.HashSetAsync(orchestrationStateKey, orchestrationId, stateJson); return(this); }
public async Task SendOrchestrationMessages(params TaskMessage[] messages) { IDatabase redisDb = this.redisConnection.GetDatabase(); Task[] sendMessageTasks = new Task[messages.Length]; RedisChannel notificationChannel = new RedisChannel(this.partitionControlQueueNotificationChannelKey, RedisChannel.PatternMode.Literal); for (int i = 0; i < messages.Length; i++) { string messageJson = RedisSerializer.SerializeObject(messages[i]); sendMessageTasks[i] = redisDb.ListLeftPushAsync(this.partitionControlQueueKey, messageJson); } await Task.WhenAll(sendMessageTasks); await redisDb.PublishAsync(notificationChannel, "batch received"); }
private async Task RestoreLocks() { IDatabase redisDatabase = this.redisConnection.GetDatabase(); string runtimeEventsKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition); HashEntry[] runtimeEvents = await redisDatabase.HashGetAllAsync(runtimeEventsKey); foreach (var hashEntry in runtimeEvents) { OrchestrationRuntimeState state = RedisSerializer.DeserializeRuntimeState(hashEntry.Value); if (state.OrchestrationStatus == OrchestrationStatus.Pending || state.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew || state.OrchestrationStatus == OrchestrationStatus.Running) { // Still running, so add lock this.activeOrchestrationLocks[hashEntry.Name] = new SemaphoreSlim(1); } } }
public async Task <TaskActivityWorkItem> GetSingleTaskItem(TimeSpan receiveTimeout, CancellationToken cancellationToken) { await this.redisLogger.LogAsync($"Attempting to grab an activity task item for {receiveTimeout}"); var timeoutCancellationSource = new CancellationTokenSource(receiveTimeout); var globalCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationSource.Token); var globalCancellationToken = globalCancellationSource.Token; IDatabase redisDb = this.redisConnection.GetDatabase(); int numTries = 0; while (!cancellationToken.IsCancellationRequested) { string messageJson = await redisDb.ListRightPopLeftPushAsync(this.incomingTaskQueueKey, this.processingTaskQueueKey); if (messageJson == null) { numTries += 1; await Task.Delay(50); continue; } await this.redisLogger.LogAsync($"Got activity task item after {numTries} attempts"); TaskMessage message = RedisSerializer.DeserializeObject <TaskMessage>(messageJson); return(new TaskActivityWorkItem { Id = Guid.NewGuid().ToString(), LockedUntilUtc = DateTime.UtcNow.AddHours(1), // technically it is locked indefinitely TaskMessage = message, }); } // If task cancelled due to timeout or cancellation token, just return null return(null); }
/// <summary> /// For now this class just cleans up all workers on the task hub, but eventually it will run in the background /// and clean up any workers that haven't proven they are alive in the last n seconds. For now it must be called BEFORE /// creating a new <see cref="ActivityTaskHandler"/>. /// </summary> /// <returns></returns> public async Task CleanupWorkersAsync() { IDatabase redisDatabase = this.redisConnection.GetDatabase(); RedisValue[] deadWorkerIds = await redisDatabase.SetMembersAsync(this.workerSetKey); foreach (string deadWorkerId in deadWorkerIds) { string processingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.taskHub, deadWorkerId); long itemsToRestore = await redisDatabase.ListLengthAsync(processingQueueKey); await this.logger.LogAsync($"Moving {itemsToRestore} from processing queue back to incoming queue"); string restoredMessage = await redisDatabase.ListRightPopLeftPushAsync(processingQueueKey, incomingActivityQueueKey); while (restoredMessage != null) { TaskMessage message = RedisSerializer.DeserializeObject <TaskMessage>(restoredMessage); await this.logger.LogAsync($"Moved activity with id {message.Event.EventId} from processing queue back to incoming queue"); restoredMessage = await redisDatabase.ListRightPopLeftPushAsync(processingQueueKey, incomingActivityQueueKey); } } }