private async Task ProcessQueueItemsAsync() { await this.controlQueueLock.WaitAsync(); IDatabase database = this.redisConnection.GetDatabase(); RedisValue[] messagesJson = await database.ListRangeAsync(this.partitionControlQueueKey); foreach (string messageJson in messagesJson) { TaskMessage taskMessage = RedisSerializer.DeserializeObject <TaskMessage>(messageJson); string instanceId = taskMessage.OrchestrationInstance.InstanceId; string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition); string orchestrationInstanceQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, instanceId); // Enter empty runtime state if it is an execution start event if (taskMessage.Event is ExecutionStartedEvent startedEvent) { OrchestrationRuntimeState emptyState = GetEmptyState(taskMessage); await database.HashSetAsync(orchestrationStateKey, instanceId, RedisSerializer.SerializeObject(emptyState.Events), When.NotExists); } await database.ListRightPopLeftPushAsync(this.partitionControlQueueKey, orchestrationInstanceQueueKey); this.activeOrchestrationLocks.TryAdd(instanceId, new SemaphoreSlim(1, 1)); } //Release lock so another notification can be handled this.controlQueueLock.Release(); }
public async Task CreateTaskOrchestration(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) { if (!(creationMessage.Event is ExecutionStartedEvent executionStartEvent)) { throw new InvalidOperationException("Invalid creation task message"); } string orchestrationId = creationMessage.OrchestrationInstance.InstanceId; // Lock so that two clients can't create the same orchestrationId at the same time. await this.orchestrationCurrentStateLock.WaitAsync(); IDatabase redisDB = this.redisConnection.GetDatabase(); string orchestrationHistoryKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId); string currentStateJson = await redisDB.StringGetAsync(orchestrationHistoryKey); if (currentStateJson != null) { OrchestrationState currentState = RedisSerializer.DeserializeObject <OrchestrationState>(currentStateJson); if (dedupeStatuses == null || dedupeStatuses.Contains(currentState.OrchestrationStatus)) { // An orchestration with same instance id is already running throw new OrchestrationAlreadyExistsException($"An orchestration with id '{creationMessage.OrchestrationInstance.InstanceId}' already exists. It is in state {currentState.OrchestrationStatus}"); } } RedisTransactionBuilder transactionBuilder = this.CreateNonExistingOrchestrationTransaction(orchestrationId); var state = new OrchestrationState { OrchestrationInstance = new OrchestrationInstance { InstanceId = creationMessage.OrchestrationInstance.InstanceId, ExecutionId = creationMessage.OrchestrationInstance.ExecutionId, }, CreatedTime = DateTime.UtcNow, OrchestrationStatus = OrchestrationStatus.Pending, Version = executionStartEvent.Version, Name = executionStartEvent.Name, Input = executionStartEvent.Input, }; transactionBuilder.AddOrchestrationToSet(orchestrationId); transactionBuilder.AddOrchestrationStateToHistory(orchestrationId, state); transactionBuilder.SendControlQueueMessage(creationMessage); bool transactionSucceeded = await transactionBuilder.CommitTransactionAsync(); string logMessage; if (transactionSucceeded) { logMessage = "Succeeded in transaction of creating task orchestration"; } else { logMessage = "Failed in transaction of creating task orchestration"; } await this.logger.LogAsync(logMessage); this.orchestrationCurrentStateLock.Release(); }
private async Task <List <TaskMessage> > GetOrchestrationControlQueueMessages(string orchestrationId) { IDatabase redisDatabase = this.redisConnection.GetDatabase(); string orchestrationControlQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, orchestrationId); return((await redisDatabase.ListRangeAsync(orchestrationControlQueueKey)) .Select(json => RedisSerializer.DeserializeObject <TaskMessage>(json)) .ToList()); }
public async Task <OrchestrationState> GetOrchestrationState(string orchestrationId) { IDatabase redisDatabase = this.redisConnection.GetDatabase(); string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId); string orchestrationStateJson = await redisDatabase.StringGetAsync(orchestrationStateKey); if (orchestrationStateJson == null) { return(null); } return(RedisSerializer.DeserializeObject <OrchestrationState>(orchestrationStateJson)); }
public async Task <TaskActivityWorkItem> GetSingleTaskItem(TimeSpan receiveTimeout, CancellationToken cancellationToken) { await this.redisLogger.LogAsync($"Attempting to grab an activity task item for {receiveTimeout}"); var timeoutCancellationSource = new CancellationTokenSource(receiveTimeout); var globalCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationSource.Token); var globalCancellationToken = globalCancellationSource.Token; IDatabase redisDb = this.redisConnection.GetDatabase(); int numTries = 0; while (!cancellationToken.IsCancellationRequested) { string messageJson = await redisDb.ListRightPopLeftPushAsync(this.incomingTaskQueueKey, this.processingTaskQueueKey); if (messageJson == null) { numTries += 1; await Task.Delay(50); continue; } await this.redisLogger.LogAsync($"Got activity task item after {numTries} attempts"); TaskMessage message = RedisSerializer.DeserializeObject <TaskMessage>(messageJson); return(new TaskActivityWorkItem { Id = Guid.NewGuid().ToString(), LockedUntilUtc = DateTime.UtcNow.AddHours(1), // technically it is locked indefinitely TaskMessage = message, }); } // If task cancelled due to timeout or cancellation token, just return null return(null); }
/// <summary> /// For now this class just cleans up all workers on the task hub, but eventually it will run in the background /// and clean up any workers that haven't proven they are alive in the last n seconds. For now it must be called BEFORE /// creating a new <see cref="ActivityTaskHandler"/>. /// </summary> /// <returns></returns> public async Task CleanupWorkersAsync() { IDatabase redisDatabase = this.redisConnection.GetDatabase(); RedisValue[] deadWorkerIds = await redisDatabase.SetMembersAsync(this.workerSetKey); foreach (string deadWorkerId in deadWorkerIds) { string processingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.taskHub, deadWorkerId); long itemsToRestore = await redisDatabase.ListLengthAsync(processingQueueKey); await this.logger.LogAsync($"Moving {itemsToRestore} from processing queue back to incoming queue"); string restoredMessage = await redisDatabase.ListRightPopLeftPushAsync(processingQueueKey, incomingActivityQueueKey); while (restoredMessage != null) { TaskMessage message = RedisSerializer.DeserializeObject <TaskMessage>(restoredMessage); await this.logger.LogAsync($"Moved activity with id {message.Event.EventId} from processing queue back to incoming queue"); restoredMessage = await redisDatabase.ListRightPopLeftPushAsync(processingQueueKey, incomingActivityQueueKey); } } }