private async Task ProcessQueueItemsAsync() { await this.controlQueueLock.WaitAsync(); IDatabase database = this.redisConnection.GetDatabase(); RedisValue[] messagesJson = await database.ListRangeAsync(this.partitionControlQueueKey); foreach (string messageJson in messagesJson) { TaskMessage taskMessage = RedisSerializer.DeserializeObject <TaskMessage>(messageJson); string instanceId = taskMessage.OrchestrationInstance.InstanceId; string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition); string orchestrationInstanceQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, instanceId); // Enter empty runtime state if it is an execution start event if (taskMessage.Event is ExecutionStartedEvent startedEvent) { OrchestrationRuntimeState emptyState = GetEmptyState(taskMessage); await database.HashSetAsync(orchestrationStateKey, instanceId, RedisSerializer.SerializeObject(emptyState.Events), When.NotExists); } await database.ListRightPopLeftPushAsync(this.partitionControlQueueKey, orchestrationInstanceQueueKey); this.activeOrchestrationLocks.TryAdd(instanceId, new SemaphoreSlim(1, 1)); } //Release lock so another notification can be handled this.controlQueueLock.Release(); }
public async Task CreateTaskOrchestration(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) { if (!(creationMessage.Event is ExecutionStartedEvent executionStartEvent)) { throw new InvalidOperationException("Invalid creation task message"); } string orchestrationId = creationMessage.OrchestrationInstance.InstanceId; // Lock so that two clients can't create the same orchestrationId at the same time. await this.orchestrationCurrentStateLock.WaitAsync(); IDatabase redisDB = this.redisConnection.GetDatabase(); string orchestrationHistoryKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId); string currentStateJson = await redisDB.StringGetAsync(orchestrationHistoryKey); if (currentStateJson != null) { OrchestrationState currentState = RedisSerializer.DeserializeObject <OrchestrationState>(currentStateJson); if (dedupeStatuses == null || dedupeStatuses.Contains(currentState.OrchestrationStatus)) { // An orchestration with same instance id is already running throw new OrchestrationAlreadyExistsException($"An orchestration with id '{creationMessage.OrchestrationInstance.InstanceId}' already exists. It is in state {currentState.OrchestrationStatus}"); } } RedisTransactionBuilder transactionBuilder = this.CreateNonExistingOrchestrationTransaction(orchestrationId); var state = new OrchestrationState { OrchestrationInstance = new OrchestrationInstance { InstanceId = creationMessage.OrchestrationInstance.InstanceId, ExecutionId = creationMessage.OrchestrationInstance.ExecutionId, }, CreatedTime = DateTime.UtcNow, OrchestrationStatus = OrchestrationStatus.Pending, Version = executionStartEvent.Version, Name = executionStartEvent.Name, Input = executionStartEvent.Input, }; transactionBuilder.AddOrchestrationToSet(orchestrationId); transactionBuilder.AddOrchestrationStateToHistory(orchestrationId, state); transactionBuilder.SendControlQueueMessage(creationMessage); bool transactionSucceeded = await transactionBuilder.CommitTransactionAsync(); string logMessage; if (transactionSucceeded) { logMessage = "Succeeded in transaction of creating task orchestration"; } else { logMessage = "Failed in transaction of creating task orchestration"; } await this.logger.LogAsync(logMessage); this.orchestrationCurrentStateLock.Release(); }
public RedisTransactionBuilder RemoveTaskMessageFromActivityQueue(TaskMessage message, string workerId) { string activityProcessingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.taskHub, workerId); string messageJson = RedisSerializer.SerializeObject(message); transaction.ListRemoveAsync(activityProcessingQueueKey, messageJson); return(this); }
public RedisTransactionBuilder SendActivityMessage(TaskMessage message) { string activityIncomingQueueKey = RedisKeyNameResolver.GetTaskActivityIncomingQueueKey(this.taskHub); string messageJson = RedisSerializer.SerializeObject(message); transaction.ListLeftPushAsync(activityIncomingQueueKey, messageJson); return(this); }
private async Task <OrchestrationRuntimeState> GetOrchestrationRuntimeState(string orchestrationId) { IDatabase redisDatabase = this.redisConnection.GetDatabase(); string orchestrationRuntimeState = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition); string eventListJson = await redisDatabase.HashGetAsync(orchestrationRuntimeState, orchestrationId); return(RedisSerializer.DeserializeRuntimeState(eventListJson)); }
public WorkerRecycler(string taskHub, ConnectionMultiplexer connection) { this.taskHub = taskHub; this.redisConnection = connection; this.workerSetKey = RedisKeyNameResolver.GetWorkerSetKey(this.taskHub); this.incomingActivityQueueKey = RedisKeyNameResolver.GetTaskActivityIncomingQueueKey(this.taskHub); this.logger = new RedisLogger(this.redisConnection, this.taskHub); }
private void RegisterWorker() { IDatabase database = this.redisConnection.GetDatabase(); string workerSetKey = RedisKeyNameResolver.GetWorkerSetKey(this.settings.TaskHubName); database.SetAdd(workerSetKey, this.workerGuid); // TODO: Set up ping background job for multi-worker scenario }
private async Task <List <TaskMessage> > GetOrchestrationControlQueueMessages(string orchestrationId) { IDatabase redisDatabase = this.redisConnection.GetDatabase(); string orchestrationControlQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, orchestrationId); return((await redisDatabase.ListRangeAsync(orchestrationControlQueueKey)) .Select(json => RedisSerializer.DeserializeObject <TaskMessage>(json)) .ToList()); }
public ActivityTaskHandler(string taskHub, string workerId, ConnectionMultiplexer connection) { this.workerId = workerId; this.taskHub = taskHub; this.taskLocks = new ConcurrentDictionary <int, SemaphoreSlim>(); this.redisConnection = connection; this.processingTaskQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.taskHub, this.workerId); this.incomingTaskQueueKey = RedisKeyNameResolver.GetTaskActivityIncomingQueueKey(this.taskHub); this.redisLogger = new RedisLogger(this.redisConnection, this.taskHub); }
/// <inheritdoc /> public async Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem) { if (this.activityTaskManager == null) { await StartAsync(); } IDatabase redisDb = this.redisConnection.GetDatabase(); string activityProcessingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.settings.TaskHubName, this.workerGuid); string messageJson = RedisSerializer.SerializeObject(workItem.TaskMessage); await redisDb.ListRemoveAsync(activityProcessingQueueKey, messageJson); }
private OrchestrationSessionPartitionHandler(ConnectionMultiplexer redisConnection, string taskHub, string partition = "singleton") { this.taskHub = taskHub; this.redisConnection = redisConnection; this.partition = partition; this.activeOrchestrationLocks = new ConcurrentDictionary <string, SemaphoreSlim>(); this.partitionControlQueueKey = RedisKeyNameResolver.GetPartitionControlQueueKey(this.taskHub, this.partition); this.partitionControlQueueNotificationChannelKey = RedisKeyNameResolver.GetPartitionControlNotificationChannelKey(this.taskHub, this.partition); this.currentOrchestrationsSetKey = RedisKeyNameResolver.GetOrchestrationsSetKey(this.taskHub, this.partition); this.logger = new RedisLogger(redisConnection, taskHub); }
public RedisTransactionBuilder AddOrchestrationToSet(string orchestrationId) { if (this.partition == null) { throw new ArgumentNullException($"Cannot call {nameof(AddOrchestrationToSet)} without a partition set."); } string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationsSetKey(this.taskHub, this.partition); transaction.SetAddAsync(orchestrationStateKey, orchestrationId); return(this); }
public async Task <OrchestrationState> GetOrchestrationState(string orchestrationId) { IDatabase redisDatabase = this.redisConnection.GetDatabase(); string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId); string orchestrationStateJson = await redisDatabase.StringGetAsync(orchestrationStateKey); if (orchestrationStateJson == null) { return(null); } return(RedisSerializer.DeserializeObject <OrchestrationState>(orchestrationStateJson)); }
public RedisTransactionBuilder AddOrchestrationStateToHistory(string orchestrationId, OrchestrationState state) { if (this.partition == null) { throw new ArgumentNullException($"Cannot call {nameof(AddOrchestrationStateToHistory)} without a partition set."); } string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId); string stateJson = RedisSerializer.SerializeObject(state); // Add to front of list so history in latest to oldest order transaction.StringSetAsync(orchestrationStateKey, stateJson); return(this); }
public RedisTransactionBuilder SendControlQueueMessage(TaskMessage message) { if (this.partition == null) { throw new ArgumentNullException($"Cannot call {nameof(SendControlQueueMessage)} without a partition set."); } string controlQueueKey = RedisKeyNameResolver.GetPartitionControlQueueKey(this.taskHub, this.partition); string controlQueueNotificationChannelKey = RedisKeyNameResolver.GetPartitionControlNotificationChannelKey(this.taskHub, this.partition); string messageJson = RedisSerializer.SerializeObject(message); this.transaction.ListLeftPushAsync(controlQueueKey, messageJson); this.transaction.PublishAsync(new RedisChannel(controlQueueNotificationChannelKey, RedisChannel.PatternMode.Literal), messageJson); return(this); }
public RedisTransactionBuilder RemoveItemsFromOrchestrationQueue(string orchestrationId, int numberOfItemsToRemove) { if (this.partition == null) { throw new ArgumentNullException($"Cannot call {nameof(RemoveItemsFromOrchestrationQueue)} without a partition set."); } string orchestrationQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, orchestrationId); for (int i = 0; i < numberOfItemsToRemove; i++) { transaction.ListRightPopAsync(orchestrationQueueKey); } return(this); }
public RedisTransactionBuilder SetOrchestrationRuntimeState(string orchestrationId, OrchestrationRuntimeState state) { if (this.partition == null) { throw new ArgumentNullException($"Cannot call {nameof(SetOrchestrationRuntimeState)} without a partition set."); } string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition); // Do not want to serialize new events; state.NewEvents.Clear(); string stateJson = RedisSerializer.SerializeObject(state.Events); transaction.HashSetAsync(orchestrationStateKey, orchestrationId, stateJson); return(this); }
private async Task RestoreLocks() { IDatabase redisDatabase = this.redisConnection.GetDatabase(); string runtimeEventsKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition); HashEntry[] runtimeEvents = await redisDatabase.HashGetAllAsync(runtimeEventsKey); foreach (var hashEntry in runtimeEvents) { OrchestrationRuntimeState state = RedisSerializer.DeserializeRuntimeState(hashEntry.Value); if (state.OrchestrationStatus == OrchestrationStatus.Pending || state.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew || state.OrchestrationStatus == OrchestrationStatus.Running) { // Still running, so add lock this.activeOrchestrationLocks[hashEntry.Name] = new SemaphoreSlim(1); } } }
/// <summary> /// For now this class just cleans up all workers on the task hub, but eventually it will run in the background /// and clean up any workers that haven't proven they are alive in the last n seconds. For now it must be called BEFORE /// creating a new <see cref="ActivityTaskHandler"/>. /// </summary> /// <returns></returns> public async Task CleanupWorkersAsync() { IDatabase redisDatabase = this.redisConnection.GetDatabase(); RedisValue[] deadWorkerIds = await redisDatabase.SetMembersAsync(this.workerSetKey); foreach (string deadWorkerId in deadWorkerIds) { string processingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.taskHub, deadWorkerId); long itemsToRestore = await redisDatabase.ListLengthAsync(processingQueueKey); await this.logger.LogAsync($"Moving {itemsToRestore} from processing queue back to incoming queue"); string restoredMessage = await redisDatabase.ListRightPopLeftPushAsync(processingQueueKey, incomingActivityQueueKey); while (restoredMessage != null) { TaskMessage message = RedisSerializer.DeserializeObject <TaskMessage>(restoredMessage); await this.logger.LogAsync($"Moved activity with id {message.Event.EventId} from processing queue back to incoming queue"); restoredMessage = await redisDatabase.ListRightPopLeftPushAsync(processingQueueKey, incomingActivityQueueKey); } } }
public RedisLogger(ConnectionMultiplexer connection, string taskHub) { this.redisConnection = connection; this.logKey = RedisKeyNameResolver.GetTraceLogsKey(taskHub); }