private async Task ProcessQueueItemsAsync()
{
await this.controlQueueLock.WaitAsync();
IDatabase database = this.redisConnection.GetDatabase();
RedisValue[] messagesJson = await database.ListRangeAsync(this.partitionControlQueueKey);
foreach (string messageJson in messagesJson)
{
TaskMessage taskMessage = RedisSerializer.DeserializeObject <TaskMessage>(messageJson);
string instanceId = taskMessage.OrchestrationInstance.InstanceId;
string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition);
string orchestrationInstanceQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, instanceId);
// Enter empty runtime state if it is an execution start event
if (taskMessage.Event is ExecutionStartedEvent startedEvent)
{
OrchestrationRuntimeState emptyState = GetEmptyState(taskMessage);
await database.HashSetAsync(orchestrationStateKey, instanceId, RedisSerializer.SerializeObject(emptyState.Events), When.NotExists);
}
await database.ListRightPopLeftPushAsync(this.partitionControlQueueKey, orchestrationInstanceQueueKey);
this.activeOrchestrationLocks.TryAdd(instanceId, new SemaphoreSlim(1, 1));
}
//Release lock so another notification can be handled
this.controlQueueLock.Release();
}
public async Task CreateTaskOrchestration(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses)
{
if (!(creationMessage.Event is ExecutionStartedEvent executionStartEvent))
{
throw new InvalidOperationException("Invalid creation task message");
}
string orchestrationId = creationMessage.OrchestrationInstance.InstanceId;
// Lock so that two clients can't create the same orchestrationId at the same time.
await this.orchestrationCurrentStateLock.WaitAsync();
IDatabase redisDB = this.redisConnection.GetDatabase();
string orchestrationHistoryKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId);
string currentStateJson = await redisDB.StringGetAsync(orchestrationHistoryKey);
if (currentStateJson != null)
{
OrchestrationState currentState = RedisSerializer.DeserializeObject <OrchestrationState>(currentStateJson);
if (dedupeStatuses == null || dedupeStatuses.Contains(currentState.OrchestrationStatus))
{
// An orchestration with same instance id is already running
throw new OrchestrationAlreadyExistsException($"An orchestration with id '{creationMessage.OrchestrationInstance.InstanceId}' already exists. It is in state {currentState.OrchestrationStatus}");
}
}
RedisTransactionBuilder transactionBuilder = this.CreateNonExistingOrchestrationTransaction(orchestrationId);
var state = new OrchestrationState
{
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = creationMessage.OrchestrationInstance.InstanceId,
ExecutionId = creationMessage.OrchestrationInstance.ExecutionId,
},
CreatedTime = DateTime.UtcNow,
OrchestrationStatus = OrchestrationStatus.Pending,
Version = executionStartEvent.Version,
Name = executionStartEvent.Name,
Input = executionStartEvent.Input,
};
transactionBuilder.AddOrchestrationToSet(orchestrationId);
transactionBuilder.AddOrchestrationStateToHistory(orchestrationId, state);
transactionBuilder.SendControlQueueMessage(creationMessage);
bool transactionSucceeded = await transactionBuilder.CommitTransactionAsync();
string logMessage;
if (transactionSucceeded)
{
logMessage = "Succeeded in transaction of creating task orchestration";
}
else
{
logMessage = "Failed in transaction of creating task orchestration";
}
await this.logger.LogAsync(logMessage);
this.orchestrationCurrentStateLock.Release();
}
public RedisTransactionBuilder RemoveTaskMessageFromActivityQueue(TaskMessage message, string workerId)
{
string activityProcessingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.taskHub, workerId);
string messageJson = RedisSerializer.SerializeObject(message);
transaction.ListRemoveAsync(activityProcessingQueueKey, messageJson);
return(this);
}
public RedisTransactionBuilder SendActivityMessage(TaskMessage message)
{
string activityIncomingQueueKey = RedisKeyNameResolver.GetTaskActivityIncomingQueueKey(this.taskHub);
string messageJson = RedisSerializer.SerializeObject(message);
transaction.ListLeftPushAsync(activityIncomingQueueKey, messageJson);
return(this);
}
private async Task <OrchestrationRuntimeState> GetOrchestrationRuntimeState(string orchestrationId)
{
IDatabase redisDatabase = this.redisConnection.GetDatabase();
string orchestrationRuntimeState = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition);
string eventListJson = await redisDatabase.HashGetAsync(orchestrationRuntimeState, orchestrationId);
return(RedisSerializer.DeserializeRuntimeState(eventListJson));
}
public WorkerRecycler(string taskHub, ConnectionMultiplexer connection)
{
this.taskHub = taskHub;
this.redisConnection = connection;
this.workerSetKey = RedisKeyNameResolver.GetWorkerSetKey(this.taskHub);
this.incomingActivityQueueKey = RedisKeyNameResolver.GetTaskActivityIncomingQueueKey(this.taskHub);
this.logger = new RedisLogger(this.redisConnection, this.taskHub);
}
private void RegisterWorker()
{
IDatabase database = this.redisConnection.GetDatabase();
string workerSetKey = RedisKeyNameResolver.GetWorkerSetKey(this.settings.TaskHubName);
database.SetAdd(workerSetKey, this.workerGuid);
// TODO: Set up ping background job for multi-worker scenario
}
private async Task <List <TaskMessage> > GetOrchestrationControlQueueMessages(string orchestrationId)
{
IDatabase redisDatabase = this.redisConnection.GetDatabase();
string orchestrationControlQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, orchestrationId);
return((await redisDatabase.ListRangeAsync(orchestrationControlQueueKey))
.Select(json => RedisSerializer.DeserializeObject <TaskMessage>(json))
.ToList());
}
public ActivityTaskHandler(string taskHub, string workerId, ConnectionMultiplexer connection)
{
this.workerId = workerId;
this.taskHub = taskHub;
this.taskLocks = new ConcurrentDictionary <int, SemaphoreSlim>();
this.redisConnection = connection;
this.processingTaskQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.taskHub, this.workerId);
this.incomingTaskQueueKey = RedisKeyNameResolver.GetTaskActivityIncomingQueueKey(this.taskHub);
this.redisLogger = new RedisLogger(this.redisConnection, this.taskHub);
}
/// <inheritdoc />
public async Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem)
{
if (this.activityTaskManager == null)
{
await StartAsync();
}
IDatabase redisDb = this.redisConnection.GetDatabase();
string activityProcessingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.settings.TaskHubName, this.workerGuid);
string messageJson = RedisSerializer.SerializeObject(workItem.TaskMessage);
await redisDb.ListRemoveAsync(activityProcessingQueueKey, messageJson);
}
private OrchestrationSessionPartitionHandler(ConnectionMultiplexer redisConnection, string taskHub, string partition = "singleton")
{
this.taskHub = taskHub;
this.redisConnection = redisConnection;
this.partition = partition;
this.activeOrchestrationLocks = new ConcurrentDictionary <string, SemaphoreSlim>();
this.partitionControlQueueKey = RedisKeyNameResolver.GetPartitionControlQueueKey(this.taskHub, this.partition);
this.partitionControlQueueNotificationChannelKey = RedisKeyNameResolver.GetPartitionControlNotificationChannelKey(this.taskHub, this.partition);
this.currentOrchestrationsSetKey = RedisKeyNameResolver.GetOrchestrationsSetKey(this.taskHub, this.partition);
this.logger = new RedisLogger(redisConnection, taskHub);
}
public RedisTransactionBuilder AddOrchestrationToSet(string orchestrationId)
{
if (this.partition == null)
{
throw new ArgumentNullException($"Cannot call {nameof(AddOrchestrationToSet)} without a partition set.");
}
string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationsSetKey(this.taskHub, this.partition);
transaction.SetAddAsync(orchestrationStateKey, orchestrationId);
return(this);
}
public async Task <OrchestrationState> GetOrchestrationState(string orchestrationId)
{
IDatabase redisDatabase = this.redisConnection.GetDatabase();
string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId);
string orchestrationStateJson = await redisDatabase.StringGetAsync(orchestrationStateKey);
if (orchestrationStateJson == null)
{
return(null);
}
return(RedisSerializer.DeserializeObject <OrchestrationState>(orchestrationStateJson));
}
public RedisTransactionBuilder AddOrchestrationStateToHistory(string orchestrationId, OrchestrationState state)
{
if (this.partition == null)
{
throw new ArgumentNullException($"Cannot call {nameof(AddOrchestrationStateToHistory)} without a partition set.");
}
string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId);
string stateJson = RedisSerializer.SerializeObject(state);
// Add to front of list so history in latest to oldest order
transaction.StringSetAsync(orchestrationStateKey, stateJson);
return(this);
}
public RedisTransactionBuilder SendControlQueueMessage(TaskMessage message)
{
if (this.partition == null)
{
throw new ArgumentNullException($"Cannot call {nameof(SendControlQueueMessage)} without a partition set.");
}
string controlQueueKey = RedisKeyNameResolver.GetPartitionControlQueueKey(this.taskHub, this.partition);
string controlQueueNotificationChannelKey = RedisKeyNameResolver.GetPartitionControlNotificationChannelKey(this.taskHub, this.partition);
string messageJson = RedisSerializer.SerializeObject(message);
this.transaction.ListLeftPushAsync(controlQueueKey, messageJson);
this.transaction.PublishAsync(new RedisChannel(controlQueueNotificationChannelKey, RedisChannel.PatternMode.Literal), messageJson);
return(this);
}
public RedisTransactionBuilder RemoveItemsFromOrchestrationQueue(string orchestrationId, int numberOfItemsToRemove)
{
if (this.partition == null)
{
throw new ArgumentNullException($"Cannot call {nameof(RemoveItemsFromOrchestrationQueue)} without a partition set.");
}
string orchestrationQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, orchestrationId);
for (int i = 0; i < numberOfItemsToRemove; i++)
{
transaction.ListRightPopAsync(orchestrationQueueKey);
}
return(this);
}
public RedisTransactionBuilder SetOrchestrationRuntimeState(string orchestrationId, OrchestrationRuntimeState state)
{
if (this.partition == null)
{
throw new ArgumentNullException($"Cannot call {nameof(SetOrchestrationRuntimeState)} without a partition set.");
}
string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition);
// Do not want to serialize new events;
state.NewEvents.Clear();
string stateJson = RedisSerializer.SerializeObject(state.Events);
transaction.HashSetAsync(orchestrationStateKey, orchestrationId, stateJson);
return(this);
}
private async Task RestoreLocks()
{
IDatabase redisDatabase = this.redisConnection.GetDatabase();
string runtimeEventsKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition);
HashEntry[] runtimeEvents = await redisDatabase.HashGetAllAsync(runtimeEventsKey);
foreach (var hashEntry in runtimeEvents)
{
OrchestrationRuntimeState state = RedisSerializer.DeserializeRuntimeState(hashEntry.Value);
if (state.OrchestrationStatus == OrchestrationStatus.Pending ||
state.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew ||
state.OrchestrationStatus == OrchestrationStatus.Running)
{
// Still running, so add lock
this.activeOrchestrationLocks[hashEntry.Name] = new SemaphoreSlim(1);
}
}
}
/// <summary>
/// For now this class just cleans up all workers on the task hub, but eventually it will run in the background
/// and clean up any workers that haven't proven they are alive in the last n seconds. For now it must be called BEFORE
/// creating a new <see cref="ActivityTaskHandler"/>.
/// </summary>
/// <returns></returns>
public async Task CleanupWorkersAsync()
{
IDatabase redisDatabase = this.redisConnection.GetDatabase();
RedisValue[] deadWorkerIds = await redisDatabase.SetMembersAsync(this.workerSetKey);
foreach (string deadWorkerId in deadWorkerIds)
{
string processingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.taskHub, deadWorkerId);
long itemsToRestore = await redisDatabase.ListLengthAsync(processingQueueKey);
await this.logger.LogAsync($"Moving {itemsToRestore} from processing queue back to incoming queue");
string restoredMessage = await redisDatabase.ListRightPopLeftPushAsync(processingQueueKey, incomingActivityQueueKey);
while (restoredMessage != null)
{
TaskMessage message = RedisSerializer.DeserializeObject <TaskMessage>(restoredMessage);
await this.logger.LogAsync($"Moved activity with id {message.Event.EventId} from processing queue back to incoming queue");
restoredMessage = await redisDatabase.ListRightPopLeftPushAsync(processingQueueKey, incomingActivityQueueKey);
}
}
}
public RedisLogger(ConnectionMultiplexer connection, string taskHub)
{
this.redisConnection = connection;
this.logKey = RedisKeyNameResolver.GetTraceLogsKey(taskHub);
}
