private async Task ProcessQueueItemsAsync()
{
await this.controlQueueLock.WaitAsync();
IDatabase database = this.redisConnection.GetDatabase();
RedisValue[] messagesJson = await database.ListRangeAsync(this.partitionControlQueueKey);
foreach (string messageJson in messagesJson)
{
TaskMessage taskMessage = RedisSerializer.DeserializeObject <TaskMessage>(messageJson);
string instanceId = taskMessage.OrchestrationInstance.InstanceId;
string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationRuntimeStateHashKey(this.taskHub, this.partition);
string orchestrationInstanceQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, instanceId);
// Enter empty runtime state if it is an execution start event
if (taskMessage.Event is ExecutionStartedEvent startedEvent)
{
OrchestrationRuntimeState emptyState = GetEmptyState(taskMessage);
await database.HashSetAsync(orchestrationStateKey, instanceId, RedisSerializer.SerializeObject(emptyState.Events), When.NotExists);
}
await database.ListRightPopLeftPushAsync(this.partitionControlQueueKey, orchestrationInstanceQueueKey);
this.activeOrchestrationLocks.TryAdd(instanceId, new SemaphoreSlim(1, 1));
}
//Release lock so another notification can be handled
this.controlQueueLock.Release();
}
public async Task CreateTaskOrchestration(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses)
{
if (!(creationMessage.Event is ExecutionStartedEvent executionStartEvent))
{
throw new InvalidOperationException("Invalid creation task message");
}
string orchestrationId = creationMessage.OrchestrationInstance.InstanceId;
// Lock so that two clients can't create the same orchestrationId at the same time.
await this.orchestrationCurrentStateLock.WaitAsync();
IDatabase redisDB = this.redisConnection.GetDatabase();
string orchestrationHistoryKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId);
string currentStateJson = await redisDB.StringGetAsync(orchestrationHistoryKey);
if (currentStateJson != null)
{
OrchestrationState currentState = RedisSerializer.DeserializeObject <OrchestrationState>(currentStateJson);
if (dedupeStatuses == null || dedupeStatuses.Contains(currentState.OrchestrationStatus))
{
// An orchestration with same instance id is already running
throw new OrchestrationAlreadyExistsException($"An orchestration with id '{creationMessage.OrchestrationInstance.InstanceId}' already exists. It is in state {currentState.OrchestrationStatus}");
}
}
RedisTransactionBuilder transactionBuilder = this.CreateNonExistingOrchestrationTransaction(orchestrationId);
var state = new OrchestrationState
{
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = creationMessage.OrchestrationInstance.InstanceId,
ExecutionId = creationMessage.OrchestrationInstance.ExecutionId,
},
CreatedTime = DateTime.UtcNow,
OrchestrationStatus = OrchestrationStatus.Pending,
Version = executionStartEvent.Version,
Name = executionStartEvent.Name,
Input = executionStartEvent.Input,
};
transactionBuilder.AddOrchestrationToSet(orchestrationId);
transactionBuilder.AddOrchestrationStateToHistory(orchestrationId, state);
transactionBuilder.SendControlQueueMessage(creationMessage);
bool transactionSucceeded = await transactionBuilder.CommitTransactionAsync();
string logMessage;
if (transactionSucceeded)
{
logMessage = "Succeeded in transaction of creating task orchestration";
}
else
{
logMessage = "Failed in transaction of creating task orchestration";
}
await this.logger.LogAsync(logMessage);
this.orchestrationCurrentStateLock.Release();
}
private async Task <List <TaskMessage> > GetOrchestrationControlQueueMessages(string orchestrationId)
{
IDatabase redisDatabase = this.redisConnection.GetDatabase();
string orchestrationControlQueueKey = RedisKeyNameResolver.GetOrchestrationQueueKey(this.taskHub, this.partition, orchestrationId);
return((await redisDatabase.ListRangeAsync(orchestrationControlQueueKey))
.Select(json => RedisSerializer.DeserializeObject <TaskMessage>(json))
.ToList());
}
public async Task <OrchestrationState> GetOrchestrationState(string orchestrationId)
{
IDatabase redisDatabase = this.redisConnection.GetDatabase();
string orchestrationStateKey = RedisKeyNameResolver.GetOrchestrationStateKey(this.taskHub, this.partition, orchestrationId);
string orchestrationStateJson = await redisDatabase.StringGetAsync(orchestrationStateKey);
if (orchestrationStateJson == null)
{
return(null);
}
return(RedisSerializer.DeserializeObject <OrchestrationState>(orchestrationStateJson));
}
public async Task <TaskActivityWorkItem> GetSingleTaskItem(TimeSpan receiveTimeout, CancellationToken cancellationToken)
{
await this.redisLogger.LogAsync($"Attempting to grab an activity task item for {receiveTimeout}");
var timeoutCancellationSource = new CancellationTokenSource(receiveTimeout);
var globalCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationSource.Token);
var globalCancellationToken = globalCancellationSource.Token;
IDatabase redisDb = this.redisConnection.GetDatabase();
int numTries = 0;
while (!cancellationToken.IsCancellationRequested)
{
string messageJson = await redisDb.ListRightPopLeftPushAsync(this.incomingTaskQueueKey, this.processingTaskQueueKey);
if (messageJson == null)
{
numTries += 1;
await Task.Delay(50);
continue;
}
await this.redisLogger.LogAsync($"Got activity task item after {numTries} attempts");
TaskMessage message = RedisSerializer.DeserializeObject <TaskMessage>(messageJson);
return(new TaskActivityWorkItem
{
Id = Guid.NewGuid().ToString(),
LockedUntilUtc = DateTime.UtcNow.AddHours(1), // technically it is locked indefinitely
TaskMessage = message,
});
}
// If task cancelled due to timeout or cancellation token, just return null
return(null);
}
/// <summary>
/// For now this class just cleans up all workers on the task hub, but eventually it will run in the background
/// and clean up any workers that haven't proven they are alive in the last n seconds. For now it must be called BEFORE
/// creating a new <see cref="ActivityTaskHandler"/>.
/// </summary>
/// <returns></returns>
public async Task CleanupWorkersAsync()
{
IDatabase redisDatabase = this.redisConnection.GetDatabase();
RedisValue[] deadWorkerIds = await redisDatabase.SetMembersAsync(this.workerSetKey);
foreach (string deadWorkerId in deadWorkerIds)
{
string processingQueueKey = RedisKeyNameResolver.GetTaskActivityProcessingQueueKey(this.taskHub, deadWorkerId);
long itemsToRestore = await redisDatabase.ListLengthAsync(processingQueueKey);
await this.logger.LogAsync($"Moving {itemsToRestore} from processing queue back to incoming queue");
string restoredMessage = await redisDatabase.ListRightPopLeftPushAsync(processingQueueKey, incomingActivityQueueKey);
while (restoredMessage != null)
{
TaskMessage message = RedisSerializer.DeserializeObject <TaskMessage>(restoredMessage);
await this.logger.LogAsync($"Moved activity with id {message.Event.EventId} from processing queue back to incoming queue");
restoredMessage = await redisDatabase.ListRightPopLeftPushAsync(processingQueueKey, incomingActivityQueueKey);
}
}
}