internal Task DeadletterMessageAsync(RedisMessage <T> message, int deadLetterLimit)
{
return(Task.WhenAll(
_db.HashSetAsync(message.HashKey, "MaxDeliveryCountExceeded", $"DeliveryCount exceeded limit of {deadLetterLimit}"),
_db.ListLeftPushAsync(RedisQueueConventions.GetDeadLetterQueueName(_queueName), message.RedisValue),
_db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), message.RedisValue, -1)));
}
private async Task <RedisMessage <T> > GetMessageAsync()
{
try
{
var listItem = await _db.ListRightPopLeftPushAsync(_queueName, RedisQueueConventions.GetProcessingQueueName(_queueName)).ConfigureAwait(false);
if (listItem.IsNullOrEmpty)
{
return(null);
}
var message = _redisConfiguration.MessageSerializer.Deserialize <T>(listItem);
var hashKey = RedisQueueConventions.GetMessageHashKey(_queueName, message.Id);
var tasks = new Task[]
{
_db.StringSetAsync(RedisQueueConventions.GetMessageExpirationKey(_queueName, message.Id), DateTimeOffset.Now.ToUnixTimeMilliseconds()),
_db.HashIncrementAsync(hashKey, RedisHashKeys.DeliveryCount, 1),
};
await Task.WhenAll(tasks).ConfigureAwait(false);
var hash = await _db.HashGetAllAsync(hashKey).ConfigureAwait(false);
return(new RedisMessage <T>(listItem, message, hash, _queueName));
}
catch (RedisTimeoutException e)
{
_hostConfiguration.Log.Error(e, "Error retrieving redis message");
return(null);
}
catch (RedisException e)
{
_hostConfiguration.Log.Error(e, "Error retrieving redis message");
return(null);
}
}
internal Task AbandonMessageByErrorAsync(RedisMessage <T> message, Exception e)
{
return(Task.WhenAll(
_db.HashSetAsync(message.HashKey, RedisHashKeys.Errors, $"{e.Message}\n{e.StackTrace}"),
_db.ListLeftPushAsync(_queueName, message.RedisValue),
_db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), message.RedisValue, -1)));
}
private async Task DetectAndHandleLostMessages(string queueName)
{
Debug.WriteLine($"Finding lost messages from {queueName}");
const int take = 50;
var start = -take;
var stop = -1;
try
{
while (true)
{
var listItems = await _db.ListRangeAsync(RedisQueueConventions.GetProcessingQueueName(queueName), start, stop).ConfigureAwait(false);
if (!listItems.Any())
{
break;
}
Debug.WriteLine($"Found {listItems.Length} processing in {queueName}");
var checkedItems = await Task.WhenAll(listItems.Select(item => HandlePotentiallyLostMessage(queueName, item))).ConfigureAwait(false);
foreach (var(lost, message, listItem) in checkedItems.Where(item => item.lost))
{
if (await RecoverLostMessageAsync(queueName, listItem, message.Id).ConfigureAwait(false))
{
//Shift offset since we manipulated the end of the list and are using offsets
start += 1;
stop += 1;
}
}
if (listItems.Length < take)
{
break;
}
start -= take;
stop -= take;
}
}
catch (Exception e)
{
_log.Error(e, "Error in redis lost message service");
}
}
private async Task <RedisMessage <T> > GetMessageAsync(CancellationTokenSource cancellationsSource)
{
if (cancellationsSource.IsCancellationRequested)
{
return(null);
}
try
{
byte[] listItem = await _db.ListRightPopLeftPushAsync(_queueName, RedisQueueConventions.GetProcessingQueueName(_queueName)).ConfigureAwait(false);
if (listItem == null)
{
cancellationsSource.Cancel();
return(null);
}
var message = _serializer.Deserialize <RedisListItem <T> >(listItem.AsSpan());
var hashKey = RedisQueueConventions.GetMessageHashKey(_queueName, message.Id);
var tasks = new Task[]
{
_db.StringSetAsync(RedisQueueConventions.GetMessageExpirationKey(_queueName, message.Id), DateTimeOffset.Now.ToUnixTimeMilliseconds()),
_db.HashIncrementAsync(hashKey, RedisHashKeys.DeliveryCount)
};
await Task.WhenAll(tasks).ConfigureAwait(false);
var hash = await _db.HashGetAllAsync(hashKey).ConfigureAwait(false);
return(new RedisMessage <T>(listItem, message.Id, message.Body, hash, _queueName));
}
catch (RedisTimeoutException e)
{
_log.Error(e, "Error retrieving redis message");
return(null);
}
catch (RedisException e)
{
_log.Error(e, "Error retrieving redis message");
return(null);
}
}
internal async Task RequeueDeadletterAsync()
{
var deadLetterQueueName = RedisQueueConventions.GetDeadLetterQueueName(_queueName);
var deadLetterProcessingQueueName = RedisQueueConventions.GetProcessingQueueName(deadLetterQueueName);
byte[] listItem = await _db.ListRightPopLeftPushAsync(deadLetterQueueName, deadLetterProcessingQueueName).ConfigureAwait(false);
if (listItem == null)
{
return;
}
var message = _serializer.Deserialize <RedisListItem <T> >(listItem.AsSpan());
var hashKey = RedisQueueConventions.GetMessageHashKey(_queueName, message.Id);
var expirationKey = RedisQueueConventions.GetMessageExpirationKey(_queueName, message.Id);
await _db.HashDeleteAsync(hashKey, new RedisValue[] { expirationKey, RedisHashKeys.DeliveryCount }).ConfigureAwait(false);
await _db.ListRightPopLeftPushAsync(deadLetterProcessingQueueName, _queueName).ConfigureAwait(false);
await _db.PublishAsync(_queueName, 0, CommandFlags.FireAndForget).ConfigureAwait(false);
}
internal Task CompleteMessageAsync(RedisMessage <T> message)
{
return(Task.WhenAll(
_db.KeyDeleteAsync(new RedisKey[] { message.HashKey, message.ExpirationKey }),
_db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), message.RedisValue, -1)));
}
public Task DeadLetterAsync(int deadLetterLimit)
{
return(Task.WhenAll(
_db.ListLeftPushAsync(RedisQueueConventions.GetDeadLetterQueueName(_queueName), _redisMessage.RedisValue),
_db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), _redisMessage.RedisValue, -1)));
}