internal Task AbandonMessageByErrorAsync(RedisMessage <T> message, Exception e)
{
return(Task.WhenAll(
_db.HashSetAsync(message.HashKey, RedisHashKeys.Errors, $"{e.Message}\n{e.StackTrace}"),
_db.ListLeftPushAsync(_queueName, message.RedisValue),
_db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), message.RedisValue, -1)));
}
internal Task DeleteDeadletterAsync(RedisDeadletter <T> deadletter)
{
var deadletterQueueName = RedisQueueConventions.GetDeadLetterQueueName(_queueName);
var hash = RedisQueueConventions.GetMessageHashKey(_queueName, deadletter.Message.Id);
return(Task.WhenAll(_db.KeyDeleteAsync(hash), _db.ListRemoveAsync(deadletterQueueName, _serializer.Serialize(deadletter.Message), -1)));
}
private async Task <RedisMessage <T> > GetMessageAsync()
{
try
{
var listItem = await _db.ListRightPopLeftPushAsync(_queueName, RedisQueueConventions.GetProcessingQueueName(_queueName)).ConfigureAwait(false);
if (listItem.IsNullOrEmpty)
{
return(null);
}
var message = _redisConfiguration.MessageSerializer.Deserialize <T>(listItem);
var hashKey = RedisQueueConventions.GetMessageHashKey(_queueName, message.Id);
var tasks = new Task[]
{
_db.StringSetAsync(RedisQueueConventions.GetMessageExpirationKey(_queueName, message.Id), DateTimeOffset.Now.ToUnixTimeMilliseconds()),
_db.HashIncrementAsync(hashKey, RedisHashKeys.DeliveryCount, 1),
};
await Task.WhenAll(tasks).ConfigureAwait(false);
var hash = await _db.HashGetAllAsync(hashKey).ConfigureAwait(false);
return(new RedisMessage <T>(listItem, message, hash, _queueName));
}
catch (RedisTimeoutException e)
{
_hostConfiguration.Log.Error(e, "Error retrieving redis message");
return(null);
}
catch (RedisException e)
{
_hostConfiguration.Log.Error(e, "Error retrieving redis message");
return(null);
}
}
internal Task DeadletterMessageAsync(RedisMessage <T> message, int deadLetterLimit)
{
return(Task.WhenAll(
_db.HashSetAsync(message.HashKey, "MaxDeliveryCountExceeded", $"DeliveryCount exceeded limit of {deadLetterLimit}"),
_db.ListLeftPushAsync(RedisQueueConventions.GetDeadLetterQueueName(_queueName), message.RedisValue),
_db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), message.RedisValue, -1)));
}
public override async Task StartAsync(CancellationToken cancellationToken)
{
var db = ConnectionMultiplexer.GetDatabase(_redisConfiguration.DatabaseId);
await db.SetAddAsync(RedisQueueConventions.GetSubscriptionKey(AutoMessageMapper.GetQueueName <T>()), _subscription.Name).ConfigureAwait(false);
await base.StartAsync(cancellationToken).ConfigureAwait(false);
}
public RedisMessage(RedisValue redisValue, string id, T message, HashEntry[] hashEntries, string queueName)
{
RedisValue = redisValue;
HashKey = RedisQueueConventions.GetMessageHashKey(queueName, id);
ExpirationKey = RedisQueueConventions.GetMessageExpirationKey(queueName, id);
Message = message;
HashEntries = hashEntries.ToStringDictionary();
}
public async Task <IMessageAttachment> GetAttachmentAsync(string queueName, string id, CancellationToken cancellationToken = default(CancellationToken))
{
var db = _multiplexer.GetDatabase(_configuration.DatabaseId);
var metadataHash = await db.HashGetAllAsync(RedisQueueConventions.GetAttachmentMetadataKey(queueName, id)).ConfigureAwait(false);
var data = await db.StringGetAsync(RedisQueueConventions.GetAttachmentBinaryKey(queueName, id)).ConfigureAwait(false);
var metadata = metadataHash.ToStringDictionary();
return(new MessageAttachment(metadata[FileName], metadata[ContentType], new MemoryStream(data)));
}
public async Task PublishAsync <T>(T message) where T : IRedisEvent
{
var db = _multiplexer.GetDatabase(_configuration.DatabaseId);
var queueName = AutoMessageMapper.GetQueueName <T>();
var subscriptions = await db.SetMembersAsync(RedisQueueConventions.GetSubscriptionKey(queueName)).ConfigureAwait(false);
if (subscriptions == null)
{
return;
}
await Task.WhenAll(subscriptions.Select(sub => SendAsync(message, RedisQueueConventions.GetSubscriptionQueueName(queueName, sub)))).ConfigureAwait(false);
}
private async Task UploadAttachment <T>(RedisListItem <T> message, string queueName, IDatabase db) where T : IRedisMessage
{
if (message.Body is ICommandWithAttachment attachmentMessage)
{
if (_attachmentProvider == null)
{
throw new AttachmentProviderMissingException();
}
if (attachmentMessage.Attachment != null)
{
var attachmentId = Guid.NewGuid().ToString("N");
await db.HashSetAsync(RedisQueueConventions.GetMessageHashKey(queueName, message.Id), AttachmentUtility.AttachmentKey, attachmentId).ConfigureAwait(false);
await _attachmentProvider.UploadAttachmentAsync(queueName, attachmentId, attachmentMessage.Attachment).ConfigureAwait(false);
}
}
}
public async Task <bool> DeleteAttachmentAsync(string queueName, string id, CancellationToken cancellationToken = default(CancellationToken))
{
var db = _multiplexer.GetDatabase(_configuration.DatabaseId);
try
{
await Task.WhenAll(
db.KeyDeleteAsync(RedisQueueConventions.GetAttachmentMetadataKey(queueName, id)),
db.KeyDeleteAsync(RedisQueueConventions.GetAttachmentBinaryKey(queueName, id)))
.ConfigureAwait(false);
return(true);
}
catch (Exception)
{
return(false);
}
}
private async Task DetectAndHandleLostMessages(string queueName)
{
Debug.WriteLine($"Finding lost messages from {queueName}");
const int take = 50;
var start = -take;
var stop = -1;
try
{
while (true)
{
var listItems = await _db.ListRangeAsync(RedisQueueConventions.GetProcessingQueueName(queueName), start, stop).ConfigureAwait(false);
if (!listItems.Any())
{
break;
}
Debug.WriteLine($"Found {listItems.Length} processing in {queueName}");
var checkedItems = await Task.WhenAll(listItems.Select(item => HandlePotentiallyLostMessage(queueName, item))).ConfigureAwait(false);
foreach (var(lost, message, listItem) in checkedItems.Where(item => item.lost))
{
if (await RecoverLostMessageAsync(queueName, listItem, message.Id).ConfigureAwait(false))
{
//Shift offset since we manipulated the end of the list and are using offsets
start += 1;
stop += 1;
}
}
if (listItems.Length < take)
{
break;
}
start -= take;
stop -= take;
}
}
catch (Exception e)
{
_log.Error(e, "Error in redis lost message service");
}
}
private async Task <RedisMessage <T> > GetMessageAsync(CancellationTokenSource cancellationsSource)
{
if (cancellationsSource.IsCancellationRequested)
{
return(null);
}
try
{
byte[] listItem = await _db.ListRightPopLeftPushAsync(_queueName, RedisQueueConventions.GetProcessingQueueName(_queueName)).ConfigureAwait(false);
if (listItem == null)
{
cancellationsSource.Cancel();
return(null);
}
var message = _serializer.Deserialize <RedisListItem <T> >(listItem.AsSpan());
var hashKey = RedisQueueConventions.GetMessageHashKey(_queueName, message.Id);
var tasks = new Task[]
{
_db.StringSetAsync(RedisQueueConventions.GetMessageExpirationKey(_queueName, message.Id), DateTimeOffset.Now.ToUnixTimeMilliseconds()),
_db.HashIncrementAsync(hashKey, RedisHashKeys.DeliveryCount)
};
await Task.WhenAll(tasks).ConfigureAwait(false);
var hash = await _db.HashGetAllAsync(hashKey).ConfigureAwait(false);
return(new RedisMessage <T>(listItem, message.Id, message.Body, hash, _queueName));
}
catch (RedisTimeoutException e)
{
_log.Error(e, "Error retrieving redis message");
return(null);
}
catch (RedisException e)
{
_log.Error(e, "Error retrieving redis message");
return(null);
}
}
internal async IAsyncEnumerable <RedisDeadletter <T> > PeekDeadlettersAsync(int limit)
{
if (limit >= 1)
{
limit--; //0 is the first element of the list, thus 0 will return 1
}
var values = await _db.ListRangeAsync(RedisQueueConventions.GetDeadLetterQueueName(_queueName), 0, limit).ConfigureAwait(false);
foreach (byte[] value in values)
{
var deadletter = new RedisDeadletter <T> {
Message = _serializer.Deserialize <RedisListItem <T> >(value.AsSpan())
};
var hash = RedisQueueConventions.GetMessageHashKey(_queueName, deadletter.Message.Id);
var hashes = await _db.HashGetAllAsync(hash).ConfigureAwait(false);
deadletter.HashEntries = hashes.ToStringDictionary();
yield return(deadletter);
}
}
private async Task <(bool lost, RedisListItem <T> message, RedisValue listItem)> HandlePotentiallyLostMessage(string queueName, byte[] listItem)
{
var message = _serializer.Deserialize <RedisListItem <T> >(listItem.AsSpan());
var lastProcessedKey = RedisQueueConventions.GetMessageExpirationKey(queueName, message.Id);
var hash = await _db.StringGetAsync(lastProcessedKey).ConfigureAwait(false);
if (!hash.IsNull)
{
var processTimeStamp = DateTimeOffset.FromUnixTimeMilliseconds(long.Parse(hash));
if (processTimeStamp + _messageTimeout < DateTimeOffset.Now)
{
//Message is lost or has exceeded maximum processing time
return(true, message, listItem);
}
}
else
{
await _db.StringSetAsync(lastProcessedKey, DateTimeOffset.Now.Add(-_messageTimeout).ToUnixTimeMilliseconds(), GetDelay() + GetDelay()).ConfigureAwait(false);
}
return(false, default, listItem);
internal async Task RequeueDeadletterAsync()
{
var deadLetterQueueName = RedisQueueConventions.GetDeadLetterQueueName(_queueName);
var deadLetterProcessingQueueName = RedisQueueConventions.GetProcessingQueueName(deadLetterQueueName);
byte[] listItem = await _db.ListRightPopLeftPushAsync(deadLetterQueueName, deadLetterProcessingQueueName).ConfigureAwait(false);
if (listItem == null)
{
return;
}
var message = _serializer.Deserialize <RedisListItem <T> >(listItem.AsSpan());
var hashKey = RedisQueueConventions.GetMessageHashKey(_queueName, message.Id);
var expirationKey = RedisQueueConventions.GetMessageExpirationKey(_queueName, message.Id);
await _db.HashDeleteAsync(hashKey, new RedisValue[] { expirationKey, RedisHashKeys.DeliveryCount }).ConfigureAwait(false);
await _db.ListRightPopLeftPushAsync(deadLetterProcessingQueueName, _queueName).ConfigureAwait(false);
await _db.PublishAsync(_queueName, 0, CommandFlags.FireAndForget).ConfigureAwait(false);
}
public async Task UploadAttachmentAsync(string queueName, string id, IMessageAttachment attachment, CancellationToken cancellationToken = default(CancellationToken))
{
var db = _multiplexer.GetDatabase(_configuration.DatabaseId);
var hash = new HashEntry[]
{
new HashEntry(FileName, attachment.Filename),
new HashEntry(ContentType, attachment.ContentType),
};
using (var memoryStream = new MemoryStream())
{
await attachment.Stream.CopyToAsync(memoryStream).ConfigureAwait(false);
await Task.WhenAll(
db.HashSetAsync(RedisQueueConventions.GetAttachmentMetadataKey(queueName, id), hash),
db.StringSetAsync(RedisQueueConventions.GetAttachmentBinaryKey(queueName, id), memoryStream.ToArray()))
.ConfigureAwait(false);
}
}
public RedisEventChannelReceiver(IConnectionMultiplexer connectionMultiplexer, IProcessingSettings settings, IMessageSerializer serializer, IEventSubscription <T> subscription, RedisConfiguration configuration, IHostConfiguration hostConfiguration, IMessageProcessor processor)
: base(connectionMultiplexer, RedisQueueConventions.GetSubscriptionQueueName(AutoMessageMapper.GetQueueName <T>(), subscription.Name), settings, serializer, configuration, hostConfiguration, processor)
{
_subscription = subscription;
_redisConfiguration = configuration;
}
internal Task <long> GetDeadletterMessageCount()
{
return(GetMessageCount(RedisQueueConventions.GetDeadLetterQueueName(_queueName)));
}
internal Task CompleteMessageAsync(RedisMessage <T> message)
{
return(Task.WhenAll(
_db.KeyDeleteAsync(new RedisKey[] { message.HashKey, message.ExpirationKey }),
_db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), message.RedisValue, -1)));
}
public Task DeadLetterAsync(int deadLetterLimit)
{
return(Task.WhenAll(
_db.ListLeftPushAsync(RedisQueueConventions.GetDeadLetterQueueName(_queueName), _redisMessage.RedisValue),
_db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), _redisMessage.RedisValue, -1)));
}
