internal Task AbandonMessageByErrorAsync(RedisMessage <T> message, Exception e) { return(Task.WhenAll( _db.HashSetAsync(message.HashKey, RedisHashKeys.Errors, $"{e.Message}\n{e.StackTrace}"), _db.ListLeftPushAsync(_queueName, message.RedisValue), _db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), message.RedisValue, -1))); }
internal Task DeleteDeadletterAsync(RedisDeadletter <T> deadletter) { var deadletterQueueName = RedisQueueConventions.GetDeadLetterQueueName(_queueName); var hash = RedisQueueConventions.GetMessageHashKey(_queueName, deadletter.Message.Id); return(Task.WhenAll(_db.KeyDeleteAsync(hash), _db.ListRemoveAsync(deadletterQueueName, _serializer.Serialize(deadletter.Message), -1))); }
private async Task <RedisMessage <T> > GetMessageAsync() { try { var listItem = await _db.ListRightPopLeftPushAsync(_queueName, RedisQueueConventions.GetProcessingQueueName(_queueName)).ConfigureAwait(false); if (listItem.IsNullOrEmpty) { return(null); } var message = _redisConfiguration.MessageSerializer.Deserialize <T>(listItem); var hashKey = RedisQueueConventions.GetMessageHashKey(_queueName, message.Id); var tasks = new Task[] { _db.StringSetAsync(RedisQueueConventions.GetMessageExpirationKey(_queueName, message.Id), DateTimeOffset.Now.ToUnixTimeMilliseconds()), _db.HashIncrementAsync(hashKey, RedisHashKeys.DeliveryCount, 1), }; await Task.WhenAll(tasks).ConfigureAwait(false); var hash = await _db.HashGetAllAsync(hashKey).ConfigureAwait(false); return(new RedisMessage <T>(listItem, message, hash, _queueName)); } catch (RedisTimeoutException e) { _hostConfiguration.Log.Error(e, "Error retrieving redis message"); return(null); } catch (RedisException e) { _hostConfiguration.Log.Error(e, "Error retrieving redis message"); return(null); } }
internal Task DeadletterMessageAsync(RedisMessage <T> message, int deadLetterLimit) { return(Task.WhenAll( _db.HashSetAsync(message.HashKey, "MaxDeliveryCountExceeded", $"DeliveryCount exceeded limit of {deadLetterLimit}"), _db.ListLeftPushAsync(RedisQueueConventions.GetDeadLetterQueueName(_queueName), message.RedisValue), _db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), message.RedisValue, -1))); }
public override async Task StartAsync(CancellationToken cancellationToken) { var db = ConnectionMultiplexer.GetDatabase(_redisConfiguration.DatabaseId); await db.SetAddAsync(RedisQueueConventions.GetSubscriptionKey(AutoMessageMapper.GetQueueName <T>()), _subscription.Name).ConfigureAwait(false); await base.StartAsync(cancellationToken).ConfigureAwait(false); }
public RedisMessage(RedisValue redisValue, string id, T message, HashEntry[] hashEntries, string queueName) { RedisValue = redisValue; HashKey = RedisQueueConventions.GetMessageHashKey(queueName, id); ExpirationKey = RedisQueueConventions.GetMessageExpirationKey(queueName, id); Message = message; HashEntries = hashEntries.ToStringDictionary(); }
public async Task <IMessageAttachment> GetAttachmentAsync(string queueName, string id, CancellationToken cancellationToken = default(CancellationToken)) { var db = _multiplexer.GetDatabase(_configuration.DatabaseId); var metadataHash = await db.HashGetAllAsync(RedisQueueConventions.GetAttachmentMetadataKey(queueName, id)).ConfigureAwait(false); var data = await db.StringGetAsync(RedisQueueConventions.GetAttachmentBinaryKey(queueName, id)).ConfigureAwait(false); var metadata = metadataHash.ToStringDictionary(); return(new MessageAttachment(metadata[FileName], metadata[ContentType], new MemoryStream(data))); }
public async Task PublishAsync <T>(T message) where T : IRedisEvent { var db = _multiplexer.GetDatabase(_configuration.DatabaseId); var queueName = AutoMessageMapper.GetQueueName <T>(); var subscriptions = await db.SetMembersAsync(RedisQueueConventions.GetSubscriptionKey(queueName)).ConfigureAwait(false); if (subscriptions == null) { return; } await Task.WhenAll(subscriptions.Select(sub => SendAsync(message, RedisQueueConventions.GetSubscriptionQueueName(queueName, sub)))).ConfigureAwait(false); }
private async Task UploadAttachment <T>(RedisListItem <T> message, string queueName, IDatabase db) where T : IRedisMessage { if (message.Body is ICommandWithAttachment attachmentMessage) { if (_attachmentProvider == null) { throw new AttachmentProviderMissingException(); } if (attachmentMessage.Attachment != null) { var attachmentId = Guid.NewGuid().ToString("N"); await db.HashSetAsync(RedisQueueConventions.GetMessageHashKey(queueName, message.Id), AttachmentUtility.AttachmentKey, attachmentId).ConfigureAwait(false); await _attachmentProvider.UploadAttachmentAsync(queueName, attachmentId, attachmentMessage.Attachment).ConfigureAwait(false); } } }
public async Task <bool> DeleteAttachmentAsync(string queueName, string id, CancellationToken cancellationToken = default(CancellationToken)) { var db = _multiplexer.GetDatabase(_configuration.DatabaseId); try { await Task.WhenAll( db.KeyDeleteAsync(RedisQueueConventions.GetAttachmentMetadataKey(queueName, id)), db.KeyDeleteAsync(RedisQueueConventions.GetAttachmentBinaryKey(queueName, id))) .ConfigureAwait(false); return(true); } catch (Exception) { return(false); } }
private async Task DetectAndHandleLostMessages(string queueName) { Debug.WriteLine($"Finding lost messages from {queueName}"); const int take = 50; var start = -take; var stop = -1; try { while (true) { var listItems = await _db.ListRangeAsync(RedisQueueConventions.GetProcessingQueueName(queueName), start, stop).ConfigureAwait(false); if (!listItems.Any()) { break; } Debug.WriteLine($"Found {listItems.Length} processing in {queueName}"); var checkedItems = await Task.WhenAll(listItems.Select(item => HandlePotentiallyLostMessage(queueName, item))).ConfigureAwait(false); foreach (var(lost, message, listItem) in checkedItems.Where(item => item.lost)) { if (await RecoverLostMessageAsync(queueName, listItem, message.Id).ConfigureAwait(false)) { //Shift offset since we manipulated the end of the list and are using offsets start += 1; stop += 1; } } if (listItems.Length < take) { break; } start -= take; stop -= take; } } catch (Exception e) { _log.Error(e, "Error in redis lost message service"); } }
private async Task <RedisMessage <T> > GetMessageAsync(CancellationTokenSource cancellationsSource) { if (cancellationsSource.IsCancellationRequested) { return(null); } try { byte[] listItem = await _db.ListRightPopLeftPushAsync(_queueName, RedisQueueConventions.GetProcessingQueueName(_queueName)).ConfigureAwait(false); if (listItem == null) { cancellationsSource.Cancel(); return(null); } var message = _serializer.Deserialize <RedisListItem <T> >(listItem.AsSpan()); var hashKey = RedisQueueConventions.GetMessageHashKey(_queueName, message.Id); var tasks = new Task[] { _db.StringSetAsync(RedisQueueConventions.GetMessageExpirationKey(_queueName, message.Id), DateTimeOffset.Now.ToUnixTimeMilliseconds()), _db.HashIncrementAsync(hashKey, RedisHashKeys.DeliveryCount) }; await Task.WhenAll(tasks).ConfigureAwait(false); var hash = await _db.HashGetAllAsync(hashKey).ConfigureAwait(false); return(new RedisMessage <T>(listItem, message.Id, message.Body, hash, _queueName)); } catch (RedisTimeoutException e) { _log.Error(e, "Error retrieving redis message"); return(null); } catch (RedisException e) { _log.Error(e, "Error retrieving redis message"); return(null); } }
internal async IAsyncEnumerable <RedisDeadletter <T> > PeekDeadlettersAsync(int limit) { if (limit >= 1) { limit--; //0 is the first element of the list, thus 0 will return 1 } var values = await _db.ListRangeAsync(RedisQueueConventions.GetDeadLetterQueueName(_queueName), 0, limit).ConfigureAwait(false); foreach (byte[] value in values) { var deadletter = new RedisDeadletter <T> { Message = _serializer.Deserialize <RedisListItem <T> >(value.AsSpan()) }; var hash = RedisQueueConventions.GetMessageHashKey(_queueName, deadletter.Message.Id); var hashes = await _db.HashGetAllAsync(hash).ConfigureAwait(false); deadletter.HashEntries = hashes.ToStringDictionary(); yield return(deadletter); } }
private async Task <(bool lost, RedisListItem <T> message, RedisValue listItem)> HandlePotentiallyLostMessage(string queueName, byte[] listItem) { var message = _serializer.Deserialize <RedisListItem <T> >(listItem.AsSpan()); var lastProcessedKey = RedisQueueConventions.GetMessageExpirationKey(queueName, message.Id); var hash = await _db.StringGetAsync(lastProcessedKey).ConfigureAwait(false); if (!hash.IsNull) { var processTimeStamp = DateTimeOffset.FromUnixTimeMilliseconds(long.Parse(hash)); if (processTimeStamp + _messageTimeout < DateTimeOffset.Now) { //Message is lost or has exceeded maximum processing time return(true, message, listItem); } } else { await _db.StringSetAsync(lastProcessedKey, DateTimeOffset.Now.Add(-_messageTimeout).ToUnixTimeMilliseconds(), GetDelay() + GetDelay()).ConfigureAwait(false); } return(false, default, listItem);
internal async Task RequeueDeadletterAsync() { var deadLetterQueueName = RedisQueueConventions.GetDeadLetterQueueName(_queueName); var deadLetterProcessingQueueName = RedisQueueConventions.GetProcessingQueueName(deadLetterQueueName); byte[] listItem = await _db.ListRightPopLeftPushAsync(deadLetterQueueName, deadLetterProcessingQueueName).ConfigureAwait(false); if (listItem == null) { return; } var message = _serializer.Deserialize <RedisListItem <T> >(listItem.AsSpan()); var hashKey = RedisQueueConventions.GetMessageHashKey(_queueName, message.Id); var expirationKey = RedisQueueConventions.GetMessageExpirationKey(_queueName, message.Id); await _db.HashDeleteAsync(hashKey, new RedisValue[] { expirationKey, RedisHashKeys.DeliveryCount }).ConfigureAwait(false); await _db.ListRightPopLeftPushAsync(deadLetterProcessingQueueName, _queueName).ConfigureAwait(false); await _db.PublishAsync(_queueName, 0, CommandFlags.FireAndForget).ConfigureAwait(false); }
public async Task UploadAttachmentAsync(string queueName, string id, IMessageAttachment attachment, CancellationToken cancellationToken = default(CancellationToken)) { var db = _multiplexer.GetDatabase(_configuration.DatabaseId); var hash = new HashEntry[] { new HashEntry(FileName, attachment.Filename), new HashEntry(ContentType, attachment.ContentType), }; using (var memoryStream = new MemoryStream()) { await attachment.Stream.CopyToAsync(memoryStream).ConfigureAwait(false); await Task.WhenAll( db.HashSetAsync(RedisQueueConventions.GetAttachmentMetadataKey(queueName, id), hash), db.StringSetAsync(RedisQueueConventions.GetAttachmentBinaryKey(queueName, id), memoryStream.ToArray())) .ConfigureAwait(false); } }
public RedisEventChannelReceiver(IConnectionMultiplexer connectionMultiplexer, IProcessingSettings settings, IMessageSerializer serializer, IEventSubscription <T> subscription, RedisConfiguration configuration, IHostConfiguration hostConfiguration, IMessageProcessor processor) : base(connectionMultiplexer, RedisQueueConventions.GetSubscriptionQueueName(AutoMessageMapper.GetQueueName <T>(), subscription.Name), settings, serializer, configuration, hostConfiguration, processor) { _subscription = subscription; _redisConfiguration = configuration; }
internal Task <long> GetDeadletterMessageCount() { return(GetMessageCount(RedisQueueConventions.GetDeadLetterQueueName(_queueName))); }
internal Task CompleteMessageAsync(RedisMessage <T> message) { return(Task.WhenAll( _db.KeyDeleteAsync(new RedisKey[] { message.HashKey, message.ExpirationKey }), _db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), message.RedisValue, -1))); }
public Task DeadLetterAsync(int deadLetterLimit) { return(Task.WhenAll( _db.ListLeftPushAsync(RedisQueueConventions.GetDeadLetterQueueName(_queueName), _redisMessage.RedisValue), _db.ListRemoveAsync(RedisQueueConventions.GetProcessingQueueName(_queueName), _redisMessage.RedisValue, -1))); }