/// <summary>
/// Receives the next message from the in-mem prefetch buffer, possibly trying to prefetch into the buffer first
/// </summary>
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
if (_incomingMessages.TryDequeue(out var next))
{
var envelope = next.Envelope;
var transportMessage = new TransportMessage(envelope.Headers, envelope.Body);
if (transportMessage.Headers.TryGetValue(Headers.TimeToBeReceived, out var timeToBeReceivedString))
{
if (transportMessage.Headers.TryGetValue(Headers.SentTime, out var sentTimeString))
{
if (TimeSpan.TryParse(timeToBeReceivedString, CultureInfo.InvariantCulture, out var timeToBeReceived))
{
if (DateTimeOffset.TryParse(sentTimeString, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var sentTime))
{
var messageExpired = _rebusTime.Now > sentTime + timeToBeReceived;
if (messageExpired)
{
using (next)
{
next.Complete();
}
continue;
}
}
}
}
}
context.OnCompleted(async _ => next.Complete());
context.OnDisposed(_ =>
{
if (next.IsCompleted)
{
next.Dispose();
}
else
{
_incomingMessages.Enqueue(next);
}
});
return(transportMessage);
}
ReceiveNextBatch();
// just let backoff strategy do its thing, if there's nothing at this point
if (!_incomingMessages.Any())
{
return(null);
}
}
}
/// <summary>
/// Receives the next message (if any) from the transport's input queue <see cref="ITransport.Address"/>
/// </summary>
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
if (_inputQueueName == null)
{
throw new InvalidOperationException("This Azure Storage Queues transport does not have an input queue, hence it is not possible to receive anything");
}
var inputQueue = GetQueue(_inputQueueName);
var cloudQueueMessage = await inputQueue.GetMessageAsync(_initialVisibilityDelay, new QueueRequestOptions(), new OperationContext(), cancellationToken);
if (cloudQueueMessage == null)
{
return(null);
}
context.OnCompleted(async() =>
{
// if we get this far, don't pass on the cancellation token
// ReSharper disable once MethodSupportsCancellation
await inputQueue.DeleteMessageAsync(cloudQueueMessage);
});
context.OnAborted(() =>
{
const MessageUpdateFields fields = MessageUpdateFields.Visibility;
var visibilityTimeout = TimeSpan.FromSeconds(0);
AsyncHelpers.RunSync(() => inputQueue.UpdateMessageAsync(cloudQueueMessage, visibilityTimeout, fields));
});
return(Deserialize(cloudQueueMessage));
}
/// <summary>
/// Receives the next message (if any) from the transport's input queue <see cref="ITransport.Address"/>
/// </summary>
public async Task <TransportMessage> Receive(ITransactionContext context)
{
if (_inputQueueName == null)
{
throw new InvalidOperationException("This Azure Storage Queues transport does not have an input queue, hence it is not possible to reveive anything");
}
var inputQueue = GetQueue(_inputQueueName);
var cloudQueueMessage = await inputQueue.GetMessageAsync(_initialVisibilityDelay, new QueueRequestOptions(), new OperationContext());
if (cloudQueueMessage == null)
{
return(null);
}
context.OnCompleted(async() =>
{
await inputQueue.DeleteMessageAsync(cloudQueueMessage);
});
context.OnAborted(() =>
{
inputQueue.UpdateMessage(cloudQueueMessage, TimeSpan.FromSeconds(0), MessageUpdateFields.Visibility);
});
return(Deserialize(cloudQueueMessage));
}
/// <summary>
/// Receives the next transport message or null if none is available. Can block if it wants to, just respect the <paramref name="cancellationToken"/>
/// </summary>
public override async Task<TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
if (amqpReceiver == null) return null;
AmqpLite.Message msg = await amqpReceiver.ReceiveAsync(TimeSpan.FromSeconds(0.5));
if (msg == null) return null;
if (!(msg.Properties?.AbsoluteExpiryTime != DateTime.MinValue && msg.Properties?.AbsoluteExpiryTime.ToUniversalTime() > DateTime.Now.ToUniversalTime())) return null;
context.OnCompleted(async ctx =>
{
amqpReceiver.Accept(msg);
});
context.OnAborted(async ctx =>
{
amqpReceiver.Modify(msg, true);
});
context.OnDisposed(async ctx =>
{
//amqpReceiver.Close();
});
var result = new TransportMessage(GetHeaders(msg), GetBytes(msg.Body.ToString()));
return result;
}
public async Task <TransportMessage> Receive(ITransactionContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (Address == null)
{
throw new InvalidOperationException("This Amazon SQS transport does not have an input queue, hence it is not possible to reveive anything");
}
var client = GetClientFromTransactionContext(context);
var request = new ReceiveMessageRequest(_queueUrl)
{
MaxNumberOfMessages = 1,
WaitTimeSeconds = 1,
AttributeNames = new List <string>(new[] { "All" }),
MessageAttributeNames = new List <string>(new[] { "All" })
};
var response = await client.ReceiveMessageAsync(request);
if (!response.Messages.Any())
{
return(null);
}
var message = response.Messages.First();
var renewalTask = CreateRenewalTaskForMessage(message, client);
context.OnCompleted(async() =>
{
renewalTask.Dispose();
await client.DeleteMessageAsync(new DeleteMessageRequest(_queueUrl, message.ReceiptHandle));
});
context.OnAborted(() =>
{
renewalTask.Dispose();
client.ChangeMessageVisibility(_queueUrl, message.ReceiptHandle, 0);
});
if (MessageIsExpired(message))
{
await client.DeleteMessageAsync(new DeleteMessageRequest(_queueUrl, message.ReceiptHandle));
return(null);
}
renewalTask.Start();
var transportMessage = GetTransportMessage(message);
return(transportMessage);
}
public override Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
context.OnCompleted(ctx =>
{
Interlocked.Decrement(ref InProcessMessageCount);
return(Task.CompletedTask);
});
return(base.Receive(context, cancellationToken));
}
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
var scope = new TransactionScope(TransactionScopeOption.Required, _transactionOptions,
TransactionScopeAsyncFlowOption.Enabled);
context.OnCompleted(async ctx => scope.Complete());
context.OnDisposed(ctx => scope.Dispose());
// stash current tx so we can re-attach it later
context.Items[TransactionScopeIncomingStep.CurrentTransactionContextKey] = Transaction.Current;
return(await _transport.Receive(context, cancellationToken));
}
public async Task <TransportMessage> Receive(ITransactionContext context)
{
if (Address == null)
{
throw new InvalidOperationException("This RabbitMQ transport does not have an input queue, hence it is not possible to reveive anything");
}
var model = GetModel(context);
var result = model.BasicGet(Address, false);
if (result == null)
{
return(null);
}
var deliveryTag = result.DeliveryTag;
context.OnCompleted(async() =>
{
model.BasicAck(deliveryTag, false);
});
context.OnAborted(() =>
{
model.BasicNack(deliveryTag, false, true);
});
var headers = result.BasicProperties.Headers
.ToDictionary(kvp => kvp.Key, kvp =>
{
var headerValue = kvp.Value;
if (headerValue is byte[])
{
var stringHeaderValue = HeaderValueEncoding.GetString((byte[])headerValue);
return(stringHeaderValue);
}
return(headerValue.ToString());
});
return(new TransportMessage(headers, result.Body));
}
static void SetUpCompletion(ITransactionContext context, CloudQueueMessage cloudQueueMessage, CloudQueue inputQueue)
{
var messageId = cloudQueueMessage.Id;
var popReceipt = cloudQueueMessage.PopReceipt;
context.OnCompleted(async() =>
{
try
{
// if we get this far, don't pass on the cancellation token
// ReSharper disable once MethodSupportsCancellation
await inputQueue.DeleteMessageAsync(
messageId,
popReceipt,
ExponentialRetryRequestOptions,
new OperationContext()
);
}
catch (Exception exception)
{
throw new RebusApplicationException(exception, $"Could not delete message with ID {messageId} and pop receipt {popReceipt} from the input queue");
}
});
context.OnAborted(() =>
{
const MessageUpdateFields fields = MessageUpdateFields.Visibility;
var visibilityTimeout = TimeSpan.FromSeconds(0);
AsyncHelpers.RunSync(async() =>
{
// ignore if this fails
try
{
await inputQueue.UpdateMessageAsync(cloudQueueMessage, visibilityTimeout, fields);
}
catch { }
});
});
}
public async Task <TransportMessage> Receive(ITransactionContext context)
{
var model = GetModel(context);
var result = model.BasicGet(Address, false);
if (result == null)
{
return(null);
}
var deliveryTag = result.DeliveryTag;
context.OnCompleted(async() =>
{
model.BasicAck(deliveryTag, false);
});
context.OnAborted(() =>
{
model.BasicNack(deliveryTag, false, true);
});
var headers = result.BasicProperties.Headers
.ToDictionary(kvp => kvp.Key, kvp =>
{
var headerValue = kvp.Value;
if (headerValue is byte[])
{
var stringHeaderValue = HeaderValueEncoding.GetString((byte[])headerValue);
return(stringHeaderValue);
}
return(headerValue.ToString());
});
return(new TransportMessage(headers, result.Body));
}
/// <summary>
/// Receives the next message (if any) from the transport's input queue <see cref="ITransport.Address"/>
/// </summary>
public async Task <TransportMessage> Receive(ITransactionContext context)
{
var inputQueue = GetQueue(_inputQueueName);
var cloudQueueMessage = await inputQueue.GetMessageAsync(_initialVisibilityDelay, new QueueRequestOptions(), new OperationContext());
if (cloudQueueMessage == null)
{
return(null);
}
context.OnCompleted(async() =>
{
await inputQueue.DeleteMessageAsync(cloudQueueMessage);
});
context.OnAborted(() =>
{
inputQueue.UpdateMessage(cloudQueueMessage, TimeSpan.FromSeconds(0), MessageUpdateFields.Visibility);
});
return(Deserialize(cloudQueueMessage));
}
/// <summary>
/// Responsible for releasing the lease on message failure and removing the message on transaction commit
/// </summary>
/// <param name="context">Transaction context of the message processing</param>
/// <param name="messageId">Identifier of the message currently being processed</param>
/// <param name="cancellationToken">Token to abort processing</param>
private void ApplyTransactionSemantics(ITransactionContext context, long messageId, CancellationToken cancellationToken)
{
AutomaticLeaseRenewer renewal = null;
if (_automaticLeaseRenewal)
{
renewal = new AutomaticLeaseRenewer(
this, ReceiveTableName.QualifiedName, messageId, ConnectionProvider, _automaticLeaseRenewalInterval, _leaseInterval, cancellationToken);
}
context.OnAborted(_ =>
{
renewal?.Dispose();
try
{
AsyncHelpers.RunSync(() => UpdateLease(ConnectionProvider, ReceiveTableName.QualifiedName,
messageId, null, cancellationToken));
}
catch (Exception ex)
{
Log.Error(ex, "While Resetting Lease");
}
});
context.OnCompleted(async _ =>
{
renewal?.Dispose();
try
{
await DeleteMessage(messageId, cancellationToken);
}
catch (Exception ex)
{
Log.Error(ex, "While Deleteing Message");
}
});
}
/// <summary>
/// Receives the next message from the logical input queue by loading the next file from the corresponding directory,
/// deserializing it, deleting it when the transaction is committed.
/// </summary>
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
TransportMessage receivedTransportMessage = null;
string fullPath = null;
bool loopAgain = false;
do
{
loopAgain = false;
receivedTransportMessage = null;
fullPath = _fileQueue.Dequeue(cancellationToken);
if (!string.IsNullOrEmpty(fullPath))
{
var jsonText = await TransportHelper.ReadAllText(fullPath);
receivedTransportMessage = Deserialize(jsonText);
if (!_CheckIsValid(fullPath, receivedTransportMessage))
{
File.Delete(fullPath);
loopAgain = true;
}
}
} while (loopAgain);
if (receivedTransportMessage != null)
{
context.OnCompleted(async() => {
await TransportHelper.DeleteFile(fullPath);
});
context.OnAborted(async() => {
TransportHelper.RenameToError(fullPath, out var _);
});
}
return(receivedTransportMessage);
}
public async Task<TransportMessage> Receive(ITransactionContext context)
{
if (context == null) throw new ArgumentNullException("context");
if (_inputQueueAddress == null)
{
throw new InvalidOperationException("This Amazon SQS transport does not have an input queue, hence it is not possible to reveive anything");
}
var client = GetClientFromTransactionContext(context);
var response = await client.ReceiveMessageAsync(new ReceiveMessageRequest(_queueUrl)
{
MaxNumberOfMessages = 1,
WaitTimeSeconds = 1,
AttributeNames = new List<string>(new[] { "All" }),
MessageAttributeNames = new List<string>(new[] { "All" })
});
if (response.Messages.Any())
{
var message = response.Messages.First();
var renewalTask = CreateRenewalTaskForMessage(message, client);
context.OnCompleted(async () =>
{
renewalTask.Dispose();
var result = await client.DeleteMessageBatchAsync(new DeleteMessageBatchRequest(_queueUrl,
response.Messages
.Select(m => new DeleteMessageBatchRequestEntry(m.MessageId, m.ReceiptHandle))
.ToList()));
if (result.Failed.Any())
{
GenerateErrorsAndLog(result);
}
});
context.OnAborted(() =>
{
renewalTask.Dispose();
var result = client.ChangeMessageVisibilityBatch(new ChangeMessageVisibilityBatchRequest(_queueUrl,
response.Messages
.Select(m => new ChangeMessageVisibilityBatchRequestEntry(m.MessageId, m.ReceiptHandle)
{
VisibilityTimeout = 0
}).ToList()));
if (result.Failed.Any())
{
GenerateErrorsAndLog(result);
}
});
if (MessageIsExpired(message))
{
await client.DeleteMessageAsync(new DeleteMessageRequest(_queueUrl, message.ReceiptHandle));
return null;
}
renewalTask.Start();
var transportMessage = GetTransportMessage(message);
return transportMessage;
}
return null;
}
/// <summary>
/// Receives the next message from the input queue. Returns null if no message was available
/// </summary>
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
var receivedMessage = await ReceiveInternal().ConfigureAwait(false);
if (receivedMessage == null)
{
return(null);
}
var message = receivedMessage.Message;
var messageReceiver = receivedMessage.MessageReceiver;
if (!message.SystemProperties.IsLockTokenSet)
{
throw new RebusApplicationException($"OMG that's weird - message with ID {message.MessageId} does not have a lock token!");
}
var lockToken = message.SystemProperties.LockToken;
var messageId = message.MessageId;
if (AutomaticallyRenewPeekLock && !_prefetchingEnabled)
{
var now = DateTime.UtcNow;
var leaseDuration = message.SystemProperties.LockedUntilUtc - now;
var lockRenewalInterval = TimeSpan.FromMinutes(0.5 * leaseDuration.TotalMinutes);
var cancellationTokenSource = new CancellationTokenSource();
var renewalTask = _asyncTaskFactory
.Create(description: $"RenewPeekLock-{messageId}",
action: () => RenewPeekLock(messageReceiver, messageId, lockToken, cancellationTokenSource),
intervalSeconds: (int)lockRenewalInterval.TotalSeconds,
prettyInsignificant: true
);
// be sure to stop the renewal task regardless of whether we're committing or aborting
context.OnCommitted(async() => renewalTask.Dispose());
context.OnAborted(() => renewalTask.Dispose());
context.OnDisposed(() =>
{
renewalTask.Dispose();
cancellationTokenSource.Dispose();
});
cancellationTokenSource.Token.Register(renewalTask.Dispose);
renewalTask.Start();
}
context.OnCompleted(async() =>
{
try
{
await messageReceiver.CompleteAsync(lockToken).ConfigureAwait(false);
}
catch (Exception exception)
{
throw new RebusApplicationException(exception,
$"Could not complete message with ID {message.MessageId} and lock token {lockToken}");
}
});
context.OnAborted(() =>
{
try
{
AsyncHelpers.RunSync(async() => await messageReceiver.AbandonAsync(lockToken).ConfigureAwait(false));
}
catch (Exception exception)
{
throw new RebusApplicationException(exception,
$"Could not abandon message with ID {message.MessageId} and lock token {lockToken}");
}
});
var userProperties = message.UserProperties;
var headers = userProperties.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.ToString());
var body = message.Body;
return(new TransportMessage(headers, body));
}
/// <inheritdoc />
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
if (Address == null)
{
throw new InvalidOperationException("This RabbitMQ transport does not have an input queue - therefore, it is not possible to receive anything");
}
try
{
if (!_consumers.TryDequeue(out var consumer))
{
consumer = InitializeConsumer();
}
else
{
try
{
// When a consumer is dequeued from the the "consumers" pool, it might be bound to a queue, which does not exist anymore,
// eg. expired and deleted by RabittMQ server policy). In this case this calling QueueDeclarePassive will result in
// an OperationInterruptedException and "consumer.Model.IsOpen" will be set to false (this is handled later in the code by
// disposing this consumer). There is no need to handle this exception. The logic of InitializeConsumer() will make sure
// that the queue is recreated later based on assumption about how ReBus is handling null-result of ITransport.Receive().
consumer?.Model.QueueDeclarePassive(Address);
}
catch { }
}
if (consumer == null)
{
// initialization must have failed
return(null);
}
if (!consumer.Model.IsOpen)
{
consumer.Dispose();
return(null);
}
context.OnDisposed((tc) => _consumers.Enqueue(consumer));
if (!consumer.Queue.Dequeue(TwoSeconds, out var result))
{
return(null);
}
if (result == null)
{
return(null);
}
// ensure we use the consumer's model throughtout the handling of this message
context.Items[CurrentModelItemsKey] = consumer.Model;
var deliveryTag = result.DeliveryTag;
context.OnCompleted(async(tc) =>
{
var model = GetModel(context);
model.BasicAck(deliveryTag, false);
});
context.OnAborted((tc) =>
{
// we might not be able to do this, but it doesn't matter that much if it succeeds
try
{
var model = GetModel(context);
model.BasicNack(deliveryTag, false, true);
}
catch { }
});
return(CreateTransportMessage(result.BasicProperties, result.Body));
}
catch (EndOfStreamException)
{
return(null);
}
catch (Exception exception)
{
Thread.Sleep(1000);
throw new RebusApplicationException(exception, $"Unexpected exception thrown while trying to dequeue a message from rabbitmq, queue address: {Address}");
}
}
public async Task<TransportMessage> Receive(ITransactionContext context)
{
var model = GetModel(context);
var result = model.BasicGet(Address, false);
if (result == null) return null;
var deliveryTag = result.DeliveryTag;
context.OnCompleted(async () =>
{
model.BasicAck(deliveryTag, false);
});
context.OnAborted(() =>
{
model.BasicNack(deliveryTag, false, true);
});
var headers = result.BasicProperties.Headers
.ToDictionary(kvp => kvp.Key, kvp =>
{
var headerValue = kvp.Value;
if (headerValue is byte[])
{
var stringHeaderValue = HeaderValueEncoding.GetString((byte[])headerValue);
return stringHeaderValue;
}
return headerValue.ToString();
});
return new TransportMessage(headers, result.Body);
}
public async Task<TransportMessage> Receive(ITransactionContext context)
{
if (_inputQueueAddress == null)
{
throw new InvalidOperationException("This Azure Service Bus transport does not have an input queue, hence it is not possible to reveive anything");
}
using (await _bottleneck.Enter())
{
var brokeredMessage = await ReceiveBrokeredMessage();
if (brokeredMessage == null) return null;
var headers = brokeredMessage.Properties
.Where(kvp => kvp.Value is string)
.ToDictionary(kvp => kvp.Key, kvp => (string)kvp.Value);
var messageId = headers.GetValueOrNull(Headers.MessageId);
var leaseDuration = (brokeredMessage.LockedUntilUtc - DateTime.UtcNow);
var lockRenewalInterval = TimeSpan.FromMinutes(0.8 * leaseDuration.TotalMinutes);
var renewalTask = GetRenewalTaskOrFakeDisposable(messageId, brokeredMessage, lockRenewalInterval);
context.OnAborted(() =>
{
renewalTask.Dispose();
try
{
brokeredMessage.Abandon();
}
catch (Exception exception)
{
// if it fails, it'll be back on the queue anyway....
_log.Warn("Could not abandon message: {0}", exception);
}
});
context.OnCommitted(async () => renewalTask.Dispose());
context.OnCompleted(async () =>
{
await brokeredMessage.CompleteAsync();
});
context.OnDisposed(() =>
{
renewalTask.Dispose();
brokeredMessage.Dispose();
});
using (var memoryStream = new MemoryStream())
{
await brokeredMessage.GetBody<Stream>().CopyToAsync(memoryStream);
return new TransportMessage(headers, memoryStream.ToArray());
}
}
}
public async Task<TransportMessage> Receive(ITransactionContext context)
{
using (await _bottleneck.Enter())
{
var brokeredMessage = await ReceiveBrokeredMessage();
if (brokeredMessage == null) return null;
var headers = brokeredMessage.Properties
.Where(kvp => kvp.Value is string)
.ToDictionary(kvp => kvp.Key, kvp => (string)kvp.Value);
var messageId = headers.GetValueOrNull(Headers.MessageId);
_log.Debug("Received brokered message with ID {0}", messageId);
var leaseDuration = (brokeredMessage.LockedUntilUtc - DateTime.UtcNow);
var lockRenewalInterval = TimeSpan.FromMinutes(0.8 * leaseDuration.TotalMinutes);
var renewalTask = GetRenewalTaskOrFakeDisposable(messageId, brokeredMessage, lockRenewalInterval);
context.OnAborted(() =>
{
renewalTask.Dispose();
_log.Debug("Abandoning message with ID {0}", messageId);
try
{
brokeredMessage.Abandon();
}
catch (Exception exception)
{
// if it fails, it'll be back on the queue anyway....
_log.Warn("Could not abandon message: {0}", exception);
}
});
context.OnCommitted(async () =>
{
renewalTask.Dispose();
});
context.OnCompleted(async () =>
{
_log.Debug("Completing message with ID {0}", messageId);
await GetRetrier().Execute(() => brokeredMessage.CompleteAsync());
});
context.OnDisposed(() =>
{
renewalTask.Dispose();
_log.Debug("Disposing message with ID {0}", messageId);
brokeredMessage.Dispose();
});
return new TransportMessage(headers, brokeredMessage.GetBody<byte[]>());
}
}
public async Task<TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken = default(CancellationToken))
{
if (context == null) throw new ArgumentNullException(nameof(context));
if (Address == null)
{
throw new InvalidOperationException("This Amazon SQS transport does not have an input queue, hence it is not possible to reveive anything");
}
var client = GetClientFromTransactionContext(context);
var request = new ReceiveMessageRequest(_queueUrl)
{
MaxNumberOfMessages = 1,
WaitTimeSeconds = 1,
AttributeNames = new List<string>(new[] { "All" }),
MessageAttributeNames = new List<string>(new[] { "All" })
};
var response = await client.ReceiveMessageAsync(request, cancellationToken);
if (!response.Messages.Any()) return null;
var message = response.Messages.First();
var renewalTask = CreateRenewalTaskForMessage(message, client);
context.OnCompleted(async () =>
{
renewalTask.Dispose();
// if we get this far, we don't want to pass on the cancellation token
await client.DeleteMessageAsync(new DeleteMessageRequest(_queueUrl, message.ReceiptHandle));
});
context.OnAborted(() =>
{
renewalTask.Dispose();
client.ChangeMessageVisibility(_queueUrl, message.ReceiptHandle, 0);
});
if (MessageIsExpired(message))
{
// if the message is expired , we don't want to pass on the cancellation token
await client.DeleteMessageAsync(new DeleteMessageRequest(_queueUrl, message.ReceiptHandle));
return null;
}
renewalTask.Start();
var transportMessage = GetTransportMessage(message);
return transportMessage;
}
/// <summary>
/// Receives the next message (if any) from the transport's input queue <see cref="ITransport.Address"/>
/// </summary>
public async Task<TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken = default(CancellationToken))
{
if (_inputQueueName == null)
{
throw new InvalidOperationException("This Azure Storage Queues transport does not have an input queue, hence it is not possible to reveive anything");
}
var inputQueue = GetQueue(_inputQueueName);
var cloudQueueMessage = await inputQueue.GetMessageAsync(_initialVisibilityDelay, new QueueRequestOptions(), new OperationContext(), cancellationToken);
if (cloudQueueMessage == null) return null;
context.OnCompleted(async () =>
{
// if we get this far, don't pass on the cancellation token
await inputQueue.DeleteMessageAsync(cloudQueueMessage);
});
context.OnAborted(() =>
{
inputQueue.UpdateMessage(cloudQueueMessage, TimeSpan.FromSeconds(0), MessageUpdateFields.Visibility);
});
return Deserialize(cloudQueueMessage);
}
/// <summary>
/// Receives the next message (if any) from the transport's input queue <see cref="ITransport.Address"/>
/// </summary>
public async Task<TransportMessage> Receive(ITransactionContext context)
{
var inputQueue = GetQueue(_inputQueueName);
var cloudQueueMessage = await inputQueue.GetMessageAsync(_initialVisibilityDelay, new QueueRequestOptions(), new OperationContext());
if (cloudQueueMessage == null) return null;
context.OnCompleted(async () =>
{
await inputQueue.DeleteMessageAsync(cloudQueueMessage);
});
context.OnAborted(() =>
{
inputQueue.UpdateMessage(cloudQueueMessage, TimeSpan.FromSeconds(0), MessageUpdateFields.Visibility);
});
return Deserialize(cloudQueueMessage);
}
public async Task <TransportMessage> Receive(ITransactionContext context)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (_inputQueueAddress == null)
{
throw new InvalidOperationException("This Amazon SQS transport does not have an input queue, hence it is not possible to reveive anything");
}
var client = GetClientFromTransactionContext(context);
var response = await client.ReceiveMessageAsync(new ReceiveMessageRequest(_queueUrl)
{
MaxNumberOfMessages = 1,
WaitTimeSeconds = 1,
AttributeNames = new List <string>(new[] { "All" }),
MessageAttributeNames = new List <string>(new[] { "All" })
});
if (response.Messages.Any())
{
var message = response.Messages.First();
var renewalTask = CreateRenewalTaskForMessage(message, client);
context.OnCompleted(async() =>
{
renewalTask.Dispose();
var result = await client.DeleteMessageBatchAsync(new DeleteMessageBatchRequest(_queueUrl,
response.Messages
.Select(m => new DeleteMessageBatchRequestEntry(m.MessageId, m.ReceiptHandle))
.ToList()));
if (result.Failed.Any())
{
GenerateErrorsAndLog(result);
}
});
context.OnAborted(() =>
{
renewalTask.Dispose();
var result = client.ChangeMessageVisibilityBatch(new ChangeMessageVisibilityBatchRequest(_queueUrl,
response.Messages
.Select(m => new ChangeMessageVisibilityBatchRequestEntry(m.MessageId, m.ReceiptHandle)
{
VisibilityTimeout = 0
}).ToList()));
if (result.Failed.Any())
{
GenerateErrorsAndLog(result);
}
});
if (MessageIsExpired(message))
{
await client.DeleteMessageAsync(new DeleteMessageRequest(_queueUrl, message.ReceiptHandle));
return(null);
}
renewalTask.Start();
var transportMessage = GetTransportMessage(message);
return(transportMessage);
}
return(null);
}
/// <summary>
/// Received the next available transport message from the input queue via MSMQ. Will create a new <see cref="MessageQueueTransaction"/> and stash
/// it under the <see cref="CurrentTransactionKey"/> key in the given <paramref name="context"/>. If one already exists, an exception will be thrown
/// (because we should never have to receive multiple messages in the same transaction)
/// </summary>
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (_inputQueueName == null)
{
throw new InvalidOperationException("This MSMQ transport does not have an input queue, hence it is not possible to reveive anything");
}
var queue = GetInputQueue();
if (context.Items.ContainsKey(CurrentTransactionKey))
{
throw new InvalidOperationException("Tried to receive with an already existing MSMQ queue transaction - although it is possible with MSMQ to do so, with Rebus it is an indication that something is wrong!");
}
var messageQueueTransaction = new MessageQueueTransaction();
messageQueueTransaction.Begin();
context.OnDisposed(ctx => messageQueueTransaction.Dispose());
context.Items[CurrentTransactionKey] = messageQueueTransaction;
try
{
var message = queue.Receive(TimeSpan.FromSeconds(0.5), messageQueueTransaction);
if (message == null)
{
messageQueueTransaction.Abort();
return(null);
}
context.OnCompleted(async ctx => messageQueueTransaction.Commit());
context.OnDisposed(ctx => message.Dispose());
var headers = _msmqHeaderSerializer.Deserialize(message) ?? new Dictionary <string, string>();
var body = new byte[message.BodyStream.Length];
await message.BodyStream.ReadAsync(body, 0, body.Length, cancellationToken);
return(new TransportMessage(headers, body));
}
catch (MessageQueueException exception)
{
if (exception.MessageQueueErrorCode == MessageQueueErrorCode.IOTimeout)
{
return(null);
}
if (exception.MessageQueueErrorCode == MessageQueueErrorCode.InvalidHandle)
{
_log.Warn("Queue handle for {queueName} was invalid - will try to reinitialize the queue", _inputQueueName);
ReinitializeInputQueue();
return(null);
}
if (exception.MessageQueueErrorCode == MessageQueueErrorCode.QueueDeleted)
{
_log.Warn("Queue {queueName} was deleted - will not receive any more messages", _inputQueueName);
return(null);
}
throw new IOException($"Could not receive next message from MSMQ queue '{_inputQueueName}'", exception);
}
}
public override async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
if (_subscriberClient == null)
{
return(null);
}
ReceivedMessage receivedMessage = null;
try
{
var response = await _subscriberClient.PullAsync(
new PullRequest()
{
SubscriptionAsSubscriptionName = _subscriptionName, MaxMessages = 1
},
CallSettings.FromCancellationToken(cancellationToken)
);
receivedMessage = response.ReceivedMessages.FirstOrDefault();
}
catch (RpcException ex) when(ex.Status.StatusCode == StatusCode.Unavailable)
{
throw new RebusApplicationException(ex, "GooglePubSub UNAVAILABLE due to too many concurrent pull requests pending for the given subscription");
}
catch (RpcException ex) when(ex.StatusCode == StatusCode.Cancelled)
{
if (!cancellationToken.IsCancellationRequested)
{
//Rebus has not ordered this cancellation - therefore throwing
throw new RebusApplicationException(ex, "Cancelled when fetching messages from GooglePubSub");
}
}
catch (Exception ex)
{
throw new RebusApplicationException(ex, "Failed when fetching messages from GooglePubSub");
}
if (receivedMessage == null)
{
return(null);
}
var receivedTransportMessage = receivedMessage.ToRebusTransportMessage();
var utcNow = DateTimeOffset.UtcNow;
if (receivedTransportMessage.IsExpired(utcNow))
{
Log.Debug($"Discarded message {string.Join(",", receivedTransportMessage.Headers.Select(a => a.Key + " : " + a.Value).ToArray())} because message expired {receivedTransportMessage.AbsoluteExpiryTimeUtc()} which is lesser than current time {utcNow}");
return(null);
}
context.OnCompleted(async ctx =>
{
await _subscriberClient.AcknowledgeAsync(_subscriptionName, new[] { receivedMessage.AckId });
});
context.OnAborted(async ctx =>
{
await _subscriberClient.ModifyAckDeadlineAsync(_subscriptionName, new[] { receivedMessage.AckId }, 0);
});
return(receivedTransportMessage);
}
/// <summary>
/// Receives the next message from the input queue. Returns null if no message was available
/// </summary>
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
var receivedMessage = await ReceiveInternal().ConfigureAwait(false);
if (receivedMessage == null)
{
return(null);
}
var message = receivedMessage.Message;
var messageReceiver = receivedMessage.MessageReceiver;
if (!message.SystemProperties.IsLockTokenSet)
{
throw new RebusApplicationException($"OMG that's weird - message with ID {message.MessageId} does not have a lock token!");
}
var lockToken = message.SystemProperties.LockToken;
var messageId = message.MessageId;
CancellationTokenSource cancellationTokenSource = null;
IAsyncTask renewalTask = null;
if (AutomaticallyRenewPeekLock && !_prefetchingEnabled)
{
var now = DateTime.UtcNow;
var leaseDuration = message.SystemProperties.LockedUntilUtc - now;
var lockRenewalInterval = TimeSpan.FromMinutes(0.5 * leaseDuration.TotalMinutes);
cancellationTokenSource = new CancellationTokenSource();
renewalTask = _asyncTaskFactory
.Create(description: $"RenewPeekLock-{messageId}",
action: () => RenewPeekLock(messageReceiver, messageId, lockToken, cancellationTokenSource),
intervalSeconds: (int)lockRenewalInterval.TotalSeconds,
prettyInsignificant: true
);
cancellationTokenSource.Token.Register(renewalTask.Dispose);
renewalTask.Start();
}
context.OnCompleted(async ctx =>
{
try
{
await messageReceiver.CompleteAsync(lockToken).ConfigureAwait(false);
}
catch (Exception exception)
{
throw new RebusApplicationException(exception,
$"Could not complete message with ID {message.MessageId} and lock token {lockToken}");
}
// Dispose the renewal task after the message has been removed.
// Note that we could get a MessageLockLostException and log an error in RenewPeekLock in the off chance that renewal runs between CompleteAsync and Dispose here,
// but that's better than disposing the renewal first and potentially loosing the lock before calling complete.
renewalTask?.Dispose();
});
context.OnAborted(async ctx =>
{
// Dispose the renewal before abandoning the message, otherwise renewal could grab the lock again.
renewalTask?.Dispose();
try
{
await messageReceiver.AbandonAsync(lockToken).ConfigureAwait(false);
}
catch (Exception exception)
{
throw new RebusApplicationException(exception,
$"Could not abandon message with ID {message.MessageId} and lock token {lockToken}");
}
});
context.OnDisposed(ctx =>
{
renewalTask?.Dispose();
cancellationTokenSource?.Dispose();
});
var userProperties = message.UserProperties;
var headers = userProperties.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.ToString());
var body = message.Body;
return(new TransportMessage(headers, body));
}
public async Task<TransportMessage> Receive(ITransactionContext context)
{
if (Address == null)
{
throw new InvalidOperationException("This RabbitMQ transport does not have an input queue, hence it is not possible to reveive anything");
}
var model = GetModel(context);
var result = model.BasicGet(Address, false);
if (result == null) return null;
var deliveryTag = result.DeliveryTag;
context.OnCompleted(async () =>
{
model.BasicAck(deliveryTag, false);
});
context.OnAborted(() =>
{
model.BasicNack(deliveryTag, false, true);
});
var headers = result.BasicProperties.Headers
.ToDictionary(kvp => kvp.Key, kvp =>
{
var headerValue = kvp.Value;
if (headerValue is byte[])
{
var stringHeaderValue = HeaderValueEncoding.GetString((byte[])headerValue);
return stringHeaderValue;
}
return headerValue.ToString();
});
return new TransportMessage(headers, result.Body);
}
public async Task <TransportMessage> Receive(ITransactionContext context)
{
using (await _bottleneck.Enter())
{
var brokeredMessage = await ReceiveBrokeredMessage();
if (brokeredMessage == null)
{
return(null);
}
var headers = brokeredMessage.Properties
.Where(kvp => kvp.Value is string)
.ToDictionary(kvp => kvp.Key, kvp => (string)kvp.Value);
var messageId = headers.GetValueOrNull(Headers.MessageId);
_log.Debug("Received brokered message with ID {0}", messageId);
var leaseDuration = (brokeredMessage.LockedUntilUtc - DateTime.UtcNow);
var lockRenewalInterval = TimeSpan.FromMinutes(0.8 * leaseDuration.TotalMinutes);
var renewalTask = GetRenewalTaskOrFakeDisposable(messageId, brokeredMessage, lockRenewalInterval);
context.OnAborted(() =>
{
renewalTask.Dispose();
_log.Debug("Abandoning message with ID {0}", messageId);
try
{
brokeredMessage.Abandon();
}
catch (Exception exception)
{
// if it fails, it'll be back on the queue anyway....
_log.Warn("Could not abandon message: {0}", exception);
}
});
context.OnCommitted(async() =>
{
renewalTask.Dispose();
});
context.OnCompleted(async() =>
{
_log.Debug("Completing message with ID {0}", messageId);
await GetRetrier().Execute(() => brokeredMessage.CompleteAsync());
});
context.OnDisposed(() =>
{
renewalTask.Dispose();
_log.Debug("Disposing message with ID {0}", messageId);
brokeredMessage.Dispose();
});
return(new TransportMessage(headers, brokeredMessage.GetBody <byte[]>()));
}
}
/// <summary>
/// Receives the next message from the logical input queue by loading the next file from the corresponding directory,
/// deserializing it, deleting it when the transaction is committed.
/// </summary>
public async Task <TransportMessage> Receive(ITransactionContext context)
{
string fullPath = null;
try
{
var fileNames = Directory.GetFiles(GetDirectoryForQueueNamed(_inputQueue), "*.rebusmessage.json")
.OrderBy(f => f)
.ToList();
var index = 0;
while (index < fileNames.Count)
{
fullPath = fileNames[index++];
// attempt to capture a "lock" on the file
if (_messagesBeingHandled.TryAdd(fullPath, new object()))
{
break;
}
fullPath = null;
}
if (fullPath == null)
{
return(null);
}
var jsonText = await ReadAllText(fullPath);
var receivedTransportMessage = Deserialize(jsonText);
string timeToBeReceived;
if (receivedTransportMessage.Headers.TryGetValue(Headers.TimeToBeReceived, out timeToBeReceived))
{
var maxAge = TimeSpan.Parse(timeToBeReceived);
var creationTimeUtc = File.GetCreationTimeUtc(fullPath);
var nowUtc = RebusTime.Now.UtcDateTime;
var messageAge = nowUtc - creationTimeUtc;
if (messageAge > maxAge)
{
try
{
File.Delete(fullPath);
return(null);
}
finally
{
object _;
_messagesBeingHandled.TryRemove(fullPath, out _);
}
}
}
context.OnCompleted(async() => File.Delete(fullPath));
context.OnDisposed(() =>
{
object _;
_messagesBeingHandled.TryRemove(fullPath, out _);
});
return(receivedTransportMessage);
}
catch (IOException)
{
if (fullPath != null)
{
object _;
_messagesBeingHandled.TryRemove(fullPath, out _);
}
return(null);
}
}
/// <inheritdoc />
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
if (Address == null)
{
throw new InvalidOperationException("This RabbitMQ transport does not have an input queue - therefore, it is not possible to receive anything");
}
try
{
if (!_consumers.TryDequeue(out var consumer))
{
consumer = InitializeConsumer();
}
if (consumer == null)
{
// initialization must have failed
return(null);
}
if (!consumer.Model.IsOpen)
{
consumer.Dispose();
return(null);
}
context.OnDisposed(() => _consumers.Enqueue(consumer));
if (!consumer.Queue.Dequeue(TwoSeconds, out var result))
{
return(null);
}
if (result == null)
{
return(null);
}
// ensure we use the consumer's model throughtout the handling of this message
context.Items[CurrentModelItemsKey] = consumer.Model;
var deliveryTag = result.DeliveryTag;
context.OnCompleted(async() =>
{
var model = GetModel(context);
model.BasicAck(deliveryTag, false);
});
context.OnAborted(() =>
{
// we might not be able to do this, but it doesn't matter that much if it succeeds
try
{
var model = GetModel(context);
model.BasicNack(deliveryTag, false, true);
}
catch { }
});
return(CreateTransportMessage(result.BasicProperties, result.Body));
}
catch (EndOfStreamException)
{
return(null);
}
catch (Exception exception)
{
Thread.Sleep(1000);
throw new RebusApplicationException(exception, $"Unexpected exception thrown while trying to dequeue a message from rabbitmq, queue address: {Address}");
}
}
/// <inheritdoc />
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (Address == null)
{
throw new InvalidOperationException("This Amazon SQS transport does not have an input queue, hence it is not possible to reveive anything");
}
if (string.IsNullOrWhiteSpace(_queueUrl))
{
throw new InvalidOperationException("The queue URL is empty - has the transport not been initialized?");
}
var client = GetClientFromTransactionContext(context);
var request = new ReceiveMessageRequest(_queueUrl)
{
MaxNumberOfMessages = 1,
WaitTimeSeconds = _options.ReceiveWaitTimeSeconds,
AttributeNames = new List <string>(new[] { "All" }),
MessageAttributeNames = new List <string>(new[] { "All" })
};
var response = await client.ReceiveMessageAsync(request, cancellationToken);
if (!response.Messages.Any())
{
return(null);
}
var sqsMessage = response.Messages.First();
var renewalTask = CreateRenewalTaskForMessage(sqsMessage, client);
context.OnCompleted(async() =>
{
renewalTask.Dispose();
// if we get this far, we don't want to pass on the cancellation token
// ReSharper disable once MethodSupportsCancellation
await client.DeleteMessageAsync(new DeleteMessageRequest(_queueUrl, sqsMessage.ReceiptHandle));
});
context.OnAborted(() =>
{
renewalTask.Dispose();
Task.Run(() => client.ChangeMessageVisibilityAsync(_queueUrl, sqsMessage.ReceiptHandle, 0, cancellationToken), cancellationToken).Wait(cancellationToken);
});
var transportMessage = ExtractTransportMessageFrom(sqsMessage);
if (MessageIsExpired(transportMessage, sqsMessage))
{
// if the message is expired , we don't want to pass on the cancellation token
// ReSharper disable once MethodSupportsCancellation
await client.DeleteMessageAsync(new DeleteMessageRequest(_queueUrl, sqsMessage.ReceiptHandle));
return(null);
}
renewalTask.Start();
return(transportMessage);
}
/// <summary>
/// Receives the next message from the logical input queue by loading the next file from the corresponding directory,
/// deserializing it, deleting it when the transaction is committed.
/// </summary>
public async Task<TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
string fullPath = null;
try
{
var fileNames = Directory.GetFiles(GetDirectoryForQueueNamed(_inputQueue), "*.rebusmessage.json")
.OrderBy(f => f)
.ToList();
var index = 0;
while (index < fileNames.Count)
{
fullPath = fileNames[index++];
// attempt to capture a "lock" on the file
if (_messagesBeingHandled.TryAdd(fullPath, new object()))
break;
fullPath = null;
}
if (fullPath == null) return null;
var jsonText = await ReadAllText(fullPath);
var receivedTransportMessage = Deserialize(jsonText);
string timeToBeReceived;
if (receivedTransportMessage.Headers.TryGetValue(Headers.TimeToBeReceived, out timeToBeReceived))
{
var maxAge = TimeSpan.Parse(timeToBeReceived);
var creationTimeUtc = File.GetCreationTimeUtc(fullPath);
var nowUtc = RebusTime.Now.UtcDateTime;
var messageAge = nowUtc - creationTimeUtc;
if (messageAge > maxAge)
{
try
{
File.Delete(fullPath);
return null;
}
finally
{
object _;
_messagesBeingHandled.TryRemove(fullPath, out _);
}
}
}
context.OnCompleted(async () => File.Delete(fullPath));
context.OnDisposed(() =>
{
object _;
_messagesBeingHandled.TryRemove(fullPath, out _);
});
return receivedTransportMessage;
}
catch (IOException)
{
if (fullPath != null)
{
object _;
_messagesBeingHandled.TryRemove(fullPath, out _);
}
return null;
}
}
public async Task <TransportMessage> Receive(ITransactionContext context)
{
if (_inputQueueAddress == null)
{
throw new InvalidOperationException("This Azure Service Bus transport does not have an input queue, hence it is not possible to reveive anything");
}
using (await _bottleneck.Enter())
{
var brokeredMessage = await ReceiveBrokeredMessage();
if (brokeredMessage == null)
{
return(null);
}
var headers = brokeredMessage.Properties
.Where(kvp => kvp.Value is string)
.ToDictionary(kvp => kvp.Key, kvp => (string)kvp.Value);
var messageId = headers.GetValueOrNull(Headers.MessageId);
var leaseDuration = (brokeredMessage.LockedUntilUtc - DateTime.UtcNow);
var lockRenewalInterval = TimeSpan.FromMinutes(0.8 * leaseDuration.TotalMinutes);
var renewalTask = GetRenewalTaskOrFakeDisposable(messageId, brokeredMessage, lockRenewalInterval);
context.OnAborted(() =>
{
renewalTask.Dispose();
try
{
brokeredMessage.Abandon();
}
catch (Exception exception)
{
// if it fails, it'll be back on the queue anyway....
_log.Warn("Could not abandon message: {0}", exception);
}
});
context.OnCommitted(async() => renewalTask.Dispose());
context.OnCompleted(async() =>
{
await brokeredMessage.CompleteAsync();
});
context.OnDisposed(() =>
{
renewalTask.Dispose();
brokeredMessage.Dispose();
});
using (var memoryStream = new MemoryStream())
{
await brokeredMessage.GetBody <Stream>().CopyToAsync(memoryStream);
return(new TransportMessage(headers, memoryStream.ToArray()));
}
}
}
/// <inheritdoc />
public async Task <TransportMessage> Receive(ITransactionContext context, string address, CancellationToken cancellationToken)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (address == null)
{
throw new InvalidOperationException("This Amazon SQS transport does not have an input queue, hence it is not possible to receive anything");
}
var queueUrl = m_amazonSqsQueueContext.GetDestinationQueueUrlByName(address, context);
if (string.IsNullOrWhiteSpace(queueUrl))
{
throw new InvalidOperationException("The queue URL is empty - has the transport not been initialized?");
}
var client = m_amazonInternalSettings.CreateSqsClient(context);
var request = new ReceiveMessageRequest(queueUrl)
{
MaxNumberOfMessages = 1,
WaitTimeSeconds = m_amazonInternalSettings.AmazonSnsAndSqsTransportOptions.ReceiveWaitTimeSeconds,
AttributeNames = new List <string>(new[] { "All" }),
MessageAttributeNames = new List <string>(new[] { "All" })
};
var response = await client.ReceiveMessageAsync(request, cancellationToken);
if (response.Messages.Any() == false)
{
return(null);
}
var sqsMessage = response.Messages.First();
var renewalTask = CreateRenewalTaskForMessage(sqsMessage, queueUrl, client);
context.OnCompleted(async() =>
{
renewalTask.Dispose();
// if we get this far, we don't want to pass on the cancellation token
// ReSharper disable once MethodSupportsCancellation
await client.DeleteMessageAsync(new DeleteMessageRequest(queueUrl, sqsMessage.ReceiptHandle));
});
context.OnAborted(() =>
{
renewalTask.Dispose();
Task.Run(() => client.ChangeMessageVisibilityAsync(queueUrl, sqsMessage.ReceiptHandle, 0, cancellationToken), cancellationToken).Wait(cancellationToken);
});
IAmazonMessageProcessor amazonMessageProcessor = _amazonMessageProcessorFactory.Create(sqsMessage);
var transportMessage = amazonMessageProcessor.ProcessMessage();
if (transportMessage.MessageIsExpired(sqsMessage))
{
// if the message is expired , we don't want to pass on the cancellation token
// ReSharper disable once MethodSupportsCancellation
await client.DeleteMessageAsync(new DeleteMessageRequest(queueUrl, sqsMessage.ReceiptHandle));
return(null);
}
renewalTask.Start();
return(transportMessage);
}