public async Task<int> Retry(IncomingMessage message, TimeSpan delay, TransportTransaction transportTransaction)
{
var outgoingMessage = new OutgoingMessage(message.MessageId, new Dictionary<string, string>(message.Headers), message.Body);
var currentDelayedRetriesAttempt = message.GetDelayedDeliveriesPerformed() + 1;
outgoingMessage.SetCurrentDelayedDeliveries(currentDelayedRetriesAttempt);
outgoingMessage.SetDelayedDeliveryTimestamp(DateTime.UtcNow);
UnicastAddressTag messageDestination;
List<DeliveryConstraint> deliveryConstraints = null;
if (timeoutManagerAddress == null)
{
// transport supports native deferred messages, directly send to input queue with delay constraint:
deliveryConstraints = new List<DeliveryConstraint>(1)
{
new DelayDeliveryWith(delay)
};
messageDestination = new UnicastAddressTag(endpointInputQueue);
}
else
{
// transport doesn't support native deferred messages, reroute to timeout manager:
outgoingMessage.Headers[TimeoutManagerHeaders.RouteExpiredTimeoutTo] = endpointInputQueue;
outgoingMessage.Headers[TimeoutManagerHeaders.Expire] = DateTimeExtensions.ToWireFormattedString(DateTime.UtcNow + delay);
messageDestination = new UnicastAddressTag(timeoutManagerAddress);
}
var transportOperations = new TransportOperations(new TransportOperation(outgoingMessage, messageDestination, deliveryConstraints: deliveryConstraints));
await dispatcher.Dispatch(transportOperations, transportTransaction, new ContextBag()).ConfigureAwait(false);
return currentDelayedRetriesAttempt;
}
public override async Task ReceiveMessage()
{
Message message;
if (!TryReceive(MessageQueueTransactionType.None, out message))
{
return;
}
Dictionary<string, string> headers;
if (!TryExtractHeaders(message, out headers))
{
MovePoisonMessageToErrorQueue(message, IsQueuesTransactional ? MessageQueueTransactionType.Single : MessageQueueTransactionType.None);
return;
}
var transportTransaction = new TransportTransaction();
using (var bodyStream = message.BodyStream)
{
try
{
await TryProcessMessage(message.Id, headers, bodyStream, transportTransaction).ConfigureAwait(false);
}
catch (Exception exception)
{
message.BodyStream.Position = 0;
await HandleError(message, headers, exception, transportTransaction, 1).ConfigureAwait(false);
}
}
}
/// <summary>
/// Creates a new transport receive context.
/// </summary>
/// <param name="receivedMessage">The received message.</param>
/// <param name="transportTransaction">The transport transaction.</param>
/// <param name="cancellationTokenSource">
/// Allows the pipeline to flag that it has been aborted and the receive operation should be rolled back.
/// It also allows the transport to communicate to the pipeline to abort if possible.
/// </param>
/// <param name="parentContext">The parent context.</param>
public TransportReceiveContext(IncomingMessage receivedMessage, TransportTransaction transportTransaction, CancellationTokenSource cancellationTokenSource, IBehaviorContext parentContext)
: base(parentContext)
{
this.cancellationTokenSource = cancellationTokenSource;
Message = receivedMessage;
Set(Message);
Set(transportTransaction);
}
public async Task Should_float_transport_transaction_to_dispatcher()
{
var transportTransaction = new TransportTransaction();
var delayedRetryExecutor = CreateExecutor(nativeDeferralsOn: true);
var incomingMessage = CreateMessage();
await delayedRetryExecutor.Retry(incomingMessage, TimeSpan.Zero, transportTransaction);
Assert.AreEqual(dispatcher.Transaction, transportTransaction);
}
public Task<CompletableSynchronizedStorageSession> TryAdapt(TransportTransaction transportTransaction, ContextBag context)
{
Transaction ambientTransaction;
if (transportTransaction.TryGet(out ambientTransaction))
{
var transaction = new InMemoryTransaction();
CompletableSynchronizedStorageSession session = new InMemorySynchronizedStorageSession(transaction);
ambientTransaction.EnlistVolatile(new EnlistmentNotification(transaction), EnlistmentOptions.None);
return Task.FromResult(session);
}
return EmptyTask;
}
public async Task MoveToErrorQueue_should_dispatch_message_to_error_queue()
{
var customErrorQueue = "random_error_queue";
var transportTransaction = new TransportTransaction();
var incomingMessage = new IncomingMessage("messageId", new Dictionary<string, string>(), new byte[0]);
await moveToErrorsExecutor.MoveToErrorQueue(customErrorQueue, incomingMessage, new Exception(), transportTransaction);
Assert.That(dispatcher.TransportOperations.MulticastTransportOperations.Count(), Is.EqualTo(0));
Assert.That(dispatcher.TransportOperations.UnicastTransportOperations.Count(), Is.EqualTo(1));
Assert.That(dispatcher.Transaction, Is.EqualTo(transportTransaction));
var outgoingMessage = dispatcher.TransportOperations.UnicastTransportOperations.Single();
Assert.That(outgoingMessage.Destination, Is.EqualTo(customErrorQueue));
Assert.That(outgoingMessage.Message.MessageId, Is.EqualTo(incomingMessage.MessageId));
}
TransportTransaction PrepareTransportTransaction(SqlConnection connection, SqlTransaction transaction)
{
var transportTransaction = new TransportTransaction();
//these resources are meant to be used by anyone except message dispatcher e.g. persister
transportTransaction.Set(SettingsKeys.TransportTransactionSqlConnectionKey, connection);
transportTransaction.Set(SettingsKeys.TransportTransactionSqlTransactionKey, transaction);
if (transactionForReceiveOnly)
{
//this indicates to MessageDispatcher that it should not reuse connection or transaction for sends
transportTransaction.Set(ReceiveOnlyTransactionMode, true);
}
return(transportTransaction);
}
/// <summary>
/// Enables the use of custom SqlConnection for publish operations.
/// </summary>
/// <param name="options">The <see cref="PublishOptions" /> to extend.</param>
/// <param name="connection">SqlConnection instance that will be used by any operations performed by the transport.</param>
public static void UseCustomSqlConnection(this PublishOptions options, SqlConnection connection)
{
if (connection == null)
{
throw new ArgumentException(nameof(connection));
}
options.RequireImmediateDispatch();
var transportTransaction = new TransportTransaction();
transportTransaction.Set(SettingsKeys.IsUserProvidedTransactionKey, true);
transportTransaction.Set(SettingsKeys.TransportTransactionSqlConnectionKey, connection);
options.GetExtensions().Set(transportTransaction);
}
public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction, ContextBag context)
{
Guard.AgainstNull(nameof(outgoingMessages), outgoingMessages);
if (outgoingMessages.MulticastTransportOperations.Any())
{
throw new Exception("The MSMQ transport only supports unicast transport operations.");
}
foreach (var unicastTransportOperation in outgoingMessages.UnicastTransportOperations)
{
ExecuteTransportOperation(transaction, unicastTransportOperation);
}
return TaskEx.CompletedTask;
}
async Task DispatchDefault(SortingResult sortedOperations, TransportTransaction transportTransaction)
{
if (sortedOperations.DefaultDispatch == null)
{
return;
}
if (InReceiveWithNoTransactionMode(transportTransaction) || InReceiveOnlyTransportTransactionMode(transportTransaction))
{
await DispatchUsingNewConnectionAndTransaction(sortedOperations.DefaultDispatch).ConfigureAwait(false);
return;
}
await DispatchUsingReceiveTransaction(transportTransaction, sortedOperations.DefaultDispatch).ConfigureAwait(false);
}
public override async Task ProcessMessage(CancellationTokenSource stopBatchCancellationTokenSource, CancellationToken cancellationToken = default)
{
using (var connection = await connectionFactory.OpenNewConnection(cancellationToken).ConfigureAwait(false))
{
MessageReadResult receiveResult;
using (var transaction = connection.BeginTransaction(IsolationLevel.ReadCommitted))
{
receiveResult = await InputQueue.TryReceive(connection, transaction, cancellationToken).ConfigureAwait(false);
if (receiveResult == MessageReadResult.NoMessage)
{
stopBatchCancellationTokenSource.Cancel();
return;
}
if (receiveResult.IsPoison)
{
await ErrorQueue.DeadLetter(receiveResult.PoisonMessage, connection, transaction, cancellationToken).ConfigureAwait(false);
transaction.Commit();
return;
}
if (await TryHandleDelayedMessage(receiveResult.Message, connection, transaction, cancellationToken).ConfigureAwait(false))
{
transaction.Commit();
return;
}
transaction.Commit();
}
var context = new ContextBag();
var transportTransaction = new TransportTransaction();
transportTransaction.Set(SettingsKeys.TransportTransactionSqlConnectionKey, connection);
try
{
await TryHandleMessage(receiveResult.Message, transportTransaction, context, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when(!ex.IsCausedBy(cancellationToken))
{
// Since this is TransactionMode.None, we don't care whether error handling says handled or retry. Message is gone either way.
_ = await HandleError(ex, receiveResult.Message, transportTransaction, 1, context, cancellationToken).ConfigureAwait(false);
}
}
}
static ErrorContext ToErrorContext(ResolvedEvent evnt, Exception ex, TransportTransaction transportTransaction, int failures)
{
var metadata = evnt.Event.Metadata.ParseJson <MessageMetadata>();
var headers = new Dictionary <string, string>(metadata.Headers);
var data = metadata.Empty //because EventStore inserts {}
? new byte[0]
: evnt.Event.Data;
string contentType;
if (headers.TryGetValue(Headers.ContentType, out contentType))
{
data = contentType != ContentTypes.Json
? Convert.FromBase64String(Encoding.UTF8.GetString(data))
: data;
}
return(new ErrorContext(ex, headers, metadata.MessageId, data, transportTransaction, failures));
}
public Task Dispatch(TransportOperations operations, TransportTransaction transportTransaction, ContextBag context)
{
var outgoingBatches = batcher.ToBatches(operations);
if (!TryGetReceiveContext(transportTransaction, out var receiveContext)) // not in a receive context, so send out immediately
{
return(routeOutgoingBatches.RouteBatches(outgoingBatches, null, DispatchConsistency.Default));
}
if (receiveContext is BrokeredMessageReceiveContextInternal brokeredMessageReceiveContext) // apply brokered message specific dispatching rules
{
return(DispatchBatches(outgoingBatches, brokeredMessageReceiveContext));
}
// case when the receive context is different from brokered messaging (like eventhub)
return(routeOutgoingBatches.RouteBatches(outgoingBatches, receiveContext, DispatchConsistency.Default)); // otherwise send out immediately
}
public async Task Should_upload_large_isolated_operations_request_to_s3()
{
var settings = new SettingsHolder();
var transportExtensions = new TransportExtensions <SqsTransport>(settings);
transportExtensions.S3("someBucket", "somePrefix");
var mockS3Client = new MockS3Client();
var mockSqsClient = new MockSqsClient();
var transportConfiguration = new TransportConfiguration(settings);
var dispatcher = new MessageDispatcher(transportConfiguration, mockS3Client, mockSqsClient, null, new QueueCache(mockSqsClient, transportConfiguration), null);
var transportOperations = new TransportOperations(
new TransportOperation(
new OutgoingMessage(Guid.NewGuid().ToString(), new Dictionary <string, string>(), Encoding.Default.GetBytes(new string('x', 256 * 1024))),
new UnicastAddressTag("address1"),
DispatchConsistency.Isolated),
new TransportOperation(
new OutgoingMessage(Guid.NewGuid().ToString(), new Dictionary <string, string>(), Encoding.Default.GetBytes(new string('x', 256 * 1024))),
new UnicastAddressTag("address2"),
DispatchConsistency.Isolated),
new TransportOperation( /* Crazy long message id will cause the message to go over limits because attributes count as well */
new OutgoingMessage(new string('x', 256 * 1024), new Dictionary <string, string>(), Encoding.Default.GetBytes("{}")),
new UnicastAddressTag("address2"),
DispatchConsistency.Isolated));
var transportTransaction = new TransportTransaction();
var context = new ContextBag();
await dispatcher.Dispatch(transportOperations, transportTransaction, context);
Assert.AreEqual(3, mockSqsClient.RequestsSent.Count);
Assert.AreEqual(3, mockS3Client.PutObjectRequestsSent.Count);
var firstUpload = mockS3Client.PutObjectRequestsSent.ElementAt(0);
var secondUpload = mockS3Client.PutObjectRequestsSent.ElementAt(1);
var thirdUpload = mockS3Client.PutObjectRequestsSent.ElementAt(2);
Assert.AreEqual("someBucket", firstUpload.BucketName);
Assert.AreEqual("someBucket", secondUpload.BucketName);
Assert.AreEqual("someBucket", thirdUpload.BucketName);
StringAssert.Contains($@"""Body"":"""",""S3BodyKey"":""{firstUpload.Key}", mockSqsClient.RequestsSent.ElementAt(0).MessageBody);
StringAssert.Contains($@"""Body"":"""",""S3BodyKey"":""{secondUpload.Key}", mockSqsClient.RequestsSent.ElementAt(1).MessageBody);
StringAssert.Contains($@"""Body"":"""",""S3BodyKey"":""{thirdUpload.Key}", mockSqsClient.RequestsSent.ElementAt(2).MessageBody);
}
public async Task Adapts_transport_connection()
{
var transportTransaction = new TransportTransaction();
var transportConnection = connectionManager.BuildNonContextual();
var transaction = transportConnection.BeginTransaction();
transportTransaction.Set("System.Data.SqlClient.SqlConnection", transportConnection);
transportTransaction.Set("System.Data.SqlClient.SqlTransaction", transaction);
var altConnectionManager = new ConnectionManager(() => throw new Exception("Should not be called"));
var result = await sqlDialect.Convert().TryAdaptTransportConnection(transportTransaction, new ContextBag(), altConnectionManager,
(conn, tx, arg3) => new StorageSession(conn, tx, false, null));
Assert.IsNotNull(result);
}
internal override async Task <StorageSession> TryAdaptTransportConnection(
TransportTransaction transportTransaction,
ContextBag context,
IConnectionManager connectionManager,
Func <DbConnection, DbTransaction, bool, StorageSession> storageSessionFactory,
CancellationToken cancellationToken = default)
{
if (DoNotUseTransportConnection)
{
return(null);
}
// SQL server transport in native TX mode
if (transportTransaction.TryGet("System.Data.SqlClient.SqlConnection", out DbConnection existingSqlConnection) &&
transportTransaction.TryGet("System.Data.SqlClient.SqlTransaction", out DbTransaction existingSqlTransaction))
{
return(storageSessionFactory(existingSqlConnection, existingSqlTransaction, false));
}
// Transport supports DTC and uses TxScope owned by the transport
var scopeTx = Transaction.Current;
if (transportTransaction.TryGet(out Transaction transportTx) &&
scopeTx != null &&
transportTx != scopeTx)
{
throw new Exception("A TransactionScope has been opened in the current context overriding the one created by the transport. "
+ "This setup can result in inconsistent data because operations done via connections enlisted in the context scope won't be committed "
+ "atomically with the receive transaction. To manually control the TransactionScope in the pipeline switch the transport transaction mode "
+ $"to values lower than '{nameof(TransportTransactionMode.TransactionScope)}'.");
}
var ambientTransaction = transportTx ?? scopeTx;
if (ambientTransaction == null)
{
// Other modes handled by creating a new session.
return(null);
}
var connection = await connectionManager.OpenConnection(context.GetIncomingMessage(), cancellationToken).ConfigureAwait(false);
connection.EnlistTransaction(ambientTransaction);
return(storageSessionFactory(connection, null, true));
}
public async Task MoveToErrorQueue_should_dispatch_message_to_error_queue()
{
var customErrorQueue = "random_error_queue";
var transportTransaction = new TransportTransaction();
var incomingMessage = new IncomingMessage("messageId", new Dictionary <string, string>(), new byte[0]);
await moveToErrorsExecutor.MoveToErrorQueue(customErrorQueue, incomingMessage, new Exception(), transportTransaction);
Assert.That(dispatcher.TransportOperations.MulticastTransportOperations.Count(), Is.EqualTo(0));
Assert.That(dispatcher.TransportOperations.UnicastTransportOperations.Count(), Is.EqualTo(1));
Assert.That(dispatcher.Transaction, Is.EqualTo(transportTransaction));
var outgoingMessage = dispatcher.TransportOperations.UnicastTransportOperations.Single();
Assert.That(outgoingMessage.Destination, Is.EqualTo(customErrorQueue));
Assert.That(outgoingMessage.Message.MessageId, Is.EqualTo(incomingMessage.MessageId));
}
Task SendReadyMessage(int capacity, bool isStarting, TransportTransaction transaction)
{
//we use the actual address to make sure that the worker inside the master node will check in correctly
var readyMessage = ControlMessageFactory.Create(MessageIntentEnum.Send);
readyMessage.Headers.Add(LegacyDistributorHeaders.WorkerCapacityAvailable, capacity.ToString());
readyMessage.Headers.Add(LegacyDistributorHeaders.WorkerSessionId, workerSessionId);
readyMessage.Headers.Add(Headers.ReplyToAddress, receiveAddress);
if (isStarting)
{
readyMessage.Headers.Add(LegacyDistributorHeaders.WorkerStarting, bool.TrueString);
}
var transportOperation = new TransportOperation(readyMessage, new UnicastAddressTag(distributorControlAddress));
return dispatcher.Dispatch(new TransportOperations(transportOperation), transaction, new ContextBag());
}
public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction,
CancellationToken cancellationToken = default)
{
Guard.AgainstNull(nameof(outgoingMessages), outgoingMessages);
if (outgoingMessages.MulticastTransportOperations.Any())
{
throw new Exception("The MSMQ transport only supports unicast transport operations.");
}
foreach (var unicastTransportOperation in outgoingMessages.UnicastTransportOperations)
{
ExecuteTransportOperation(transaction, unicastTransportOperation);
}
return(Task.CompletedTask);
}
public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction, ContextBag context)
{
var allIds = outgoingMessages.MulticastTransportOperations.Select(mto => mto.Message.MessageId).ToList();
allIds.AddRange(outgoingMessages.UnicastTransportOperations.Select(mto => mto.Message.MessageId));
var alreadyFailedOnce = allIds.All(id => _idsForWhichDispatchAlreadyFailedOnce.Contains(id));
if (_failDispatchOnce && !alreadyFailedOnce)
{
_idsForWhichDispatchAlreadyFailedOnce.AddRange(allIds);
throw new Exception("Bad!");
}
return(Task.WhenAll(
DispatchUnicast(outgoingMessages.UnicastTransportOperations, transaction),
DispatchMulticast(outgoingMessages.MulticastTransportOperations, transaction)));
}
async Task TrySendDelayedMessageToErrorQueue(DelayedMessage timeout, Exception exception, CancellationToken cancellationToken)
{
try
{
bool success = await delayedMessageStore.Remove(timeout, cancellationToken).ConfigureAwait(false);
if (!success)
{
// Already dispatched
return;
}
Dictionary <string, string> headersAndProperties = MsmqUtilities.DeserializeMessageHeaders(timeout.Headers);
ExceptionHeaderHelper.SetExceptionHeaders(headersAndProperties, exception);
headersAndProperties[FaultsHeaderKeys.FailedQ] = timeoutsQueueTransportAddress;
foreach (KeyValuePair <string, string> pair in faultMetadata)
{
headersAndProperties[pair.Key] = pair.Value;
}
Log.InfoFormat("Move {0} to error queue", timeout.MessageId);
using (var transportTx = new TransactionScope(txOption, transactionOptions, TransactionScopeAsyncFlowOption.Enabled))
{
var transportTransaction = new TransportTransaction();
transportTransaction.Set(Transaction.Current);
var outgoingMessage = new OutgoingMessage(timeout.MessageId, headersAndProperties, timeout.Body);
var transportOperation = new TransportOperation(outgoingMessage, new UnicastAddressTag(errorQueue));
await dispatcher.Dispatch(new TransportOperations(transportOperation), transportTransaction, CancellationToken.None)
.ConfigureAwait(false);
transportTx.Complete();
}
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
//Shutting down
Log.Debug("Aborted sending delayed message to error queue due to shutdown.");
}
catch (Exception ex)
{
Log.Error($"Failed to move delayed message {timeout.MessageId} to the error queue {errorQueue} after {timeout.NumberOfRetries} failed attempts at dispatching it to the destination", ex);
}
}
Task SendReadyMessage(int capacity, bool isStarting, TransportTransaction transaction)
{
//we use the actual address to make sure that the worker inside the master node will check in correctly
var readyMessage = ControlMessageFactory.Create(MessageIntentEnum.Send);
readyMessage.Headers.Add(LegacyDistributorHeaders.WorkerCapacityAvailable, capacity.ToString());
readyMessage.Headers.Add(LegacyDistributorHeaders.WorkerSessionId, workerSessionId);
readyMessage.Headers.Add(Headers.ReplyToAddress, receiveAddress);
if (isStarting)
{
readyMessage.Headers.Add(LegacyDistributorHeaders.WorkerStarting, bool.TrueString);
}
var transportOperation = new TransportOperation(readyMessage, new UnicastAddressTag(distributorControlAddress));
return(dispatcher.Dispatch(new TransportOperations(transportOperation), transaction, new ContextBag()));
}
public Task <CompletableSynchronizedStorageSession> TryAdapt(TransportTransaction transportTransaction, ContextBag context)
{
Transaction ambientTransaction;
if (!transportTransaction.TryGet(out ambientTransaction))
{
return(EmptyResult);
}
var sessionFactoryImpl = sessionFactory as SessionFactoryImpl;
if (sessionFactoryImpl == null)
{
throw new NotSupportedException("Overriding default implementation of ISessionFactory is not supported.");
}
CompletableSynchronizedStorageSession session = new NHibernateLazyAmbientTransactionSynchronizedStorageSession(() => OpenConnection(sessionFactoryImpl, ambientTransaction), conn => sessionFactory.OpenSession(conn));
return(Task.FromResult(session));
}
async Task DispatchUsingReceiveTransaction(TransportTransaction transportTransaction, List <UnicastTransportOperation> operations)
{
transportTransaction.TryGet(out SqlConnection sqlTransportConnection);
transportTransaction.TryGet(out SqlTransaction sqlTransportTransaction);
transportTransaction.TryGet(out Transaction ambientTransaction);
if (ambientTransaction != null)
{
using (var connection = await connectionFactory.OpenNewConnection().ConfigureAwait(false))
{
await Send(operations, connection, null).ConfigureAwait(false);
}
}
else
{
await Send(operations, sqlTransportConnection, sqlTransportTransaction).ConfigureAwait(false);
}
}
public async Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction, ContextBag context)
{
try
{
var operations = outgoingMessages.UnicastTransportOperations;
var tasks = new Task[operations.Count];
for (var i = 0; i < operations.Count; i++)
{
tasks[i] = Dispatch(operations[i]);
}
await Task.WhenAll(tasks).ConfigureAwait(false);
}
catch (Exception e)
{
Logger.Error("Exception from Send.", e);
throw;
}
}
public async Task SendToError(Exception ex, IFullMessage message)
{
var transportTransaction = new TransportTransaction();
var messageBytes = _serializer.Serialize(message.Message);
var headers = new Dictionary <string, string>(message.Headers);
var errorContext = new ErrorContext(ex, headers,
Guid.NewGuid().ToString(),
messageBytes, transportTransaction,
int.MaxValue);
if ((await Bus.OnError(errorContext).ConfigureAwait(false)) != ErrorHandleResult.Handled)
{
throw new InvalidOperationException("Failed to send message error queue");
}
}
protected async Task <bool> TryProcessingMessage(Message message, TransportTransaction transportTransaction)
{
if (message.Expired) //Do not process expired messages
{
return(true);
}
using (var pushCancellationTokenSource = new CancellationTokenSource())
{
var messageContext = new MessageContext(message.TransportId, message.Headers, message.Body, transportTransaction, pushCancellationTokenSource, new ContextBag());
await onMessage(messageContext).ConfigureAwait(false);
// Cancellation is requested when message processing is aborted.
// We return the opposite value:
// - true when message processing completed successfully,
// - false when message processing was aborted.
return(!pushCancellationTokenSource.Token.IsCancellationRequested);
}
}
public Task Init(Func <MessageContext, Task> pump, Func <ErrorContext, Task <ErrorHandleResult> > onError, CriticalError criticalError, PushSettings pushSettings)
{
topologyOperator = DetermineTopologyOperator(pushSettings.InputQueue);
messagePump = pump;
var name = $"MessagePump on the queue `{pushSettings.InputQueue}`";
circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker(name, timeToWaitBeforeTriggering, ex => criticalError.Raise("Failed to receive message from Azure Service Bus.", ex));
if (pushSettings.PurgeOnStartup)
{
throw new InvalidOperationException("Azure Service Bus transport doesn't support PurgeOnStartup behavior");
}
inputQueue = pushSettings.InputQueue;
topologyOperator.OnIncomingMessage(async(incoming, receiveContext) =>
{
var tokenSource = new CancellationTokenSource();
receiveContext.CancellationToken = tokenSource.Token;
circuitBreaker.Success();
var transportTransaction = new TransportTransaction();
transportTransaction.Set(receiveContext);
await throttler.WaitAsync(receiveContext.CancellationToken).ConfigureAwait(false);
try
{
await messagePump(new MessageContext(incoming.MessageId, incoming.Headers, incoming.Body, transportTransaction, tokenSource, new ContextBag())).ConfigureAwait(false);
}
finally
{
throttler.Release();
}
});
topologyOperator.OnError(exception => circuitBreaker.Failure(exception));
topologyOperator.OnProcessingFailure(onError);
return(TaskEx.Completed);
}
public async Task Should_deduplicate_if_compatibility_mode_is_enabled_and_subscription_found()
{
var settings = new SettingsHolder();
var mockSnsClient = new MockSnsClient();
var mockSqsClient = new MockSqsClient();
var transportConfiguration = new TransportConfiguration(settings);
var dispatcher = new MessageDispatcher(transportConfiguration, null, mockSqsClient, mockSnsClient, new QueueCache(mockSqsClient, transportConfiguration), new TopicCache(mockSnsClient, settings.SetupMessageMetadataRegistry(), transportConfiguration));
mockSnsClient.ListSubscriptionsByTopicResponse = topic => new ListSubscriptionsByTopicResponse
{
Subscriptions = new List <Subscription>
{
new Subscription {
Endpoint = "arn:abc", SubscriptionArn = "arn:subscription"
}
}
};
var messageId = Guid.NewGuid().ToString();
var headers = new Dictionary <string, string>()
{
{ Headers.EnclosedMessageTypes, typeof(Event).AssemblyQualifiedName },
{ Headers.MessageIntent, MessageIntentEnum.Publish.ToString() }
};
var transportOperations = new TransportOperations(
new TransportOperation(
new OutgoingMessage(messageId, headers, Encoding.Default.GetBytes("{}")),
new MulticastAddressTag(typeof(Event))),
new TransportOperation(
new OutgoingMessage(messageId, headers, Encoding.Default.GetBytes("{}")),
new UnicastAddressTag("abc"))
);
var transportTransaction = new TransportTransaction();
var context = new ContextBag();
await dispatcher.Dispatch(transportOperations, transportTransaction, context);
Assert.AreEqual(1, mockSnsClient.PublishedEvents.Count);
Assert.IsEmpty(mockSqsClient.RequestsSent);
Assert.IsEmpty(mockSqsClient.BatchRequestsSent);
}
public async Task <CompletableSynchronizedStorageSession> TryAdapt(TransportTransaction transportTransaction, ContextBag context)
{
Transaction ambientTransaction;
if (transportTransaction.TryGet(out ambientTransaction))
{
NpgsqlConnection existingSqlConnection;
//SQL server transport in ambient TX mode
if (transportTransaction.TryGet(out existingSqlConnection))
{
return(new StorageSession(existingSqlConnection, null));
}
//Other transport in ambient TX mode
var connection = await NpgSqlHelpers.NewConnection(_connectionString);
return(new StorageSession(connection, connection.BeginTransaction()));
}
return(null);
}
public async Task Save_should_fail_when_data_changes_between_concurrent_instances()
{
configuration.RequiresDtcSupport();
var persister = configuration.SagaStorage;
var sagaData = new TestSagaData {
SomeId = Guid.NewGuid().ToString()
};
await SaveSaga(sagaData);
var generatedSagaId = sagaData.Id;
Assert.That(async() =>
{
var storageAdapter = configuration.SynchronizedStorageAdapter;
using (var tx = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
Transaction.Current.EnlistDurable(EnlistmentWhichEnforcesDtcEscalation.Id, new EnlistmentWhichEnforcesDtcEscalation(), EnlistmentOptions.None);
var transportTransaction = new TransportTransaction();
transportTransaction.Set(Transaction.Current);
var unenlistedContextBag = configuration.GetContextBagForSagaStorage();
using (var unenlistedSession = await configuration.SynchronizedStorage.OpenSession(unenlistedContextBag))
{
var enlistedContextBag = configuration.GetContextBagForSagaStorage();
var enlistedSession = await storageAdapter.TryAdapt(transportTransaction, enlistedContextBag);
var unenlistedRecord = await persister.Get <TestSagaData>(generatedSagaId, unenlistedSession, unenlistedContextBag);
var enlistedRecord = await persister.Get <TestSagaData>(generatedSagaId, enlistedSession, enlistedContextBag);
await persister.Update(unenlistedRecord, unenlistedSession, unenlistedContextBag);
await persister.Update(enlistedRecord, enlistedSession, enlistedContextBag);
await unenlistedSession.CompleteAsync();
}
tx.Complete();
}
}, Throws.Exception.TypeOf <TransactionAbortedException>());
}
async Task <bool> ProcessMessage(MessageQueueTransaction msmqTransaction, Message message, Dictionary <string, string> headers)
{
var transportTransaction = new TransportTransaction();
transportTransaction.Set(msmqTransaction);
MsmqFailureInfoStorage.ProcessingFailureInfo failureInfo;
var shouldTryProcessMessage = true;
if (failureInfoStorage.TryGetFailureInfoForMessage(message.Id, out failureInfo))
{
var errorHandleResult = await HandleError(message, headers, failureInfo.Exception, transportTransaction, failureInfo.NumberOfProcessingAttempts).ConfigureAwait(false);
shouldTryProcessMessage = errorHandleResult != ErrorHandleResult.Handled;
}
if (shouldTryProcessMessage)
{
try
{
using (var bodyStream = message.BodyStream)
{
var shouldAbortMessageProcessing = await TryProcessMessage(message, headers, bodyStream, transportTransaction).ConfigureAwait(false);
if (shouldAbortMessageProcessing)
{
return(false);
}
}
}
catch (Exception exception)
{
failureInfoStorage.RecordFailureInfoForMessage(message.Id, exception);
return(false);
}
}
failureInfoStorage.ClearFailureInfoForMessage(message.Id);
return(true);
}
public Task MessageProcessed(Dictionary<string, string> headers, TransportTransaction transaction)
{
//if there was a failure this "send" will be rolled back
string messageSessionId;
headers.TryGetValue(LegacyDistributorHeaders.WorkerSessionId, out messageSessionId);
if (messageSessionId == null)
{
return TaskEx.CompletedTask;
}
var messageId = headers[Headers.MessageId];
Logger.DebugFormat("Got message with id {0} and messageSessionId {1}. WorkerSessionId is {2}", messageId, messageSessionId, workerSessionId);
if (messageSessionId != workerSessionId)
{
Logger.InfoFormat("SKIPPED Ready message for message with id {0} because of sessionId mismatch. MessageSessionId {1}, WorkerSessionId {2}", messageId, messageSessionId, workerSessionId);
return TaskEx.CompletedTask;
}
return SendReadyMessage(1, false, transaction);
}
async Task<bool> ProcessMessage(MessageQueueTransaction msmqTransaction, Message message, Dictionary<string, string> headers)
{
var transportTransaction = new TransportTransaction();
transportTransaction.Set(msmqTransaction);
MsmqFailureInfoStorage.ProcessingFailureInfo failureInfo;
var shouldTryProcessMessage = true;
if (failureInfoStorage.TryGetFailureInfoForMessage(message.Id, out failureInfo))
{
var errorHandleResult = await HandleError(message, headers, failureInfo.Exception, transportTransaction, failureInfo.NumberOfProcessingAttempts).ConfigureAwait(false);
shouldTryProcessMessage = errorHandleResult != ErrorHandleResult.Handled;
}
if (shouldTryProcessMessage)
{
try
{
using (var bodyStream = message.BodyStream)
{
var shouldAbortMessageProcessing = await TryProcessMessage(message.Id, headers, bodyStream, transportTransaction).ConfigureAwait(false);
if (shouldAbortMessageProcessing)
{
return false;
}
}
}
catch (Exception exception)
{
failureInfoStorage.RecordFailureInfoForMessage(message.Id, exception);
return false;
}
}
failureInfoStorage.ClearFailureInfoForMessage(message.Id);
return true;
}
public void It_throws_if_non_transport_transaction_scope_exists()
{
if (!SupportsDistributedTransactions)
{
Assert.Ignore();
}
using (new TransactionScope())
{
var transportTransaction = new TransportTransaction();
transportTransaction.Set(Transaction.Current);
using (new TransactionScope(TransactionScopeOption.RequiresNew, TransactionScopeAsyncFlowOption.Enabled))
{
var ex = Assert.ThrowsAsync <Exception>(() => sqlDialect.Convert().TryAdaptTransportConnection(transportTransaction, new ContextBag(), dbConnection,
(conn, tx, arg3) => new StorageSession(conn, tx, false, null)));
StringAssert.StartsWith("A TransactionScope has been opened in the current context overriding the one created by the transport.", ex.Message);
}
}
}
async Task <bool> ProcessMessage(MessageQueueTransaction msmqTransaction, Message message, Dictionary <string, string> headers, ContextBag context, CancellationToken cancellationToken)
{
var transportTransaction = new TransportTransaction();
transportTransaction.Set(msmqTransaction);
var length = (int)message.BodyStream.Length;
var buffer = ArrayPool <byte> .Shared.Rent(length);
try
{
_ = await message.BodyStream.ReadAsync(buffer, 0, length, cancellationToken).ConfigureAwait(false);
var body = buffer.AsMemory(0, length);
if (failureInfoStorage.TryGetFailureInfoForMessage(message.Id, out var failureInfo))
{
var errorHandleResult = await HandleError(message, body, failureInfo.Exception, transportTransaction, failureInfo.NumberOfProcessingAttempts, failureInfo.Context, cancellationToken).ConfigureAwait(false);
if (errorHandleResult == ErrorHandleResult.Handled)
{
return(true);
}
}
try
{
await TryProcessMessage(message.Id, headers, body, transportTransaction, context, cancellationToken).ConfigureAwait(false);
return(true);
}
catch (Exception ex) when(!ex.IsCausedBy(cancellationToken))
{
failureInfoStorage.RecordFailureInfoForMessage(message.Id, ex, context);
return(false);
}
}
finally
{
ArrayPool <byte> .Shared.Return(buffer);
}
}
public async Task Save_fails_when_data_changes_between_concurrent_instances()
{
var saga = new TestSagaData
{
Id = Guid.NewGuid()
};
var persister = new InMemorySagaPersister();
var storageAdapter = new InMemoryTransactionalSynchronizedStorageAdapter();
var insertSession = new InMemorySynchronizedStorageSession();
await persister.Save(saga, SagaMetadataHelper.GetMetadata <TestSaga>(saga), insertSession, new ContextBag());
await insertSession.CompleteAsync();
Assert.That(async() =>
{
using (var tx = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
Transaction.Current.EnlistDurable(EnlistmentWhichEnforcesDtcEscalation.Id, new EnlistmentWhichEnforcesDtcEscalation(), EnlistmentOptions.None);
var transportTransaction = new TransportTransaction();
transportTransaction.Set(Transaction.Current);
var unenlistedSession = new InMemorySynchronizedStorageSession();
var enlistedSession = await storageAdapter.TryAdapt(transportTransaction, new ContextBag());
var unenlistedSessionContext = new ContextBag();
var unenlistedRecord = await persister.Get <TestSagaData>(saga.Id, unenlistedSession, unenlistedSessionContext);
var enlistedSessionContext = new ContextBag();
var enlistedRecord = await persister.Get <TestSagaData>("Id", saga.Id, enlistedSession, enlistedSessionContext);
await persister.Update(unenlistedRecord, unenlistedSession, unenlistedSessionContext);
await persister.Update(enlistedRecord, enlistedSession, enlistedSessionContext);
await unenlistedSession.CompleteAsync();
tx.Complete();
}
}, Throws.Exception.TypeOf <TransactionAbortedException>());
}
Task DispatchUnicast(IEnumerable <UnicastTransportOperation> operations, TransportTransaction transaction)
{
return(Task.WhenAll(operations.Select(operation =>
{
PathChecker.ThrowForBadPath(operation.Destination, "message destination");
if (!operation.Message.Headers.TryGetValue(Headers.MessageId, out var messageId))
{
messageId = operation.Message.MessageId;
}
if (!operation.Message.Headers.TryGetValue(Headers.MessageIntent, out var intent))
{
intent = "<Unknown>";
}
Console.WriteLine($"Sending message {messageId} with intent {intent} sent to {operation.Destination}");
return WriteMessage(operation.Destination, operation, transaction);
})));
}
public async Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction, ContextBag context)
{
var unicastTransportOperations = outgoingMessages.UnicastTransportOperations;
var multicastTransportOperations = outgoingMessages.MulticastTransportOperations;
var tasks = new List <Task>(unicastTransportOperations.Count + multicastTransportOperations.Count);
foreach (var operation in unicastTransportOperations)
{
tasks.Add(SendMessage(operation));
}
foreach (var operation in multicastTransportOperations)
{
tasks.Add(SendMessage(operation));
}
await(tasks.Count == 1 ? tasks[0] : Task.WhenAll(tasks))
.ConfigureAwait(false);
}
public async Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction, CancellationToken cancellationToken = default)
{
int totalNumberOfOperations = outgoingMessages.UnicastTransportOperations.Count + outgoingMessages.MulticastTransportOperations.Count;
var unicastOperations = new HashSet<UnicastTransportOperation>(totalNumberOfOperations, MessageIdAndDestinationEqualityComparer.Instance);
unicastOperations.AddRange(outgoingMessages.UnicastTransportOperations);
foreach (var multicastTransportOperation in outgoingMessages.MulticastTransportOperations)
{
unicastOperations.AddRange(await ConvertTo(multicastTransportOperation, cancellationToken).ConfigureAwait(false));
}
var sends = new List<Task>(totalNumberOfOperations);
foreach (var unicastTransportOperation in unicastOperations)
{
sends.Add(Send(unicastTransportOperation, cancellationToken));
}
await Task.WhenAll(sends).ConfigureAwait(false);
}
public async Task Save_fails_when_data_changes_between_concurrent_instances()
{
var saga = new TestSagaData
{
Id = Guid.NewGuid()
};
var persister = new InMemorySagaPersister();
var storageAdapter = new InMemoryTransactionalSynchronizedStorageAdapter();
var insertSession = new InMemorySynchronizedStorageSession();
await persister.Save(saga, SagaMetadataHelper.GetMetadata<TestSaga>(saga), insertSession, new ContextBag());
await insertSession.CompleteAsync();
Assert.That(async () =>
{
using (var tx = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
Transaction.Current.EnlistDurable(EnlistmentWhichEnforcesDtcEscalation.Id, new EnlistmentWhichEnforcesDtcEscalation(), EnlistmentOptions.None);
var transportTransaction = new TransportTransaction();
transportTransaction.Set(Transaction.Current);
var unenlistedSession = new InMemorySynchronizedStorageSession();
var enlistedSession = await storageAdapter.TryAdapt(transportTransaction, new ContextBag());
var unenlistedSessionContext = new ContextBag();
var unenlistedRecord = await persister.Get<TestSagaData>(saga.Id, unenlistedSession, unenlistedSessionContext);
var enlistedSessionContext = new ContextBag();
var enlistedRecord = await persister.Get<TestSagaData>("Id", saga.Id, enlistedSession, enlistedSessionContext);
await persister.Update(unenlistedRecord, unenlistedSession, unenlistedSessionContext);
await persister.Update(enlistedRecord, enlistedSession, enlistedSessionContext);
await unenlistedSession.CompleteAsync();
tx.Complete();
}
}, Throws.Exception.TypeOf<TransactionAbortedException>());
}
public Task MoveToErrorQueue(string errorQueueAddress, IncomingMessage message, Exception exception, TransportTransaction transportTransaction)
{
message.RevertToOriginalBodyIfNeeded();
var outgoingMessage = new OutgoingMessage(message.MessageId, new Dictionary<string, string>(message.Headers), message.Body);
var headers = outgoingMessage.Headers;
headers.Remove(Headers.DelayedRetries);
headers.Remove(Headers.ImmediateRetries);
ExceptionHeaderHelper.SetExceptionHeaders(headers, exception);
foreach (var faultMetadata in staticFaultMetadata)
{
headers[faultMetadata.Key] = faultMetadata.Value;
}
headerCustomizations(headers);
var transportOperations = new TransportOperations(new TransportOperation(outgoingMessage, new UnicastAddressTag(errorQueueAddress)));
return dispatcher.Dispatch(transportOperations, transportTransaction, new ContextBag());
}
public DispatchedMessage(TransportOperations operations, TransportTransaction transaction, ContextBag context)
{
Operations = operations;
Context = context;
Transaction = transaction;
}
public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transportTransaction, ContextBag context)
{
MessagesSent += outgoingMessages.MulticastTransportOperations.Count + outgoingMessages.UnicastTransportOperations.Count;
return TaskEx.CompletedTask;
}
public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction, ContextBag context)
{
DispatchedMessages.Add(new DispatchedMessage(outgoingMessages, transaction, context));
return TaskEx.CompletedTask;
}
protected async Task<bool> TryProcessMessage(string messageId, Dictionary<string, string> headers, Stream bodyStream, TransportTransaction transaction)
{
using (var tokenSource = new CancellationTokenSource())
{
var body = await ReadStream(bodyStream).ConfigureAwait(false);
var messageContext = new MessageContext(messageId, headers, body, transaction, tokenSource, new ContextBag());
await onMessage(messageContext).ConfigureAwait(false);
return tokenSource.Token.IsCancellationRequested;
}
}
public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction, ContextBag context)
{
TransportOperations = outgoingMessages;
ContextBag = context;
Transaction = transaction;
return Task.FromResult(0);
}
protected async Task<ErrorHandleResult> HandleError(Message message, Dictionary<string, string> headers, Exception exception, TransportTransaction transportTransaction, int processingAttempts)
{
try
{
var body = await ReadStream(message.BodyStream).ConfigureAwait(false);
var errorContext = new ErrorContext(exception, headers, message.Id, body, transportTransaction, processingAttempts);
return await onError(errorContext).ConfigureAwait(false);
}
catch (Exception ex)
{
criticalError.Raise($"Failed to execute recoverability actions for message `{message.Id}`", ex);
//best thing we can do is roll the message back if possible
return ErrorHandleResult.RetryRequired;
}
}
public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction, ContextBag context)
{
if (numberOfTimes.HasValue && FailedNumberOfTimes < numberOfTimes.Value)
{
FailedNumberOfTimes++;
throw new QueueNotFoundException();
}
DispatchedTransportOperations.Add(outgoingMessages);
return TaskEx.CompletedTask;
}
public Task<CompletableSynchronizedStorageSession> TryAdapt(TransportTransaction transportTransaction, ContextBag context)
{
return EmptyResult;
}
bool IsCombiningTimeToBeReceivedWithTransactions(TransportTransaction transaction, DispatchConsistency requiredDispatchConsistency, List<DeliveryConstraint> deliveryConstraints)
{
if (!settings.UseTransactionalQueues)
{
return false;
}
if (requiredDispatchConsistency == DispatchConsistency.Isolated)
{
return false;
}
DiscardIfNotReceivedBefore discardIfNotReceivedBefore;
var timeToBeReceivedRequested = deliveryConstraints.TryGet(out discardIfNotReceivedBefore) && discardIfNotReceivedBefore.MaxTime < MessageQueue.InfiniteTimeout;
if (!timeToBeReceivedRequested)
{
return false;
}
if (Transaction.Current != null)
{
return true;
}
MessageQueueTransaction activeReceiveTransaction;
return TryGetNativeTransaction(transaction, out activeReceiveTransaction);
}
static bool TryGetNativeTransaction(TransportTransaction transportTransaction, out MessageQueueTransaction transaction)
{
return transportTransaction.TryGet(out transaction);
}
void ExecuteTransportOperation(TransportTransaction transaction, UnicastTransportOperation transportOperation)
{
var message = transportOperation.Message;
var destination = transportOperation.Destination;
var destinationAddress = MsmqAddress.Parse(destination);
if (IsCombiningTimeToBeReceivedWithTransactions(
transaction,
transportOperation.RequiredDispatchConsistency,
transportOperation.DeliveryConstraints))
{
throw new Exception($"Failed to send message to address: {destinationAddress.Queue}@{destinationAddress.Machine}. Sending messages with a custom TimeToBeReceived is not supported on transactional MSMQ.");
}
try
{
using (var q = new MessageQueue(destinationAddress.FullPath, false, settings.UseConnectionCache, QueueAccessMode.Send))
{
using (var toSend = MsmqUtilities.Convert(message, transportOperation.DeliveryConstraints))
{
toSend.UseDeadLetterQueue = settings.UseDeadLetterQueue;
toSend.UseJournalQueue = settings.UseJournalQueue;
toSend.TimeToReachQueue = settings.TimeToReachQueue;
string replyToAddress;
if (message.Headers.TryGetValue(Headers.ReplyToAddress, out replyToAddress))
{
toSend.ResponseQueue = new MessageQueue(MsmqAddress.Parse(replyToAddress).FullPath);
}
var label = GetLabel(message);
if (transportOperation.RequiredDispatchConsistency == DispatchConsistency.Isolated)
{
q.Send(toSend, label, GetIsolatedTransactionType());
return;
}
MessageQueueTransaction activeTransaction;
if (TryGetNativeTransaction(transaction, out activeTransaction))
{
q.Send(toSend, label, activeTransaction);
return;
}
q.Send(toSend, label, GetTransactionTypeForSend());
}
}
}
catch (MessageQueueException ex)
{
if (ex.MessageQueueErrorCode == MessageQueueErrorCode.QueueNotFound)
{
var msg = destination == null
? "Failed to send message. Target address is null."
: $"Failed to send message to address: [{destination}]";
throw new QueueNotFoundException(destination, msg, ex);
}
ThrowFailedToSendException(destination, ex);
}
catch (Exception ex)
{
ThrowFailedToSendException(destination, ex);
}
}
public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transportTransaction, ContextBag context)
{
OutgoingTransportOperations = outgoingMessages;
TransportTransactionUsed = transportTransaction;
return TaskEx.CompletedTask;
}
public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transportTransaction, ContextBag context)
{
throw new Exception("simulated exception");
}
