public Task PublishOutboxItemAsync(
ISqlTransactionalOutboxItem <TUniqueIdentifier> outboxItem,
bool isFifoEnforcedProcessingEnabled = false
)
{
return(PublishingDelegateFunc.Invoke(outboxItem, isFifoEnforcedProcessingEnabled));
}
public static void AssertOutboxItemMatchesReceivedItem(
ISqlTransactionalOutboxItem <Guid> outboxItem,
ISqlTransactionalOutboxReceivedItem <Guid, string> receivedItem
)
{
Assert.IsNotNull(outboxItem);
Assert.IsNotNull(receivedItem);
Assert.AreEqual(outboxItem.UniqueIdentifier, receivedItem.UniqueIdentifier);
Assert.AreEqual(outboxItem.FifoGroupingIdentifier, receivedItem.FifoGroupingIdentifier);
Assert.AreEqual(OutboxReceivedItemProcessingStatus.AcknowledgeSuccessfulReceipt, receivedItem.Status);
var publishedItem = receivedItem.PublishedItem;
//Received Items should always be Successful
Assert.AreEqual(OutboxItemStatus.Successful, publishedItem.Status);
Assert.AreEqual(outboxItem.UniqueIdentifier, publishedItem.UniqueIdentifier);
Assert.AreEqual(outboxItem.FifoGroupingIdentifier, publishedItem.FifoGroupingIdentifier);
Assert.AreEqual(outboxItem.PublishTarget, publishedItem.PublishTarget);
//Publishing process should increment the PublishAttempt by 1
Assert.AreEqual(outboxItem.PublishAttempts + 1, publishedItem.PublishAttempts);
//Extract the Body from the original OutboxItem to compare (apples to apples) to the received item from
// being published where the Payload would be automatically populated from the Body property mapping.
var payloadBuilder = PayloadBuilder.FromJsonSafely(outboxItem.Payload);
Assert.AreEqual(payloadBuilder.Body, publishedItem.Payload);
//Dates need to be compared at the Millisecond level due to minute differences in Ticks when re-parsing(?)...
Assert.AreEqual(0,
(int)(publishedItem.CreatedDateTimeUtc - outboxItem.CreatedDateTimeUtc).TotalMilliseconds,
"The Created Dates differ at the Millisecond level"
);
}
public async Task PublishOutboxItemAsync(
ISqlTransactionalOutboxItem <TUniqueIdentifier> outboxItem,
bool isFifoEnforcedProcessingEnabled = false
)
{
var message = CreateEventBusMessage(outboxItem);
if (isFifoEnforcedProcessingEnabled && string.IsNullOrWhiteSpace(message.SessionId))
{
var sessionGuid = Guid.NewGuid();
Options.LogDebugCallback?.Invoke($"WARNING: FIFO Processing is Enabled but the Outbox Item [{outboxItem.UniqueIdentifier}]" +
$" does not have a valid FifoGrouping Identifier (e.g. SessionId for Azure Service Bus); " +
$" so delivery may fail therefore a surrogate Session GUID [{sessionGuid}] as been assigned to ensure delivery.");
message.SessionId = sessionGuid.ToString();
}
Options.LogDebugCallback?.Invoke($"Initializing Sender Client for Topic [{outboxItem.PublishTarget}]...");
await using var senderClient = this.AzureServiceBusClient.CreateSender(outboxItem.PublishTarget);
var uniqueIdString = ConvertUniqueIdentifierToString(outboxItem.UniqueIdentifier);
Options.LogDebugCallback?.Invoke($"Sending the Message [{message.Subject}] for outbox item [{uniqueIdString}]...");
await senderClient.SendMessageAsync(message);
Options.LogDebugCallback?.Invoke($"Azure Service Bus message [{message.Subject}] has been published successfully.");
}
private async Task AssertReceiptAndValidationOfThePublishedItem(ISqlTransactionalOutboxItem <Guid> outboxItem)
{
//*****************************************************************************************
//* Attempt to Retrieve/Receive the Message & Validate after Arrival!
//*****************************************************************************************
await using var azureServiceBusReceiver = new DefaultFifoAzureServiceBusReceiver <string>(
TestConfiguration.AzureServiceBusConnectionString,
TestConfiguration.AzureServiceBusTopic,
TestConfiguration.AzureServiceBusSubscription,
options: new AzureServiceBusReceivingOptions()
{
LogDebugCallback = (message) => Debug.WriteLine(message),
ErrorHandlerCallback = (exc) => Debug.WriteLine($"ERROR: {Environment.NewLine}{exc.GetMessagesRecursively()}")
}
);
int itemProcessedCount = 0;
try
{
var waitTime = IntegrationTestServiceBusDeliveryWaitTimeSpan;
await foreach (var item in azureServiceBusReceiver.AsAsyncEnumerable(waitTime))
{
Assert.IsNotNull(item, $"The received published outbox receivedItem is null! This should never happen!");
TestContext.Write($"Received receivedItem from Azure Service Bus receiver queue [{item.PublishedItem.UniqueIdentifier}]...");
//*****************************************************************************************
//* Validate the Item when it is detected/matched!
//*****************************************************************************************
if (item.PublishedItem.UniqueIdentifier == outboxItem.UniqueIdentifier)
{
//Finalize Status of the receivedItem we published as Successfully Received!
await item.AcknowledgeSuccessfulReceiptAsync();
itemProcessedCount++;
//VALIDATE the Matches of original Payload, Inserted item, and Received/Published Data!
TestHelper.AssertOutboxItemMatchesReceivedItem(outboxItem, item);
//We found the Item we expected, and all tests passed so we can break out and FINISH this test!
break;
}
else
{
await item.RejectAsDeadLetterAsync();
}
}
}
catch (OperationCanceledException)
{
Assert.Fail("The receivedItem published to Azure Service Bus was never received fore timing out!");
}
Assert.IsTrue(itemProcessedCount > 0, "We should have processed at least the one receivedItem we published!");
}
protected virtual async Task ProcessSingleOutboxItemInternal(
ISqlTransactionalOutboxItem <TUniqueIdentifier> item,
OutboxProcessingOptions options,
OutboxProcessingResults <TUniqueIdentifier> results
)
{
//This process will publish pending items, while also cleaning up Pending Items that need to be Failed because
// they couldn't be successfully published before and are exceeding the currently configured limits for
// retry attempts and/or time-to-live.
//NOTE: IF any of this fails due to issues with Sql Server it's ok because we have already transaction-ally
// secured the data in the outbox therefore we can repeat processing over-and-over under the promise of
// 'at-least-once' publishing attempt.
options.LogDebugCallback?.Invoke($"Processing Item [{item.UniqueIdentifier}]...");
//Validate the Item hasn't exceeded the Max Retry Attempts if enabled in the options...
if (options.MaxPublishingAttempts > 0 && item.PublishAttempts >= options.MaxPublishingAttempts)
{
options.LogDebugCallback?.Invoke(
$"Item [{item.UniqueIdentifier}] has failed due to exceeding the max number of" +
$" publishing attempts [{options.MaxPublishingAttempts}] with current PublishAttempts=[{item.PublishAttempts}]."
);
item.Status = OutboxItemStatus.FailedAttemptsExceeded;
results.FailedItems.Add(item);
}
//Validate the Item hasn't expired if enabled in the options...
else if (options.TimeSpanToLive > TimeSpan.Zero && DateTimeOffset.UtcNow.Subtract(item.CreatedDateTimeUtc) >= options.TimeSpanToLive)
{
options.LogDebugCallback?.Invoke(
$"Item [{item.UniqueIdentifier}] has failed due to exceeding the maximum time-to-live" +
$" [{options.TimeSpanToLive.ToElapsedTimeDescriptiveFormat()}] because it was created at [{item.CreatedDateTimeUtc}] UTC."
);
item.Status = OutboxItemStatus.FailedExpired;
results.FailedItems.Add(item);
}
//Finally attempt to publish the item...
else
{
item.PublishAttempts++;
await OutboxPublisher.PublishOutboxItemAsync(item, options.FifoEnforcedPublishingEnabled).ConfigureAwait(false);
options.LogDebugCallback?.Invoke(
$"Item [{item.UniqueIdentifier}] published successfully after [{item.PublishAttempts}] publishing attempt(s)!"
);
//Update the Status only AFTER successful Publishing!
item.Status = OutboxItemStatus.Successful;
results.SuccessfullyPublishedItems.Add(item);
options.LogDebugCallback?.Invoke(
$"Item [{item.UniqueIdentifier}] outbox status will be updated to [{item.Status}]."
);
}
}
public static void AssertOutboxItemsMatch(
ISqlTransactionalOutboxItem <Guid> leftItem,
ISqlTransactionalOutboxItem <Guid> rightItem
)
{
Assert.IsNotNull(leftItem);
Assert.IsNotNull(rightItem);
Assert.AreEqual(leftItem.UniqueIdentifier, rightItem.UniqueIdentifier);
Assert.AreEqual(leftItem.FifoGroupingIdentifier, rightItem.FifoGroupingIdentifier);
Assert.AreEqual(leftItem.PublishTarget, rightItem.PublishTarget);
Assert.AreEqual(leftItem.Payload, rightItem.Payload);
Assert.AreEqual(leftItem.Status, rightItem.Status);
Assert.AreEqual(leftItem.PublishAttempts, rightItem.PublishAttempts);
//Dates need to be compared at the Millisecond level due to minute differences in Ticks when re-parsing(?)...
Assert.AreEqual(0,
(int)(leftItem.CreatedDateTimeUtc - rightItem.CreatedDateTimeUtc).TotalMilliseconds,
"The Created Dates differ at the Millisecond level"
);
}
protected virtual JObject ParsePayloadAsJsonSafely(ISqlTransactionalOutboxItem <TUniqueIdentifier> outboxItem)
{
try
{
var json = JObject.Parse(outboxItem.Payload);
return(json);
}
catch (Exception exc)
{
if (Options.ThrowExceptionOnJsonPayloadParseFailure)
{
throw new ArgumentException(
$"Json parsing failure; the publishing payload for item [{outboxItem.UniqueIdentifier}] could not" +
$" be be parsed as Json.",
exc
);
}
return(null);
}
}
public OutboxReceivedItem(
ISqlTransactionalOutboxItem <TUniqueIdentifier> outboxItem,
ILookup <string, object> headersLookup,
string contentType,
Func <ISqlTransactionalOutboxItem <TUniqueIdentifier>, TPayloadBody> parsePayloadFunc,
string subject = null,
bool enableFifoEnforcedReceiving = false,
string fifoGroupingIdentifier = null,
string correlationId = null
)
{
InitBaseOutboxReceivedItem(
outboxItem,
headersLookup,
contentType,
parsePayloadFunc,
isFifoProcessingEnabled: enableFifoEnforcedReceiving,
subject: subject,
fifoGroupingIdentifier: fifoGroupingIdentifier,
correlationId: correlationId
);
}
protected void InitBaseOutboxReceivedItem(
ISqlTransactionalOutboxItem <TUniqueIdentifier> outboxItem,
ILookup <string, object> headersLookup,
string contentType,
Func <ISqlTransactionalOutboxItem <TUniqueIdentifier>, TPayloadBody> parsePayloadFunc,
bool isFifoProcessingEnabled,
string subject = null,
string fifoGroupingIdentifier = null,
string correlationId = null
)
{
Subject = subject;// Optional; Null if not specified or not supported.
PublishedItem = outboxItem.AssertNotNull(nameof(outboxItem));
HeadersLookup = headersLookup.AssertNotNull(nameof(headersLookup));
UniqueIdentifier = outboxItem.UniqueIdentifier;
ContentType = string.IsNullOrWhiteSpace(contentType) ? MessageContentTypes.PlainText : contentType;
ParsePayloadFunc = parsePayloadFunc.AssertNotNull(nameof(parsePayloadFunc));
CorrelationId = correlationId;
IsFifoEnforcedReceivingEnabled = isFifoProcessingEnabled;
FifoGroupingIdentifier = fifoGroupingIdentifier;
}
public TPayload ParsePayload(ISqlTransactionalOutboxItem <TUniqueIdentifier> outboxItem)
{
var payload = PayloadSerializer.DeserializePayload <TPayload>(outboxItem.Payload);
return(payload);
}
protected async Task HandleExceptionForOutboxItemFromQueueInternal(
ISqlTransactionalOutboxItem <TUniqueIdentifier> item,
Exception itemException,
OutboxProcessingOptions options,
OutboxProcessingResults <TUniqueIdentifier> results,
bool throwExceptionOnFailure,
HashSet <string> skipFifoGroups
)
{
var errorMessage = $"An Unexpected Exception occurred while processing outbox item [{item.UniqueIdentifier}].";
var fifoErrorMessage = $"FIFO Processing is enabled, but the outbox item [{item.UniqueIdentifier}] could not be published;" +
$" all associated items for the Fifo Group [{item.FifoGroupingIdentifier}] will be skipped.";
if (options.FifoEnforcedPublishingEnabled)
{
errorMessage = string.Concat(errorMessage, fifoErrorMessage);
}
var processingException = new Exception(errorMessage, itemException);
//Add Failed Item to the results
results.FailedItems.Add(item);
//Short circuit if we are configured to Throw the Error or if Enforce FIFO processing is enabled!
if (throwExceptionOnFailure)
{
options.ErrorHandlerCallback?.Invoke(processingException);
//If configured to throw an error then we Attempt to update the item before throwing exception
// because normally it would have been updated in bulk if exceptions were suppressed.
//NOTE: NO need to process results since we are throwing an Exception...
await UpdateProcessedItemsInternal(results, options);
throw processingException;
}
else if (options.FifoEnforcedPublishingEnabled)
{
//If Enforce Fifo processing is enabled then we also need to halt processing and drain the current queue to preserve
// the current order of processing; if we allow it to proceed we might publish the next item out of order.
//NOTE: This may create a block in the Queue if the issue isn't fixed but it's necessary to preserve
// the publishing order until the erroneous blocking item is resolved automatically (via connections restored,
// or item fails due to re-attempts), or manually (item is failed and/or removed manually).
options.LogDebugCallback?.Invoke(
$"FIFO Processing is enabled, but the outbox item [{item.UniqueIdentifier}] could not be published;" +
$" all following items for the FIFO Group [{item.FifoGroupingIdentifier}] will be skipped."
);
skipFifoGroups.Add(item.FifoGroupingIdentifier);
//ORIGINAL Logic before Fifo Grouping Identifiers were implemented...
//processingException = new Exception(
// $"The processing must be stopped because [{nameof(options.FifoEnforcedPublishingEnabled)}]"
// + $" configuration option is enabled; therefore to preserve the publishing order the remaining"
// + $" [{processingQueue.Count}] items will be delayed until the current item [{item.UniqueIdentifier}]"
// + " can be processed successfully or is failed out of the outbox queue.",
// processingException
//);
////Drain the Queue to halt current process...
//while (processingQueue.Count > 0)
//{
// results.SkippedItems.Add(processingQueue.Dequeue());
//}
}
//Log the latest initialized exception with details...
options.ErrorHandlerCallback?.Invoke(processingException);
}
protected virtual ServiceBusMessage CreateEventBusMessage(ISqlTransactionalOutboxItem <TUniqueIdentifier> outboxItem)
{
var uniqueIdString = ConvertUniqueIdentifierToString(outboxItem.UniqueIdentifier);
var defaultLabel = $"[PublishedBy={Options.SenderApplicationName}][MessageId={uniqueIdString}]";
ServiceBusMessage message = null;
Options.LogDebugCallback?.Invoke($"Creating Azure Message Object from [{uniqueIdString}]...");
//Optional FifoGrouping ID From Outbox Table should be used if specified...
var fifoGroupingId = !string.IsNullOrWhiteSpace(outboxItem.FifoGroupingIdentifier)
? outboxItem.FifoGroupingIdentifier.Trim()
: null;
//Attempt to decode teh Message as JObject to see if it contains dynamically defined parameters
//that need to be mapped into the message; otherwise if it's just a string we populate only the minimum.
var json = ParsePayloadAsJsonSafely(outboxItem);
if (json != null)
{
Options.LogDebugCallback?.Invoke($"Payload for [{uniqueIdString}] is valid Json; initializing dynamic Message Parameters from the Json root properties...");
//Get the session id; because it is needed to determine PartitionKey when defined...
fifoGroupingId ??= GetJsonValueSafely(json, JsonMessageFields.FifoGroupingId, (string)null)
?? GetJsonValueSafely(json, JsonMessageFields.SessionId, (string)null);
message = new ServiceBusMessage
{
MessageId = uniqueIdString,
SessionId = fifoGroupingId,
CorrelationId = GetJsonValueSafely(json, JsonMessageFields.CorrelationId, string.Empty),
To = GetJsonValueSafely(json, JsonMessageFields.To, string.Empty),
ReplyTo = GetJsonValueSafely(json, JsonMessageFields.ReplyTo, string.Empty),
ReplyToSessionId = GetJsonValueSafely(json, JsonMessageFields.ReplyToSessionId, string.Empty),
//NOTE: IF SessionId is Defined then the Partition Key MUST MATCH the SessionId...
PartitionKey = fifoGroupingId ?? GetJsonValueSafely(json, JsonMessageFields.PartitionKey, string.Empty),
//Transaction related Partition Key... unused so disabling this to minimize risk.
//ViaPartitionKey = sessionId ?? GetJsonValueSafely(json, "viaPartitionKey", string.Empty),
ContentType = GetJsonValueSafely(json, JsonMessageFields.ContentType, MessageContentTypes.Json),
Subject = GetJsonValueSafely <string>(json, JsonMessageFields.Subject)
?? GetJsonValueSafely(json, JsonMessageFields.Label, defaultLabel)
};
//Initialize the Body from dynamic Json if defined, or fallback to the entire body...
var messageBody = GetJsonValueSafely(json, JsonMessageFields.Body, outboxItem.Payload);
message.Body = new BinaryData(ConvertPublishingPayloadToBytes(messageBody));
//Populate HeadersLookup/User Properties if defined dynamically...
var headers = GetJsonValueSafely <JObject>(json, JsonMessageFields.Headers)
?? GetJsonValueSafely <JObject>(json, JsonMessageFields.AppProperties)
?? GetJsonValueSafely <JObject>(json, JsonMessageFields.UserProperties);
if (headers != null)
{
foreach (JProperty prop in headers.Properties())
{
message.ApplicationProperties.Add(
MessageHeaders.ToHeader(prop.Name.ToLower()),
prop.Value.ToString()
);
}
}
}
else
//Process as a string with no additional dynamic fields...
{
Options.LogDebugCallback?.Invoke($"Payload for [{uniqueIdString}] is in plain text format.");
message = new ServiceBusMessage
{
MessageId = uniqueIdString,
SessionId = fifoGroupingId,
CorrelationId = string.Empty,
Subject = defaultLabel,
ContentType = MessageContentTypes.PlainText,
Body = new BinaryData(ConvertPublishingPayloadToBytes(outboxItem.Payload))
};
}
//Add all default headers/user-properties...
var messageProps = message.ApplicationProperties;
messageProps.TryAdd(MessageHeaders.ProcessorType, nameof(SqlTransactionalOutbox));
messageProps.TryAdd(MessageHeaders.ProcessorSender, this.SenderApplicationName);
messageProps.TryAdd(MessageHeaders.OutboxUniqueIdentifier, uniqueIdString);
messageProps.TryAdd(MessageHeaders.OutboxCreatedDateUtc, outboxItem.CreatedDateTimeUtc);
messageProps.TryAdd(MessageHeaders.OutboxPublishingAttempts, outboxItem.PublishAttempts);
messageProps.TryAdd(MessageHeaders.OutboxPublishingTarget, outboxItem.PublishTarget);
return(message);
}