public static async Task <ISqlTransactionalOutboxProcessingResults <Guid> > ProcessPendingOutboxItemsAsync(
this SqlConnection sqlConnection,
ISqlTransactionalOutboxPublisher <Guid> outboxPublisher,
OutboxProcessingOptions processingOptions,
bool throwExceptionOnFailure = false
)
{
sqlConnection.AssertSqlConnectionIsValid();
outboxPublisher.AssertNotNull(nameof(outboxPublisher));
processingOptions.AssertNotNull(nameof(processingOptions));
await using var outboxTransaction = (SqlTransaction)(await sqlConnection.BeginTransactionAsync().ConfigureAwait(false));
try
{
var results = await outboxTransaction
.ProcessPendingOutboxItemsAsync(
outboxPublisher : outboxPublisher,
processingOptions : processingOptions,
throwExceptionOnFailure : throwExceptionOnFailure
)
.ConfigureAwait(false);
await outboxTransaction.CommitAsync().ConfigureAwait(false);
return(results);
}
catch (Exception)
{
await outboxTransaction.RollbackAsync().ConfigureAwait(false);
throw;
}
}
public AsyncThreadOutboxProcessingAgent(
TimeSpan processingIntervalTimeSpan,
string sqlConnectionString,
ISqlTransactionalOutboxPublisher <Guid> outboxPublisher,
OutboxProcessingOptions outboxProcessingOptions = null
) : this(processingIntervalTimeSpan, TimeSpan.Zero, sqlConnectionString, outboxPublisher, outboxProcessingOptions)
{
}
public AsyncThreadOutboxProcessingAgent(
TimeSpan processingIntervalTimeSpan,
TimeSpan historyToKeepTimeSpan,
string sqlConnectionString,
ISqlTransactionalOutboxPublisher <Guid> outboxPublisher,
OutboxProcessingOptions outboxProcessingOptions = null
)
{
this.ProcessingIntervalTimespan = processingIntervalTimeSpan.AssertNotNull(nameof(processingIntervalTimeSpan));
this.HistoryToKeepTimespan = historyToKeepTimeSpan.AssertNotNull(nameof(historyToKeepTimeSpan));
this.SqlConnectionString = sqlConnectionString.AssertNotNullOrWhiteSpace(nameof(sqlConnectionString));
this.SqlTransactionalOutboxPublisher = outboxPublisher.AssertNotNull(nameof(outboxPublisher));
this.SqlTransactionalOutboxProcessingOptions = outboxProcessingOptions ?? OutboxProcessingOptions.DefaultOutboxProcessingOptions;
}
public static async Task <ISqlTransactionalOutboxProcessingResults <Guid> > ProcessPendingOutboxItemsAsync(
this SqlConnection sqlConnection,
ISqlTransactionalOutboxPublisher <Guid> outboxPublisher,
OutboxProcessingOptions processingOptions,
bool throwExceptionOnFailure = false
)
{
sqlConnection.AssertSqlConnectionIsValid();
outboxPublisher.AssertNotNull(nameof(outboxPublisher));
processingOptions.AssertNotNull(nameof(processingOptions));
await using var outboxTransaction = (SqlTransaction)(await sqlConnection.BeginTransactionAsync().ConfigureAwait(false));
try
{
var results = await outboxTransaction
.ProcessPendingOutboxItemsAsync(
outboxPublisher : outboxPublisher,
processingOptions : processingOptions,
throwExceptionOnFailure : throwExceptionOnFailure
)
.ConfigureAwait(false);
await outboxTransaction.CommitAsync().ConfigureAwait(false);
return(results);
}
catch (Exception exc)
{
//FIRST Rollback any pending changes...
await outboxTransaction.RollbackAsync().ConfigureAwait(false);
try
{
//THEN Attempt any Mitigating Actions for the Issue...
//IF WE have issues retrieving the new items from the DB then we attempt to increment the
// Publish Attempts in case there is an issue with the entry that is causing failures, so
// that any potential problematic items will eventually be failed out and skipped.
await sqlConnection.IncrementPublishAttemptsForAllPendingItemsAsync(outboxPublisher).ConfigureAwait(false);
}
catch (Exception mitigationExc)
{
throw new AggregateException(new[] { exc, mitigationExc });
}
//FINALLY Re-throw to ensure we don't black hole the issue...
throw;
}
}
public static async Task Run([TimerTrigger("%TransactionalOutboxAgentCronSchedule%")] TimerInfo myTimer, ILogger log)
{
log.LogInformation($"Transactional Outbox Agent initiating process at: {DateTime.Now}");
var configSettings = new SampleAppConfig();
var azureServiceBusPublisher = new DefaultAzureServiceBusOutboxPublisher(
configSettings.AzureServiceBusConnectionString,
new AzureServiceBusPublishingOptions()
{
SenderApplicationName = $"{typeof(TransactionalOutboxAgentFunction).Assembly.GetName().Name}.{nameof(TransactionalOutboxAgentFunction)}",
LogDebugCallback = (s) => log.LogDebug(s),
ErrorHandlerCallback = (e) => log.LogError(e, "Unexpected Exception occurred while Processing the Transactional Outbox.")
}
);
var outboxProcessingOptions = new OutboxProcessingOptions()
{
//ItemProcessingBatchSize = 200, //Only process the top X items to keep this function responsive!
FifoEnforcedPublishingEnabled = true, //The Service Bus Topic is Session Enabled so we must processes it with FIFO Processing Enabled!
LogDebugCallback = (m) => log.LogDebug(m),
ErrorHandlerCallback = (e) => log.LogError(e, "Transactional Outbox Processing Exception"),
MaxPublishingAttempts = configSettings.OutboxMaxPublishingRetryAttempts,
TimeSpanToLive = configSettings.OutboxMaxTimeToLiveTimeSpan
};
//************************************************************
//*** Execute processing of the Transactional Outbox...
//************************************************************
await using var sqlConnection = new SqlConnection(configSettings.SqlConnectionString);
await sqlConnection.OpenAsync().ConfigureAwait(false);
await sqlConnection
.ProcessPendingOutboxItemsAsync(azureServiceBusPublisher, outboxProcessingOptions)
.ConfigureAwait(false);
//************************************************************
//*** Execute Cleanup of Historical Outbox Data...
//************************************************************
await sqlConnection
.CleanupHistoricalOutboxItemsAsync(configSettings.OutboxHistoryToKeepTimeSpan)
.ConfigureAwait(false);
}
public static async Task <ISqlTransactionalOutboxProcessingResults <Guid> > ProcessPendingOutboxItemsAsync(
this SqlTransaction sqlTransaction,
ISqlTransactionalOutboxPublisher <Guid> outboxPublisher,
OutboxProcessingOptions processingOptions,
bool throwExceptionOnFailure = false
)
{
sqlTransaction.AssertSqlTransactionIsValid();
outboxPublisher.AssertNotNull(nameof(outboxPublisher));
processingOptions.AssertNotNull(nameof(processingOptions));
//NOTE: Payload type isn't important when Publishing because we publish the already serialized
// payload anyway so to simplify the custom extension signature we can just use string payload type here.
var outboxProcessor = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction, outboxPublisher);
var results = await outboxProcessor
.ProcessPendingOutboxItemsAsync(processingOptions, throwExceptionOnFailure)
.ConfigureAwait(false);
return(results);
}
public async Task <List <ISqlTransactionalOutboxItem <Guid> > > DoTestTransactionalOutboxCrawlingOfBlockingFailureItems(
int testDataSize, int maxPublishingAttempts, bool enforceFifoProcessing, bool throwExceptionOnFailedItem)
{
//*****************************************************************************************
//* STEP 1 - Prepare/Clear the Queue Table
//*****************************************************************************************
await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(testDataSize);
//*****************************************************************************************
//* STEP 2 - Setup Custom Publisher & Processing Options...
//*****************************************************************************************
var publishedAttemptsList = new List <ISqlTransactionalOutboxItem <Guid> >();
var failingPublisher = new TestHarnessSqlTransactionalOutboxPublisher(
(i, isFifoEnabled) =>
{
publishedAttemptsList.Add(i);
TestContext.WriteLine($"Successful -- We have intentionally Failed to Publish Item: {i.UniqueIdentifier}");
//Force an Error on Failure... this should result in ALL Publishing attempts to fail...
throw new Exception("Failed to Publish!");
}
);
var outboxProcessingOptions = new OutboxProcessingOptions()
{
MaxPublishingAttempts = maxPublishingAttempts,
FifoEnforcedPublishingEnabled = enforceFifoProcessing
};
//*****************************************************************************************
//* STEP 3 - Executing processing of the Pending Items in the Queue...
//*****************************************************************************************
//Execute Processing of Items just inserted!
//NOTE: We need to re-initialize a NEW Transaction and Processor to correctly simulate this running separately!
ISqlTransactionalOutboxProcessingResults <Guid> publishedResults = null;
int loopCounter = 0;
bool handledExceptionSoItsOkToContinue = false;
do
{
await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
await using var sqlTransaction = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false);
handledExceptionSoItsOkToContinue = false;
try
{
publishedResults = await sqlTransaction.ProcessPendingOutboxItemsAsync(
failingPublisher,
outboxProcessingOptions,
throwExceptionOnFailedItem
).ConfigureAwait(false);
}
catch (Exception exc)
{
if (throwExceptionOnFailedItem)
{
//DO Nothing, as we Expect there to be exceptions when Throw Exception on Failure is Enabled!
TestContext.WriteLine($"Successfully handled expected Exception: {exc.Message}");
publishedResults = new OutboxProcessingResults <Guid>();
handledExceptionSoItsOkToContinue = true;
}
else
{
//IF we get an exception but ThrowExceptionOnFailure is disabled, then this is an issue!
throw;
}
}
await sqlTransaction.CommitAsync();
//Provide Infinite Loop fail-safe...
loopCounter++;
Assert.IsTrue(loopCounter <= (testDataSize * maxPublishingAttempts * 2), $"Infinite Loop Breaker Tripped at [{loopCounter}]!");
//Assert there are never any successfully published items...
Assert.AreEqual(0, publishedResults.SuccessfullyPublishedItems.Count);
} while (publishedResults.FailedItems.Count > 0 || (throwExceptionOnFailedItem && handledExceptionSoItsOkToContinue));
//*****************************************************************************************
//* STEP 4 - Retrieve and Validate Data in the Database is updated and all have failed out...
//*****************************************************************************************
//Assert All Items in the DB are Successful!
await using var sqlConnection2 = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
await using var sqlTransaction2 = (SqlTransaction) await sqlConnection2.BeginTransactionAsync().ConfigureAwait(false);
var outboxProcessor2 = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction2, failingPublisher);
var successfulResultsFromDb = await outboxProcessor2.OutboxRepository
.RetrieveOutboxItemsAsync(OutboxItemStatus.Pending)
.ConfigureAwait(false);
//Assert the results from the DB match those returned from the Processing method...
Assert.AreEqual(successfulResultsFromDb.Count, 0);
var failedResultsFromDb = await outboxProcessor2.OutboxRepository
.RetrieveOutboxItemsAsync(OutboxItemStatus.FailedAttemptsExceeded)
.ConfigureAwait(false);
//Assert the results from the DB match those returned from the Processing method...
Assert.AreEqual(failedResultsFromDb.Count * maxPublishingAttempts, publishedAttemptsList.Count);
foreach (var dbItem in failedResultsFromDb)
{
Assert.AreEqual(OutboxItemStatus.FailedAttemptsExceeded, dbItem.Status);
Assert.AreEqual(maxPublishingAttempts, dbItem.PublishAttempts);
}
//RETURN the Attempted Publishing list for additional validation based on the Pattern
// we expect for items to be processed.
return(publishedAttemptsList);
}
