public async Task TestNewOutboxItemInsertionByBatch()
{
//Organize
await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
//Clear the Table data for the test...
await sqlConnection.TruncateTransactionalOutboxTableAsync();
//Initialize Transaction and Outbox Processor
var outboxTestItems = TestHelper.CreateTestStringOutboxItemData(100);
//Execute
var timer = Stopwatch.StartNew();
var insertedResults = await sqlConnection
.AddTransactionalOutboxPendingItemListAsync(outboxTestItems)
.ConfigureAwait(false);
timer.Stop();
TestContext?.WriteLine($"Inserted [{insertedResults.Count}] items in [{timer.Elapsed.ToElapsedTimeDescriptiveFormat()}].");
//Assert
Assert.AreEqual(insertedResults.Count, outboxTestItems.Count);
insertedResults.ForEach(i =>
{
//Validate Created Date Time (can't match precisely but can validate it was populated as expected...
Assert.AreEqual(OutboxItemStatus.Pending, i.Status);
Assert.AreNotEqual(DateTimeOffset.MinValue, i.CreatedDateTimeUtc);
Assert.AreEqual(0, i.PublishAttempts);
Assert.IsFalse(string.IsNullOrWhiteSpace(i.PublishTarget));
Assert.IsFalse(string.IsNullOrWhiteSpace(i.Payload));
Assert.AreNotEqual(Guid.Empty, i.UniqueIdentifier);
});
}
public async Task TestOutboxHistoryCleanupTests()
{
//Clear the Table data for the test...
int testDataSize = 100;
await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(testDataSize, true);
var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
var insertedCount = await RetrievePendingItemsCountAsync(sqlConnection);
Assert.AreEqual(testDataSize, insertedCount);
await Task.Delay(TimeSpan.FromSeconds(5));
int nonPurgedDataSize = 10;
await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(nonPurgedDataSize, false);
//Execute
await sqlConnection
.CleanupHistoricalOutboxItemsAsync(TimeSpan.FromSeconds(3))
.ConfigureAwait(false);
//Assert
var purgedCount = await RetrievePendingItemsCountAsync(sqlConnection);
Assert.AreEqual(nonPurgedDataSize, purgedCount);
}
public async Task TestNewOutboxItemInsertionBenchmark()
{
//Organize
await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
//WARM UP...
var warmupTestItems = TestHelper.CreateTestStringOutboxItemData(2);
var warmupTimer = Stopwatch.StartNew();
var warmupResults = await sqlConnection
.AddTransactionalOutboxPendingItemListAsync(warmupTestItems)
.ConfigureAwait(false);
warmupTimer.Stop();
TestContext?.WriteLine($"Warm-up Inserted [{warmupResults.Count}] items in [{warmupTimer.Elapsed.ToElapsedTimeDescriptiveFormat()}].");
//Execute... Full Test...
var executionTestItems = TestHelper.CreateTestStringOutboxItemData(1000);
var timer = Stopwatch.StartNew();
var executionResults = await sqlConnection
.AddTransactionalOutboxPendingItemListAsync(warmupTestItems)
.ConfigureAwait(false);
timer.Stop();
TestContext?.WriteLine($"Benchmark Execution Inserted [{executionResults.Count}] items in [{timer.Elapsed.ToElapsedTimeDescriptiveFormat()}].");
}
public async Task TestAzureServiceBusTransactionalOutboxWithJsonPayload()
{
//*****************************************************************************************
//* STEP 1 - Prepare the Test Payload Item to be Published
//*****************************************************************************************
var testPayload = new
{
PublishTarget = TestConfiguration.AzureServiceBusTopic,
To = "CajunCoding",
FifoGroupingId = nameof(TestAzureServiceBusDirectPublishingAndReceiving),
ContentType = MessageContentTypes.PlainText,
Body = $"Testing publishing of Json Payload with PlainText Body for [{nameof(TestAzureServiceBusTransactionalOutboxWithJsonPayload)}]!",
Headers = new
{
IntegrationTestName = nameof(TestAzureServiceBusDirectPublishingAndReceiving),
IntegrationTestExecutionDateTime = DateTimeOffset.UtcNow
}
};
var jsonText = JsonConvert.SerializeObject(testPayload);
//*****************************************************************************************
//* STEP 2 - Store the Dynamic Payload into the Outbox!
//*****************************************************************************************
var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
await sqlConnection.TruncateTransactionalOutboxTableAsync();
var outboxItem = await sqlConnection
.AddTransactionalOutboxPendingItemAsync(jsonText)
.ConfigureAwait(false);
//*****************************************************************************************
//* STEP 3 - Now Publish the Outbox payloads (processing all pending items)...
//*****************************************************************************************
await using var azureServiceBusPublisher = new DefaultAzureServiceBusOutboxPublisher(
TestConfiguration.AzureServiceBusConnectionString,
new AzureServiceBusPublishingOptions()
{
LogDebugCallback = s => TestContext.WriteLine(s),
ErrorHandlerCallback = e => TestContext.WriteLine(e.Message + e.InnerException?.Message)
}
);
var processedResults = await sqlConnection.ProcessPendingOutboxItemsAsync(azureServiceBusPublisher, new OutboxProcessingOptions()
{
FifoEnforcedPublishingEnabled = true,
LogDebugCallback = s => TestContext.WriteLine(s),
ErrorHandlerCallback = e => TestContext.WriteLine(e.Message + e.InnerException?.Message),
MaxPublishingAttempts = 1,
TimeSpanToLive = TimeSpan.FromMinutes(5)
});
//*****************************************************************************************
//* STEP 4 - Attempt to Retrieve/Receive the Message & Validate after Arrival!
//*****************************************************************************************
await AssertReceiptAndValidationOfThePublishedItem(outboxItem);
}
public async Task TestNewOutboxItemInsertionIndividualItemsViaConvenienceMethods()
{
//Organize
await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
//Clear the Table data for the test...
await sqlConnection.TruncateTransactionalOutboxTableAsync();
//Initialize Transaction and Outbox Processor
var outboxTestItems = TestHelper.CreateTestStringOutboxItemData(2);
//Execute
var timer = Stopwatch.StartNew();
//Add Item 1!
var insertedResult1 = await sqlConnection
.AddTransactionalOutboxPendingItemAsync(
publishTarget : outboxTestItems[0].PublishingTarget,
payload : outboxTestItems[0].PublishingPayload,
fifoGroupingIdentifier : outboxTestItems[0].FifoGroupingIdentifier
)
.ConfigureAwait(false);
TestContext?.WriteLine($"Inserted First Item in [{timer.Elapsed.ToElapsedTimeDescriptiveFormat()}].");
Assert.IsNotNull(insertedResult1);
Assert.IsNotNull(insertedResult1.UniqueIdentifier);
Assert.IsNotNull(insertedResult1.FifoGroupingIdentifier);
Assert.AreEqual(DateTime.UtcNow.Date, insertedResult1.CreatedDateTimeUtc.Date);
//Add Item 2 (No FifoGroupingIdentifier)!
var insertedResult2 = await sqlConnection
.AddTransactionalOutboxPendingItemAsync(
publishTarget : outboxTestItems[1].PublishingTarget,
payload : outboxTestItems[1].PublishingPayload,
fifoGroupingIdentifier : outboxTestItems[1].FifoGroupingIdentifier
)
.ConfigureAwait(false);
TestContext?.WriteLine($"Inserted First Item in [{timer.Elapsed.ToElapsedTimeDescriptiveFormat()}].");
Assert.IsNotNull(insertedResult2);
Assert.IsNotNull(insertedResult2.UniqueIdentifier);
Assert.IsNotNull(insertedResult2.FifoGroupingIdentifier);
Assert.AreEqual(DateTime.UtcNow.Date, insertedResult2.CreatedDateTimeUtc.Date);
timer.Stop();
}
public async Task TestNewOutboxItemInsertionAndRetrieval()
{
var testPublisher = new TestHarnessSqlTransactionalOutboxPublisher();
//Create Test Data
var insertedResults = await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(100);
//Initialize Transaction and Outbox Processor
await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync().ConfigureAwait(false);
await using var sqlTransaction = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false);
var outboxProcessor = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction, testPublisher);
//RETRIEVE ALL Pending Items from the Outbox to validate
var pendingResults = await outboxProcessor.OutboxRepository
.RetrieveOutboxItemsAsync(OutboxItemStatus.Pending)
.ConfigureAwait(false);
//Assert
TestHelper.AssertOutboxItemResultsMatch(insertedResults, pendingResults);
}
public async Task TestTransactionalOutboxEndToEndSuccessfulProcessing()
{
//Organize
await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync().ConfigureAwait(false);
//*****************************************************************************************
//* STEP 1 - Prepare/Clear the Queue Table
//*****************************************************************************************
await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(100);
//*****************************************************************************************
//* STEP 3 - Executing processing of the Pending Items in the Queue...
//*****************************************************************************************
//Execute Processing of Items just inserted!
//NOTE: We need to re-initialize a NEW Transaction and Processor to correctly simulate this running separately!
var publishedItemList = new List <ISqlTransactionalOutboxItem <Guid> >();
var testPublisher = new TestHarnessSqlTransactionalOutboxPublisher((item, isFifoEnabled) =>
{
publishedItemList.Add(item);
TestContext.WriteLine($"Successfully Published Item: {item.UniqueIdentifier}");
return(Task.CompletedTask);
}
);
var publishedResults = await sqlConnection
.ProcessPendingOutboxItemsAsync(testPublisher, new OutboxProcessingOptions())
.ConfigureAwait(false);
//Assert results
Assert.AreEqual(publishedItemList.Count, publishedResults.SuccessfullyPublishedItems.Count);
Assert.AreEqual(0, publishedResults.FailedItems.Count);
//Assert Unique Items all match
var publishedItemLookup = publishedItemList.ToLookup(i => i.UniqueIdentifier);
publishedResults.SuccessfullyPublishedItems.ForEach(r =>
{
Assert.IsTrue(publishedItemLookup.Contains(r.UniqueIdentifier));
});
//*****************************************************************************************
//* STEP 4 - Retrieve and Validate Data is updated and no pending Items Remain...
//*****************************************************************************************
//Assert All Items in the DB are Successful!
await using var sqlTransaction3 = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false);
var outboxProcessor = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction3, testPublisher);
var successfulResultsFromDb = await outboxProcessor.OutboxRepository
.RetrieveOutboxItemsAsync(OutboxItemStatus.Successful)
.ConfigureAwait(false);
//Assert the results from the DB match those returned from the Processing method...
Assert.AreEqual(publishedResults.SuccessfullyPublishedItems.Count, successfulResultsFromDb.Count);
successfulResultsFromDb.ForEach(dbItem =>
{
Assert.AreEqual(OutboxItemStatus.Successful, dbItem.Status);
Assert.AreEqual(1, dbItem.PublishAttempts);
});
}
public async Task TestTransactionalOutboxFailureByTimeToLive()
{
var failedItemTestDataSizeByBatch = 3;
var successfulItemTestDataSize = 3;
var timeToLiveTimeSpan = TimeSpan.FromSeconds(5);
var testHarnessPublisher = new TestHarnessSqlTransactionalOutboxPublisher();
var expectedTotalSuccessCount = 0;
var expectedTotalExpiredCount = 0;
//*****************************************************************************************
//* STEP 1 - Prepare/Clear the Queue Table and populate initial Set of items (expected to Faile/Expire)
// then wait for them to expire...
//*****************************************************************************************
await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(failedItemTestDataSizeByBatch);
expectedTotalExpiredCount += failedItemTestDataSizeByBatch;
await Task.Delay(timeToLiveTimeSpan + TimeSpan.FromSeconds(1));
//*****************************************************************************************
//* STEP 2 - Add a Second & Third batch of items with different insertion Timestamps to
// ensure only the first set expire as expected...
//*****************************************************************************************
await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(successfulItemTestDataSize, false);
expectedTotalSuccessCount += successfulItemTestDataSize;
//Insert in a second batch to force different Creation Dates at the DB level...
await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(successfulItemTestDataSize, false);
expectedTotalSuccessCount += successfulItemTestDataSize;
//*****************************************************************************************
//* STEP 2 - Process Outbox and get Results
//*****************************************************************************************
await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
var processingResults = await sqlConnection.ProcessPendingOutboxItemsAsync(testHarnessPublisher, new OutboxProcessingOptions()
{
TimeSpanToLive = timeToLiveTimeSpan
});
//*****************************************************************************************
//* STEP 3 - Validate Results returned!
//*****************************************************************************************
Assert.AreEqual(expectedTotalExpiredCount, processingResults.FailedItems.Count);
Assert.AreEqual(expectedTotalSuccessCount, processingResults.SuccessfullyPublishedItems.Count);
//We expect all items to be processed before any item is failed....
//So the First Item will be repeated as the 10'th item after the next 9 are also attempted...
processingResults.SuccessfullyPublishedItems.ForEach(i =>
{
Assert.AreEqual(OutboxItemStatus.Successful, i.Status);
});
processingResults.FailedItems.ForEach(i =>
{
Assert.AreEqual(OutboxItemStatus.FailedExpired, i.Status);
});
//*****************************************************************************************
//* STEP 3 - Validate Results In the DB!
//*****************************************************************************************
await using var sqlTransaction2 = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false);
var outboxProcessor = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction2, testHarnessPublisher);
var outboxRepository = outboxProcessor.OutboxRepository;
var successfulItems = await outboxRepository.RetrieveOutboxItemsAsync(OutboxItemStatus.Successful);
Assert.AreEqual(expectedTotalSuccessCount, successfulItems.Count);
successfulItems.ForEach(i =>
{
Assert.AreEqual(OutboxItemStatus.Successful, i.Status);
});
var failedItems = await outboxRepository.RetrieveOutboxItemsAsync(OutboxItemStatus.FailedExpired);
Assert.AreEqual(expectedTotalExpiredCount, failedItems.Count);
processingResults.FailedItems.ForEach(i =>
{
Assert.AreEqual(OutboxItemStatus.FailedExpired, i.Status);
});
}
public async Task <List <ISqlTransactionalOutboxItem <Guid> > > DoTestTransactionalOutboxCrawlingOfBlockingFailureItems(
int testDataSize, int maxPublishingAttempts, bool enforceFifoProcessing, bool throwExceptionOnFailedItem)
{
//*****************************************************************************************
//* STEP 1 - Prepare/Clear the Queue Table
//*****************************************************************************************
await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(testDataSize);
//*****************************************************************************************
//* STEP 2 - Setup Custom Publisher & Processing Options...
//*****************************************************************************************
var publishedAttemptsList = new List <ISqlTransactionalOutboxItem <Guid> >();
var failingPublisher = new TestHarnessSqlTransactionalOutboxPublisher(
(i, isFifoEnabled) =>
{
publishedAttemptsList.Add(i);
TestContext.WriteLine($"Successful -- We have intentionally Failed to Publish Item: {i.UniqueIdentifier}");
//Force an Error on Failure... this should result in ALL Publishing attempts to fail...
throw new Exception("Failed to Publish!");
}
);
var outboxProcessingOptions = new OutboxProcessingOptions()
{
MaxPublishingAttempts = maxPublishingAttempts,
FifoEnforcedPublishingEnabled = enforceFifoProcessing
};
//*****************************************************************************************
//* STEP 3 - Executing processing of the Pending Items in the Queue...
//*****************************************************************************************
//Execute Processing of Items just inserted!
//NOTE: We need to re-initialize a NEW Transaction and Processor to correctly simulate this running separately!
ISqlTransactionalOutboxProcessingResults <Guid> publishedResults = null;
int loopCounter = 0;
bool handledExceptionSoItsOkToContinue = false;
do
{
await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
await using var sqlTransaction = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false);
handledExceptionSoItsOkToContinue = false;
try
{
publishedResults = await sqlTransaction.ProcessPendingOutboxItemsAsync(
failingPublisher,
outboxProcessingOptions,
throwExceptionOnFailedItem
).ConfigureAwait(false);
}
catch (Exception exc)
{
if (throwExceptionOnFailedItem)
{
//DO Nothing, as we Expect there to be exceptions when Throw Exception on Failure is Enabled!
TestContext.WriteLine($"Successfully handled expected Exception: {exc.Message}");
publishedResults = new OutboxProcessingResults <Guid>();
handledExceptionSoItsOkToContinue = true;
}
else
{
//IF we get an exception but ThrowExceptionOnFailure is disabled, then this is an issue!
throw;
}
}
await sqlTransaction.CommitAsync();
//Provide Infinite Loop fail-safe...
loopCounter++;
Assert.IsTrue(loopCounter <= (testDataSize * maxPublishingAttempts * 2), $"Infinite Loop Breaker Tripped at [{loopCounter}]!");
//Assert there are never any successfully published items...
Assert.AreEqual(0, publishedResults.SuccessfullyPublishedItems.Count);
} while (publishedResults.FailedItems.Count > 0 || (throwExceptionOnFailedItem && handledExceptionSoItsOkToContinue));
//*****************************************************************************************
//* STEP 4 - Retrieve and Validate Data in the Database is updated and all have failed out...
//*****************************************************************************************
//Assert All Items in the DB are Successful!
await using var sqlConnection2 = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
await using var sqlTransaction2 = (SqlTransaction) await sqlConnection2.BeginTransactionAsync().ConfigureAwait(false);
var outboxProcessor2 = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction2, failingPublisher);
var successfulResultsFromDb = await outboxProcessor2.OutboxRepository
.RetrieveOutboxItemsAsync(OutboxItemStatus.Pending)
.ConfigureAwait(false);
//Assert the results from the DB match those returned from the Processing method...
Assert.AreEqual(successfulResultsFromDb.Count, 0);
var failedResultsFromDb = await outboxProcessor2.OutboxRepository
.RetrieveOutboxItemsAsync(OutboxItemStatus.FailedAttemptsExceeded)
.ConfigureAwait(false);
//Assert the results from the DB match those returned from the Processing method...
Assert.AreEqual(failedResultsFromDb.Count * maxPublishingAttempts, publishedAttemptsList.Count);
foreach (var dbItem in failedResultsFromDb)
{
Assert.AreEqual(OutboxItemStatus.FailedAttemptsExceeded, dbItem.Status);
Assert.AreEqual(maxPublishingAttempts, dbItem.PublishAttempts);
}
//RETURN the Attempted Publishing list for additional validation based on the Pattern
// we expect for items to be processed.
return(publishedAttemptsList);
}
protected static async Task CleanupOutboxAsync()
{
await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync();
await sqlConnection.TruncateTransactionalOutboxTableAsync();
}