public async Task TestNewOutboxItemInsertionByBatch() { //Organize await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); //Clear the Table data for the test... await sqlConnection.TruncateTransactionalOutboxTableAsync(); //Initialize Transaction and Outbox Processor var outboxTestItems = TestHelper.CreateTestStringOutboxItemData(100); //Execute var timer = Stopwatch.StartNew(); var insertedResults = await sqlConnection .AddTransactionalOutboxPendingItemListAsync(outboxTestItems) .ConfigureAwait(false); timer.Stop(); TestContext?.WriteLine($"Inserted [{insertedResults.Count}] items in [{timer.Elapsed.ToElapsedTimeDescriptiveFormat()}]."); //Assert Assert.AreEqual(insertedResults.Count, outboxTestItems.Count); insertedResults.ForEach(i => { //Validate Created Date Time (can't match precisely but can validate it was populated as expected... Assert.AreEqual(OutboxItemStatus.Pending, i.Status); Assert.AreNotEqual(DateTimeOffset.MinValue, i.CreatedDateTimeUtc); Assert.AreEqual(0, i.PublishAttempts); Assert.IsFalse(string.IsNullOrWhiteSpace(i.PublishTarget)); Assert.IsFalse(string.IsNullOrWhiteSpace(i.Payload)); Assert.AreNotEqual(Guid.Empty, i.UniqueIdentifier); }); }
public async Task TestOutboxHistoryCleanupTests() { //Clear the Table data for the test... int testDataSize = 100; await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(testDataSize, true); var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); var insertedCount = await RetrievePendingItemsCountAsync(sqlConnection); Assert.AreEqual(testDataSize, insertedCount); await Task.Delay(TimeSpan.FromSeconds(5)); int nonPurgedDataSize = 10; await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(nonPurgedDataSize, false); //Execute await sqlConnection .CleanupHistoricalOutboxItemsAsync(TimeSpan.FromSeconds(3)) .ConfigureAwait(false); //Assert var purgedCount = await RetrievePendingItemsCountAsync(sqlConnection); Assert.AreEqual(nonPurgedDataSize, purgedCount); }
public async Task TestNewOutboxItemInsertionBenchmark() { //Organize await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); //WARM UP... var warmupTestItems = TestHelper.CreateTestStringOutboxItemData(2); var warmupTimer = Stopwatch.StartNew(); var warmupResults = await sqlConnection .AddTransactionalOutboxPendingItemListAsync(warmupTestItems) .ConfigureAwait(false); warmupTimer.Stop(); TestContext?.WriteLine($"Warm-up Inserted [{warmupResults.Count}] items in [{warmupTimer.Elapsed.ToElapsedTimeDescriptiveFormat()}]."); //Execute... Full Test... var executionTestItems = TestHelper.CreateTestStringOutboxItemData(1000); var timer = Stopwatch.StartNew(); var executionResults = await sqlConnection .AddTransactionalOutboxPendingItemListAsync(warmupTestItems) .ConfigureAwait(false); timer.Stop(); TestContext?.WriteLine($"Benchmark Execution Inserted [{executionResults.Count}] items in [{timer.Elapsed.ToElapsedTimeDescriptiveFormat()}]."); }
public async Task TestAzureServiceBusTransactionalOutboxWithJsonPayload() { //***************************************************************************************** //* STEP 1 - Prepare the Test Payload Item to be Published //***************************************************************************************** var testPayload = new { PublishTarget = TestConfiguration.AzureServiceBusTopic, To = "CajunCoding", FifoGroupingId = nameof(TestAzureServiceBusDirectPublishingAndReceiving), ContentType = MessageContentTypes.PlainText, Body = $"Testing publishing of Json Payload with PlainText Body for [{nameof(TestAzureServiceBusTransactionalOutboxWithJsonPayload)}]!", Headers = new { IntegrationTestName = nameof(TestAzureServiceBusDirectPublishingAndReceiving), IntegrationTestExecutionDateTime = DateTimeOffset.UtcNow } }; var jsonText = JsonConvert.SerializeObject(testPayload); //***************************************************************************************** //* STEP 2 - Store the Dynamic Payload into the Outbox! //***************************************************************************************** var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); await sqlConnection.TruncateTransactionalOutboxTableAsync(); var outboxItem = await sqlConnection .AddTransactionalOutboxPendingItemAsync(jsonText) .ConfigureAwait(false); //***************************************************************************************** //* STEP 3 - Now Publish the Outbox payloads (processing all pending items)... //***************************************************************************************** await using var azureServiceBusPublisher = new DefaultAzureServiceBusOutboxPublisher( TestConfiguration.AzureServiceBusConnectionString, new AzureServiceBusPublishingOptions() { LogDebugCallback = s => TestContext.WriteLine(s), ErrorHandlerCallback = e => TestContext.WriteLine(e.Message + e.InnerException?.Message) } ); var processedResults = await sqlConnection.ProcessPendingOutboxItemsAsync(azureServiceBusPublisher, new OutboxProcessingOptions() { FifoEnforcedPublishingEnabled = true, LogDebugCallback = s => TestContext.WriteLine(s), ErrorHandlerCallback = e => TestContext.WriteLine(e.Message + e.InnerException?.Message), MaxPublishingAttempts = 1, TimeSpanToLive = TimeSpan.FromMinutes(5) }); //***************************************************************************************** //* STEP 4 - Attempt to Retrieve/Receive the Message & Validate after Arrival! //***************************************************************************************** await AssertReceiptAndValidationOfThePublishedItem(outboxItem); }
public async Task TestNewOutboxItemInsertionIndividualItemsViaConvenienceMethods() { //Organize await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); //Clear the Table data for the test... await sqlConnection.TruncateTransactionalOutboxTableAsync(); //Initialize Transaction and Outbox Processor var outboxTestItems = TestHelper.CreateTestStringOutboxItemData(2); //Execute var timer = Stopwatch.StartNew(); //Add Item 1! var insertedResult1 = await sqlConnection .AddTransactionalOutboxPendingItemAsync( publishTarget : outboxTestItems[0].PublishingTarget, payload : outboxTestItems[0].PublishingPayload, fifoGroupingIdentifier : outboxTestItems[0].FifoGroupingIdentifier ) .ConfigureAwait(false); TestContext?.WriteLine($"Inserted First Item in [{timer.Elapsed.ToElapsedTimeDescriptiveFormat()}]."); Assert.IsNotNull(insertedResult1); Assert.IsNotNull(insertedResult1.UniqueIdentifier); Assert.IsNotNull(insertedResult1.FifoGroupingIdentifier); Assert.AreEqual(DateTime.UtcNow.Date, insertedResult1.CreatedDateTimeUtc.Date); //Add Item 2 (No FifoGroupingIdentifier)! var insertedResult2 = await sqlConnection .AddTransactionalOutboxPendingItemAsync( publishTarget : outboxTestItems[1].PublishingTarget, payload : outboxTestItems[1].PublishingPayload, fifoGroupingIdentifier : outboxTestItems[1].FifoGroupingIdentifier ) .ConfigureAwait(false); TestContext?.WriteLine($"Inserted First Item in [{timer.Elapsed.ToElapsedTimeDescriptiveFormat()}]."); Assert.IsNotNull(insertedResult2); Assert.IsNotNull(insertedResult2.UniqueIdentifier); Assert.IsNotNull(insertedResult2.FifoGroupingIdentifier); Assert.AreEqual(DateTime.UtcNow.Date, insertedResult2.CreatedDateTimeUtc.Date); timer.Stop(); }
public async Task TestNewOutboxItemInsertionAndRetrieval() { var testPublisher = new TestHarnessSqlTransactionalOutboxPublisher(); //Create Test Data var insertedResults = await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(100); //Initialize Transaction and Outbox Processor await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync().ConfigureAwait(false); await using var sqlTransaction = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false); var outboxProcessor = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction, testPublisher); //RETRIEVE ALL Pending Items from the Outbox to validate var pendingResults = await outboxProcessor.OutboxRepository .RetrieveOutboxItemsAsync(OutboxItemStatus.Pending) .ConfigureAwait(false); //Assert TestHelper.AssertOutboxItemResultsMatch(insertedResults, pendingResults); }
public async Task TestTransactionalOutboxEndToEndSuccessfulProcessing() { //Organize await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync().ConfigureAwait(false); //***************************************************************************************** //* STEP 1 - Prepare/Clear the Queue Table //***************************************************************************************** await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(100); //***************************************************************************************** //* STEP 3 - Executing processing of the Pending Items in the Queue... //***************************************************************************************** //Execute Processing of Items just inserted! //NOTE: We need to re-initialize a NEW Transaction and Processor to correctly simulate this running separately! var publishedItemList = new List <ISqlTransactionalOutboxItem <Guid> >(); var testPublisher = new TestHarnessSqlTransactionalOutboxPublisher((item, isFifoEnabled) => { publishedItemList.Add(item); TestContext.WriteLine($"Successfully Published Item: {item.UniqueIdentifier}"); return(Task.CompletedTask); } ); var publishedResults = await sqlConnection .ProcessPendingOutboxItemsAsync(testPublisher, new OutboxProcessingOptions()) .ConfigureAwait(false); //Assert results Assert.AreEqual(publishedItemList.Count, publishedResults.SuccessfullyPublishedItems.Count); Assert.AreEqual(0, publishedResults.FailedItems.Count); //Assert Unique Items all match var publishedItemLookup = publishedItemList.ToLookup(i => i.UniqueIdentifier); publishedResults.SuccessfullyPublishedItems.ForEach(r => { Assert.IsTrue(publishedItemLookup.Contains(r.UniqueIdentifier)); }); //***************************************************************************************** //* STEP 4 - Retrieve and Validate Data is updated and no pending Items Remain... //***************************************************************************************** //Assert All Items in the DB are Successful! await using var sqlTransaction3 = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false); var outboxProcessor = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction3, testPublisher); var successfulResultsFromDb = await outboxProcessor.OutboxRepository .RetrieveOutboxItemsAsync(OutboxItemStatus.Successful) .ConfigureAwait(false); //Assert the results from the DB match those returned from the Processing method... Assert.AreEqual(publishedResults.SuccessfullyPublishedItems.Count, successfulResultsFromDb.Count); successfulResultsFromDb.ForEach(dbItem => { Assert.AreEqual(OutboxItemStatus.Successful, dbItem.Status); Assert.AreEqual(1, dbItem.PublishAttempts); }); }
public async Task TestTransactionalOutboxFailureByTimeToLive() { var failedItemTestDataSizeByBatch = 3; var successfulItemTestDataSize = 3; var timeToLiveTimeSpan = TimeSpan.FromSeconds(5); var testHarnessPublisher = new TestHarnessSqlTransactionalOutboxPublisher(); var expectedTotalSuccessCount = 0; var expectedTotalExpiredCount = 0; //***************************************************************************************** //* STEP 1 - Prepare/Clear the Queue Table and populate initial Set of items (expected to Faile/Expire) // then wait for them to expire... //***************************************************************************************** await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(failedItemTestDataSizeByBatch); expectedTotalExpiredCount += failedItemTestDataSizeByBatch; await Task.Delay(timeToLiveTimeSpan + TimeSpan.FromSeconds(1)); //***************************************************************************************** //* STEP 2 - Add a Second & Third batch of items with different insertion Timestamps to // ensure only the first set expire as expected... //***************************************************************************************** await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(successfulItemTestDataSize, false); expectedTotalSuccessCount += successfulItemTestDataSize; //Insert in a second batch to force different Creation Dates at the DB level... await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(successfulItemTestDataSize, false); expectedTotalSuccessCount += successfulItemTestDataSize; //***************************************************************************************** //* STEP 2 - Process Outbox and get Results //***************************************************************************************** await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); var processingResults = await sqlConnection.ProcessPendingOutboxItemsAsync(testHarnessPublisher, new OutboxProcessingOptions() { TimeSpanToLive = timeToLiveTimeSpan }); //***************************************************************************************** //* STEP 3 - Validate Results returned! //***************************************************************************************** Assert.AreEqual(expectedTotalExpiredCount, processingResults.FailedItems.Count); Assert.AreEqual(expectedTotalSuccessCount, processingResults.SuccessfullyPublishedItems.Count); //We expect all items to be processed before any item is failed.... //So the First Item will be repeated as the 10'th item after the next 9 are also attempted... processingResults.SuccessfullyPublishedItems.ForEach(i => { Assert.AreEqual(OutboxItemStatus.Successful, i.Status); }); processingResults.FailedItems.ForEach(i => { Assert.AreEqual(OutboxItemStatus.FailedExpired, i.Status); }); //***************************************************************************************** //* STEP 3 - Validate Results In the DB! //***************************************************************************************** await using var sqlTransaction2 = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false); var outboxProcessor = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction2, testHarnessPublisher); var outboxRepository = outboxProcessor.OutboxRepository; var successfulItems = await outboxRepository.RetrieveOutboxItemsAsync(OutboxItemStatus.Successful); Assert.AreEqual(expectedTotalSuccessCount, successfulItems.Count); successfulItems.ForEach(i => { Assert.AreEqual(OutboxItemStatus.Successful, i.Status); }); var failedItems = await outboxRepository.RetrieveOutboxItemsAsync(OutboxItemStatus.FailedExpired); Assert.AreEqual(expectedTotalExpiredCount, failedItems.Count); processingResults.FailedItems.ForEach(i => { Assert.AreEqual(OutboxItemStatus.FailedExpired, i.Status); }); }
public async Task <List <ISqlTransactionalOutboxItem <Guid> > > DoTestTransactionalOutboxCrawlingOfBlockingFailureItems( int testDataSize, int maxPublishingAttempts, bool enforceFifoProcessing, bool throwExceptionOnFailedItem) { //***************************************************************************************** //* STEP 1 - Prepare/Clear the Queue Table //***************************************************************************************** await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(testDataSize); //***************************************************************************************** //* STEP 2 - Setup Custom Publisher & Processing Options... //***************************************************************************************** var publishedAttemptsList = new List <ISqlTransactionalOutboxItem <Guid> >(); var failingPublisher = new TestHarnessSqlTransactionalOutboxPublisher( (i, isFifoEnabled) => { publishedAttemptsList.Add(i); TestContext.WriteLine($"Successful -- We have intentionally Failed to Publish Item: {i.UniqueIdentifier}"); //Force an Error on Failure... this should result in ALL Publishing attempts to fail... throw new Exception("Failed to Publish!"); } ); var outboxProcessingOptions = new OutboxProcessingOptions() { MaxPublishingAttempts = maxPublishingAttempts, FifoEnforcedPublishingEnabled = enforceFifoProcessing }; //***************************************************************************************** //* STEP 3 - Executing processing of the Pending Items in the Queue... //***************************************************************************************** //Execute Processing of Items just inserted! //NOTE: We need to re-initialize a NEW Transaction and Processor to correctly simulate this running separately! ISqlTransactionalOutboxProcessingResults <Guid> publishedResults = null; int loopCounter = 0; bool handledExceptionSoItsOkToContinue = false; do { await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); await using var sqlTransaction = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false); handledExceptionSoItsOkToContinue = false; try { publishedResults = await sqlTransaction.ProcessPendingOutboxItemsAsync( failingPublisher, outboxProcessingOptions, throwExceptionOnFailedItem ).ConfigureAwait(false); } catch (Exception exc) { if (throwExceptionOnFailedItem) { //DO Nothing, as we Expect there to be exceptions when Throw Exception on Failure is Enabled! TestContext.WriteLine($"Successfully handled expected Exception: {exc.Message}"); publishedResults = new OutboxProcessingResults <Guid>(); handledExceptionSoItsOkToContinue = true; } else { //IF we get an exception but ThrowExceptionOnFailure is disabled, then this is an issue! throw; } } await sqlTransaction.CommitAsync(); //Provide Infinite Loop fail-safe... loopCounter++; Assert.IsTrue(loopCounter <= (testDataSize * maxPublishingAttempts * 2), $"Infinite Loop Breaker Tripped at [{loopCounter}]!"); //Assert there are never any successfully published items... Assert.AreEqual(0, publishedResults.SuccessfullyPublishedItems.Count); } while (publishedResults.FailedItems.Count > 0 || (throwExceptionOnFailedItem && handledExceptionSoItsOkToContinue)); //***************************************************************************************** //* STEP 4 - Retrieve and Validate Data in the Database is updated and all have failed out... //***************************************************************************************** //Assert All Items in the DB are Successful! await using var sqlConnection2 = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); await using var sqlTransaction2 = (SqlTransaction) await sqlConnection2.BeginTransactionAsync().ConfigureAwait(false); var outboxProcessor2 = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction2, failingPublisher); var successfulResultsFromDb = await outboxProcessor2.OutboxRepository .RetrieveOutboxItemsAsync(OutboxItemStatus.Pending) .ConfigureAwait(false); //Assert the results from the DB match those returned from the Processing method... Assert.AreEqual(successfulResultsFromDb.Count, 0); var failedResultsFromDb = await outboxProcessor2.OutboxRepository .RetrieveOutboxItemsAsync(OutboxItemStatus.FailedAttemptsExceeded) .ConfigureAwait(false); //Assert the results from the DB match those returned from the Processing method... Assert.AreEqual(failedResultsFromDb.Count * maxPublishingAttempts, publishedAttemptsList.Count); foreach (var dbItem in failedResultsFromDb) { Assert.AreEqual(OutboxItemStatus.FailedAttemptsExceeded, dbItem.Status); Assert.AreEqual(maxPublishingAttempts, dbItem.PublishAttempts); } //RETURN the Attempted Publishing list for additional validation based on the Pattern // we expect for items to be processed. return(publishedAttemptsList); }
protected static async Task CleanupOutboxAsync() { await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); await sqlConnection.TruncateTransactionalOutboxTableAsync(); }