public async Task TestOutboxHistoryCleanupTests() { //Clear the Table data for the test... int testDataSize = 100; await SystemDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(testDataSize, true); var sqlConnection = await SqlConnectionHelper.CreateSystemDataSqlConnectionAsync(); var insertedCount = await RetrievePendingItemsCountAsync(sqlConnection); Assert.AreEqual(testDataSize, insertedCount); await Task.Delay(TimeSpan.FromSeconds(5)); int nonPurgedDataSize = 10; await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(nonPurgedDataSize, false); //Execute await sqlConnection .CleanupHistoricalOutboxItemsAsync(TimeSpan.FromSeconds(3)) .ConfigureAwait(false); //Assert var purgedCount = await RetrievePendingItemsCountAsync(sqlConnection); Assert.AreEqual(nonPurgedDataSize, purgedCount); }
public async Task TestNewOutboxItemInsertionAndRetrieval() { var testPublisher = new TestHarnessSqlTransactionalOutboxPublisher(); //Create Test Data var insertedResults = await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(100); //Initialize Transaction and Outbox Processor await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync().ConfigureAwait(false); await using var sqlTransaction = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false); var outboxProcessor = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction, testPublisher); //RETRIEVE ALL Pending Items from the Outbox to validate var pendingResults = await outboxProcessor.OutboxRepository .RetrieveOutboxItemsAsync(OutboxItemStatus.Pending) .ConfigureAwait(false); //Assert TestHelper.AssertOutboxItemResultsMatch(insertedResults, pendingResults); }
public async Task TestTransactionalOutboxFailureByTimeToLive() { var failedItemTestDataSizeByBatch = 3; var successfulItemTestDataSize = 3; var timeToLiveTimeSpan = TimeSpan.FromSeconds(5); var testHarnessPublisher = new TestHarnessSqlTransactionalOutboxPublisher(); var expectedTotalSuccessCount = 0; var expectedTotalExpiredCount = 0; //***************************************************************************************** //* STEP 1 - Prepare/Clear the Queue Table and populate initial Set of items (expected to Faile/Expire) // then wait for them to expire... //***************************************************************************************** await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(failedItemTestDataSizeByBatch); expectedTotalExpiredCount += failedItemTestDataSizeByBatch; await Task.Delay(timeToLiveTimeSpan + TimeSpan.FromSeconds(1)); //***************************************************************************************** //* STEP 2 - Add a Second & Third batch of items with different insertion Timestamps to // ensure only the first set expire as expected... //***************************************************************************************** await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(successfulItemTestDataSize, false); expectedTotalSuccessCount += successfulItemTestDataSize; //Insert in a second batch to force different Creation Dates at the DB level... await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(successfulItemTestDataSize, false); expectedTotalSuccessCount += successfulItemTestDataSize; //***************************************************************************************** //* STEP 2 - Process Outbox and get Results //***************************************************************************************** await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); var processingResults = await sqlConnection.ProcessPendingOutboxItemsAsync(testHarnessPublisher, new OutboxProcessingOptions() { TimeSpanToLive = timeToLiveTimeSpan }); //***************************************************************************************** //* STEP 3 - Validate Results returned! //***************************************************************************************** Assert.AreEqual(expectedTotalExpiredCount, processingResults.FailedItems.Count); Assert.AreEqual(expectedTotalSuccessCount, processingResults.SuccessfullyPublishedItems.Count); //We expect all items to be processed before any item is failed.... //So the First Item will be repeated as the 10'th item after the next 9 are also attempted... processingResults.SuccessfullyPublishedItems.ForEach(i => { Assert.AreEqual(OutboxItemStatus.Successful, i.Status); }); processingResults.FailedItems.ForEach(i => { Assert.AreEqual(OutboxItemStatus.FailedExpired, i.Status); }); //***************************************************************************************** //* STEP 3 - Validate Results In the DB! //***************************************************************************************** await using var sqlTransaction2 = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false); var outboxProcessor = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction2, testHarnessPublisher); var outboxRepository = outboxProcessor.OutboxRepository; var successfulItems = await outboxRepository.RetrieveOutboxItemsAsync(OutboxItemStatus.Successful); Assert.AreEqual(expectedTotalSuccessCount, successfulItems.Count); successfulItems.ForEach(i => { Assert.AreEqual(OutboxItemStatus.Successful, i.Status); }); var failedItems = await outboxRepository.RetrieveOutboxItemsAsync(OutboxItemStatus.FailedExpired); Assert.AreEqual(expectedTotalExpiredCount, failedItems.Count); processingResults.FailedItems.ForEach(i => { Assert.AreEqual(OutboxItemStatus.FailedExpired, i.Status); }); }
public async Task TestTransactionalOutboxEndToEndSuccessfulProcessing() { //Organize await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync().ConfigureAwait(false); //***************************************************************************************** //* STEP 1 - Prepare/Clear the Queue Table //***************************************************************************************** await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(100); //***************************************************************************************** //* STEP 3 - Executing processing of the Pending Items in the Queue... //***************************************************************************************** //Execute Processing of Items just inserted! //NOTE: We need to re-initialize a NEW Transaction and Processor to correctly simulate this running separately! var publishedItemList = new List <ISqlTransactionalOutboxItem <Guid> >(); var testPublisher = new TestHarnessSqlTransactionalOutboxPublisher((item, isFifoEnabled) => { publishedItemList.Add(item); TestContext.WriteLine($"Successfully Published Item: {item.UniqueIdentifier}"); return(Task.CompletedTask); } ); var publishedResults = await sqlConnection .ProcessPendingOutboxItemsAsync(testPublisher, new OutboxProcessingOptions()) .ConfigureAwait(false); //Assert results Assert.AreEqual(publishedItemList.Count, publishedResults.SuccessfullyPublishedItems.Count); Assert.AreEqual(0, publishedResults.FailedItems.Count); //Assert Unique Items all match var publishedItemLookup = publishedItemList.ToLookup(i => i.UniqueIdentifier); publishedResults.SuccessfullyPublishedItems.ForEach(r => { Assert.IsTrue(publishedItemLookup.Contains(r.UniqueIdentifier)); }); //***************************************************************************************** //* STEP 4 - Retrieve and Validate Data is updated and no pending Items Remain... //***************************************************************************************** //Assert All Items in the DB are Successful! await using var sqlTransaction3 = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false); var outboxProcessor = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction3, testPublisher); var successfulResultsFromDb = await outboxProcessor.OutboxRepository .RetrieveOutboxItemsAsync(OutboxItemStatus.Successful) .ConfigureAwait(false); //Assert the results from the DB match those returned from the Processing method... Assert.AreEqual(publishedResults.SuccessfullyPublishedItems.Count, successfulResultsFromDb.Count); successfulResultsFromDb.ForEach(dbItem => { Assert.AreEqual(OutboxItemStatus.Successful, dbItem.Status); Assert.AreEqual(1, dbItem.PublishAttempts); }); }
public async Task <List <ISqlTransactionalOutboxItem <Guid> > > DoTestTransactionalOutboxCrawlingOfBlockingFailureItems( int testDataSize, int maxPublishingAttempts, bool enforceFifoProcessing, bool throwExceptionOnFailedItem) { //***************************************************************************************** //* STEP 1 - Prepare/Clear the Queue Table //***************************************************************************************** await MicrosoftDataSqlTestHelpers.PopulateTransactionalOutboxTestDataAsync(testDataSize); //***************************************************************************************** //* STEP 2 - Setup Custom Publisher & Processing Options... //***************************************************************************************** var publishedAttemptsList = new List <ISqlTransactionalOutboxItem <Guid> >(); var failingPublisher = new TestHarnessSqlTransactionalOutboxPublisher( (i, isFifoEnabled) => { publishedAttemptsList.Add(i); TestContext.WriteLine($"Successful -- We have intentionally Failed to Publish Item: {i.UniqueIdentifier}"); //Force an Error on Failure... this should result in ALL Publishing attempts to fail... throw new Exception("Failed to Publish!"); } ); var outboxProcessingOptions = new OutboxProcessingOptions() { MaxPublishingAttempts = maxPublishingAttempts, FifoEnforcedPublishingEnabled = enforceFifoProcessing }; //***************************************************************************************** //* STEP 3 - Executing processing of the Pending Items in the Queue... //***************************************************************************************** //Execute Processing of Items just inserted! //NOTE: We need to re-initialize a NEW Transaction and Processor to correctly simulate this running separately! ISqlTransactionalOutboxProcessingResults <Guid> publishedResults = null; int loopCounter = 0; bool handledExceptionSoItsOkToContinue = false; do { await using var sqlConnection = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); await using var sqlTransaction = (SqlTransaction) await sqlConnection.BeginTransactionAsync().ConfigureAwait(false); handledExceptionSoItsOkToContinue = false; try { publishedResults = await sqlTransaction.ProcessPendingOutboxItemsAsync( failingPublisher, outboxProcessingOptions, throwExceptionOnFailedItem ).ConfigureAwait(false); } catch (Exception exc) { if (throwExceptionOnFailedItem) { //DO Nothing, as we Expect there to be exceptions when Throw Exception on Failure is Enabled! TestContext.WriteLine($"Successfully handled expected Exception: {exc.Message}"); publishedResults = new OutboxProcessingResults <Guid>(); handledExceptionSoItsOkToContinue = true; } else { //IF we get an exception but ThrowExceptionOnFailure is disabled, then this is an issue! throw; } } await sqlTransaction.CommitAsync(); //Provide Infinite Loop fail-safe... loopCounter++; Assert.IsTrue(loopCounter <= (testDataSize * maxPublishingAttempts * 2), $"Infinite Loop Breaker Tripped at [{loopCounter}]!"); //Assert there are never any successfully published items... Assert.AreEqual(0, publishedResults.SuccessfullyPublishedItems.Count); } while (publishedResults.FailedItems.Count > 0 || (throwExceptionOnFailedItem && handledExceptionSoItsOkToContinue)); //***************************************************************************************** //* STEP 4 - Retrieve and Validate Data in the Database is updated and all have failed out... //***************************************************************************************** //Assert All Items in the DB are Successful! await using var sqlConnection2 = await SqlConnectionHelper.CreateMicrosoftDataSqlConnectionAsync(); await using var sqlTransaction2 = (SqlTransaction) await sqlConnection2.BeginTransactionAsync().ConfigureAwait(false); var outboxProcessor2 = new DefaultSqlServerTransactionalOutboxProcessor <string>(sqlTransaction2, failingPublisher); var successfulResultsFromDb = await outboxProcessor2.OutboxRepository .RetrieveOutboxItemsAsync(OutboxItemStatus.Pending) .ConfigureAwait(false); //Assert the results from the DB match those returned from the Processing method... Assert.AreEqual(successfulResultsFromDb.Count, 0); var failedResultsFromDb = await outboxProcessor2.OutboxRepository .RetrieveOutboxItemsAsync(OutboxItemStatus.FailedAttemptsExceeded) .ConfigureAwait(false); //Assert the results from the DB match those returned from the Processing method... Assert.AreEqual(failedResultsFromDb.Count * maxPublishingAttempts, publishedAttemptsList.Count); foreach (var dbItem in failedResultsFromDb) { Assert.AreEqual(OutboxItemStatus.FailedAttemptsExceeded, dbItem.Status); Assert.AreEqual(maxPublishingAttempts, dbItem.PublishAttempts); } //RETURN the Attempted Publishing list for additional validation based on the Pattern // we expect for items to be processed. return(publishedAttemptsList); }