protected virtual async Task ProcessSingleOutboxItemInternal( ISqlTransactionalOutboxItem <TUniqueIdentifier> item, OutboxProcessingOptions options, OutboxProcessingResults <TUniqueIdentifier> results ) { //This process will publish pending items, while also cleaning up Pending Items that need to be Failed because // they couldn't be successfully published before and are exceeding the currently configured limits for // retry attempts and/or time-to-live. //NOTE: IF any of this fails due to issues with Sql Server it's ok because we have already transaction-ally // secured the data in the outbox therefore we can repeat processing over-and-over under the promise of // 'at-least-once' publishing attempt. options.LogDebugCallback?.Invoke($"Processing Item [{item.UniqueIdentifier}]..."); //Validate the Item hasn't exceeded the Max Retry Attempts if enabled in the options... if (options.MaxPublishingAttempts > 0 && item.PublishAttempts >= options.MaxPublishingAttempts) { options.LogDebugCallback?.Invoke( $"Item [{item.UniqueIdentifier}] has failed due to exceeding the max number of" + $" publishing attempts [{options.MaxPublishingAttempts}] with current PublishAttempts=[{item.PublishAttempts}]." ); item.Status = OutboxItemStatus.FailedAttemptsExceeded; results.FailedItems.Add(item); } //Validate the Item hasn't expired if enabled in the options... else if (options.TimeSpanToLive > TimeSpan.Zero && DateTimeOffset.UtcNow.Subtract(item.CreatedDateTimeUtc) >= options.TimeSpanToLive) { options.LogDebugCallback?.Invoke( $"Item [{item.UniqueIdentifier}] has failed due to exceeding the maximum time-to-live" + $" [{options.TimeSpanToLive.ToElapsedTimeDescriptiveFormat()}] because it was created at [{item.CreatedDateTimeUtc}] UTC." ); item.Status = OutboxItemStatus.FailedExpired; results.FailedItems.Add(item); } //Finally attempt to publish the item... else { item.PublishAttempts++; await OutboxPublisher.PublishOutboxItemAsync(item, options.FifoEnforcedPublishingEnabled).ConfigureAwait(false); options.LogDebugCallback?.Invoke( $"Item [{item.UniqueIdentifier}] published successfully after [{item.PublishAttempts}] publishing attempt(s)!" ); //Update the Status only AFTER successful Publishing! item.Status = OutboxItemStatus.Successful; results.SuccessfullyPublishedItems.Add(item); options.LogDebugCallback?.Invoke( $"Item [{item.UniqueIdentifier}] outbox status will be updated to [{item.Status}]." ); } }
protected virtual async Task ProcessOutboxItemsInternalAsync( List <ISqlTransactionalOutboxItem <TUniqueIdentifier> > outboxItems, OutboxProcessingOptions options, OutboxProcessingResults <TUniqueIdentifier> results, bool throwExceptionOnFailure ) { var skipFifoGroups = new HashSet <string>(); foreach (var item in outboxItems) { //Process the item when Fifo Publishing is Disabled, or the Item has no Fifo Group specified, or the // specified Fifo Group Id is not already identified as one that needs to be skipped due to an item error // that belongs to that group! if (!options.FifoEnforcedPublishingEnabled || // Fifo is Disabled string.IsNullOrWhiteSpace(item.FifoGroupingIdentifier) || // No Fifo Group is defined !skipFifoGroups.Contains(item.FifoGroupingIdentifier) // Fifo group defined is not in the set to be skipped ) { try { await ProcessSingleOutboxItemInternal(item, options, results); } catch (Exception itemException) { await HandleExceptionForOutboxItemFromQueueInternal( item, itemException, options, results, throwExceptionOnFailure, skipFifoGroups ); } } } results.ProcessingTimer.Stop(); options.LogDebugCallback?.Invoke( $"Finished Publishing [{results.SuccessfullyPublishedItems.Count}] items in" + $" [{results.ProcessingTimer.Elapsed.ToElapsedTimeDescriptiveFormat()}]." ); //Store all updated results back into the Outbox! await UpdateProcessedItemsInternal(results, options); }
protected virtual async Task UpdateProcessedItemsInternal( OutboxProcessingResults <TUniqueIdentifier> results, OutboxProcessingOptions options ) { //Update & store the state for all Items Processed (e.g. Successful, Attempted, Failed, etc.)! var processedItems = results.GetAllProcessedItems(); options.LogDebugCallback?.Invoke($"Starting to update the Outbox for [{processedItems.Count}] items..."); if (processedItems.Count > 0) { results.ProcessingTimer.Start(); await OutboxRepository.UpdateOutboxItemsAsync(processedItems, options.ItemUpdatingBatchSize).ConfigureAwait(false); results.ProcessingTimer.Stop(); } options.LogDebugCallback?.Invoke( $"Finished updating the Outbox for [{processedItems.Count}] items in" + $" [{results.ProcessingTimer.Elapsed.ToElapsedTimeDescriptiveFormat()}]!" ); }
public virtual async Task <ISqlTransactionalOutboxProcessingResults <TUniqueIdentifier> > ProcessPendingOutboxItemsAsync( OutboxProcessingOptions processingOptions = null, bool throwExceptionOnFailure = false ) { var options = processingOptions ?? OutboxProcessingOptions.DefaultOutboxProcessingOptions; var results = new OutboxProcessingResults <TUniqueIdentifier>(); results.ProcessingTimer.Start(); //Retrieve items to e processed from the Repository (ALL Pending items available for publishing attempt!) var pendingOutboxItems = await OutboxRepository.RetrieveOutboxItemsAsync( OutboxItemStatus.Pending, options.ItemProcessingBatchSize ).ConfigureAwait(false); results.ProcessingTimer.Stop(); //Short Circuit if there is nothing to process! if (pendingOutboxItems.Count <= 0) { options.LogDebugCallback?.Invoke($"There are no outbox items to process; processing completed at [{DateTime.Now}]."); return(results); } options.LogDebugCallback?.Invoke( $"Starting Outbox Processing of [{pendingOutboxItems.Count}] outbox items, retrieved in" + $" [{results.ProcessingTimer.Elapsed.ToElapsedTimeDescriptiveFormat()}], at [{DateTime.Now}]..." ); //Attempt the acquire the Distributed Mutex Lock (if specified)! options.LogDebugCallback?.Invoke($"{nameof(options.FifoEnforcedPublishingEnabled)} = {options.FifoEnforcedPublishingEnabled}"); results.ProcessingTimer.Start(); await using var distributedMutex = options.FifoEnforcedPublishingEnabled ? await OutboxRepository.AcquireDistributedProcessingMutexAsync().ConfigureAwait(false) : new NoOpAsyncDisposable(); //The distributed Mutex will ONLY be null if it could not be acquired; otherwise our // NoOp will be successfully initialized if locking is disabled. if (distributedMutex == null) { const string mutexErrorMessage = "Distributed Mutex Lock could not be acquired; processing will not continue."; options.LogDebugCallback?.Invoke(mutexErrorMessage); if (throwExceptionOnFailure) { throw new Exception(mutexErrorMessage); } return(results); } //Now EXECUTE & PROCESS the Items and Update the Outbox appropriately... //NOTE: It's CRITICAL that we attempt to Publish BEFORE we update the results in the TransactionalOutbox, // this ensures that we guarantee at-least-once delivery because the item will be retried at a later point // if anything fails with the update. await ProcessOutboxItemsInternalAsync( pendingOutboxItems, options, results, throwExceptionOnFailure ).ConfigureAwait(false); return(results); }
protected async Task HandleExceptionForOutboxItemFromQueueInternal( ISqlTransactionalOutboxItem <TUniqueIdentifier> item, Exception itemException, OutboxProcessingOptions options, OutboxProcessingResults <TUniqueIdentifier> results, bool throwExceptionOnFailure, HashSet <string> skipFifoGroups ) { var errorMessage = $"An Unexpected Exception occurred while processing outbox item [{item.UniqueIdentifier}]."; var fifoErrorMessage = $"FIFO Processing is enabled, but the outbox item [{item.UniqueIdentifier}] could not be published;" + $" all associated items for the Fifo Group [{item.FifoGroupingIdentifier}] will be skipped."; if (options.FifoEnforcedPublishingEnabled) { errorMessage = string.Concat(errorMessage, fifoErrorMessage); } var processingException = new Exception(errorMessage, itemException); //Add Failed Item to the results results.FailedItems.Add(item); //Short circuit if we are configured to Throw the Error or if Enforce FIFO processing is enabled! if (throwExceptionOnFailure) { options.ErrorHandlerCallback?.Invoke(processingException); //If configured to throw an error then we Attempt to update the item before throwing exception // because normally it would have been updated in bulk if exceptions were suppressed. //NOTE: NO need to process results since we are throwing an Exception... await UpdateProcessedItemsInternal(results, options); throw processingException; } else if (options.FifoEnforcedPublishingEnabled) { //If Enforce Fifo processing is enabled then we also need to halt processing and drain the current queue to preserve // the current order of processing; if we allow it to proceed we might publish the next item out of order. //NOTE: This may create a block in the Queue if the issue isn't fixed but it's necessary to preserve // the publishing order until the erroneous blocking item is resolved automatically (via connections restored, // or item fails due to re-attempts), or manually (item is failed and/or removed manually). options.LogDebugCallback?.Invoke( $"FIFO Processing is enabled, but the outbox item [{item.UniqueIdentifier}] could not be published;" + $" all following items for the FIFO Group [{item.FifoGroupingIdentifier}] will be skipped." ); skipFifoGroups.Add(item.FifoGroupingIdentifier); //ORIGINAL Logic before Fifo Grouping Identifiers were implemented... //processingException = new Exception( // $"The processing must be stopped because [{nameof(options.FifoEnforcedPublishingEnabled)}]" // + $" configuration option is enabled; therefore to preserve the publishing order the remaining" // + $" [{processingQueue.Count}] items will be delayed until the current item [{item.UniqueIdentifier}]" // + " can be processed successfully or is failed out of the outbox queue.", // processingException //); ////Drain the Queue to halt current process... //while (processingQueue.Count > 0) //{ // results.SkippedItems.Add(processingQueue.Dequeue()); //} } //Log the latest initialized exception with details... options.ErrorHandlerCallback?.Invoke(processingException); }