async Task TryStageMessage(TransportOperation transportOperation, FailedMessageRetry failedMessageRetry) { try { await sender.Dispatch(new TransportOperations(transportOperation), transaction, contextBag).ConfigureAwait(false); } catch (Exception e) { var incrementedAttempts = failedMessageRetry.StageAttempts + 1; var uniqueMessageId = transportOperation.Message.Headers["ServiceControl.Retry.UniqueMessageId"]; if (incrementedAttempts < MaxStagingAttempts) { Log.Warn($"Attempt {incrementedAttempts} of {MaxStagingAttempts} to stage a retry message {uniqueMessageId} failed", e); await IncrementAttemptCounter(failedMessageRetry) .ConfigureAwait(false); } else { Log.Error($"Retry message {uniqueMessageId} reached its staging retry limit ({MaxStagingAttempts}) and is going to be removed from the batch.", e); await store.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null) .ConfigureAwait(false); await domainEvents.Raise(new MessageFailedInStaging { UniqueMessageId = uniqueMessageId }).ConfigureAwait(false); } throw new RetryStagingException(e); } }
async Task TryStageMessage(FailedMessage message, string stagingId, FailedMessageRetry failedMessageRetry) { try { await StageMessage(message, stagingId).ConfigureAwait(false); } catch (Exception e) { var incrementedAttempts = failedMessageRetry.StageAttempts + 1; if (incrementedAttempts < MaxStagingAttempts) { Log.Warn($"Attempt {incrementedAttempts} of {MaxStagingAttempts} to stage a retry message {message.UniqueMessageId} failed", e); await IncrementAttemptCounter(failedMessageRetry) .ConfigureAwait(false); } else { Log.Error($"Retry message {message.UniqueMessageId} reached its staging retry limit ({MaxStagingAttempts}) and is going to be removed from the batch.", e); await store.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(message.UniqueMessageId), null) .ConfigureAwait(false); await domainEvents.Raise(new MessageFailedInStaging { UniqueMessageId = message.UniqueMessageId }).ConfigureAwait(false); } throw new RetryStagingException(); } }
public ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, string messageUniqueId) { var failureRetryId = FailedMessageRetry.MakeDocumentId(messageUniqueId); return(new PatchCommandData { Patches = patchRequestsEmpty, PatchesIfMissing = new[] { new PatchRequest { Name = "FailedMessageId", Type = PatchCommandType.Set, Value = FailedMessage.MakeDocumentId(messageUniqueId) }, new PatchRequest { Name = "RetryBatchId", Type = PatchCommandType.Set, Value = batchDocumentId } }, Key = failureRetryId, Metadata = defaultMetadata }); }
public void ProcessingAlwaysFailsForMessage(TransportMessage message, Exception e) { try { var destination = message.Headers["ServiceControl.TargetEndpointAddress"]; var messageUniqueId = message.Headers["ServiceControl.Retry.UniqueMessageId"]; Log.Warn($"Failed to send '{messageUniqueId}' message to '{destination}' for retry. Attempting to revert message status to unresolved so it can be tried again.", e); using (var session = store.OpenSession()) { var failedMessage = session.Load <FailedMessage>(FailedMessage.MakeDocumentId(messageUniqueId)); if (failedMessage != null) { failedMessage.Status = FailedMessageStatus.Unresolved; } var failedMessageRetry = session.Load <FailedMessageRetry>(FailedMessageRetry.MakeDocumentId(messageUniqueId)); if (failedMessageRetry != null) { session.Delete(failedMessageRetry); } session.SaveChanges(); } string reason; try { reason = e.GetBaseException().Message; } catch (Exception) { reason = "Failed to retrieve reason!"; } domainEvents.Raise(new MessagesSubmittedForRetryFailed { Reason = reason, FailedMessageId = messageUniqueId, Destination = destination }); } catch (Exception ex) { // If something goes wrong here we just ignore, not the end of the world! Log.Error("A failure occurred when trying to handle a retry failure.", ex); } finally { executeOnFailure(); } }
async Task IncrementAttemptCounter(FailedMessageRetry message) { try { await store.AsyncDatabaseCommands.PatchAsync(message.Id, new[] { new PatchRequest { Type = PatchCommandType.Set, Name = "StageAttempts", Value = message.StageAttempts + 1 } }).ConfigureAwait(false); } catch (ConcurrencyException) { Log.DebugFormat("Ignoring concurrency exception while incrementing staging attempt count for {0}", message.FailedMessageId); } }
public Task RemoveFailedMessageRetryDocument(string uniqueMessageId) { return(store.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null)); }
public async Task <ErrorHandleResult> OnError(IErrorHandlingPolicyContext handlingContext, IDispatchMessages dispatcher) { try { var message = handlingContext.Error.Message; var destination = message.Headers["ServiceControl.TargetEndpointAddress"]; var messageUniqueId = message.Headers["ServiceControl.Retry.UniqueMessageId"]; Log.Warn($"Failed to send '{messageUniqueId}' message to '{destination}' for retry. Attempting to revert message status to unresolved so it can be tried again.", handlingContext.Error.Exception); using (var session = store.OpenAsyncSession()) { var failedMessage = await session.LoadAsync <FailedMessage>(FailedMessage.MakeDocumentId(messageUniqueId)) .ConfigureAwait(false); if (failedMessage != null) { failedMessage.Status = FailedMessageStatus.Unresolved; } var failedMessageRetry = await session.LoadAsync <FailedMessageRetry>(FailedMessageRetry.MakeDocumentId(messageUniqueId)) .ConfigureAwait(false); if (failedMessageRetry != null) { session.Delete(failedMessageRetry); } await session.SaveChangesAsync() .ConfigureAwait(false); } string reason; try { reason = handlingContext.Error.Exception.GetBaseException().Message; } catch (Exception) { reason = "Failed to retrieve reason!"; } await domainEvents.Raise(new MessagesSubmittedForRetryFailed { Reason = reason, FailedMessageId = messageUniqueId, Destination = destination }).ConfigureAwait(false); } catch (Exception ex) { // If something goes wrong here we just ignore, not the end of the world! Log.Error("A failure occurred when trying to handle a retry failure.", ex); } finally { executeOnFailure(); } return(ErrorHandleResult.Handled); }
public void RemoveFailedMessageRetryDocument(string uniqueMessageId) { store.DatabaseCommands.Delete(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null); }