async Task TryStageMessage(TransportOperation transportOperation, FailedMessageRetry failedMessageRetry)
{
try
{
await sender.Dispatch(new TransportOperations(transportOperation), transaction, contextBag).ConfigureAwait(false);
}
catch (Exception e)
{
var incrementedAttempts = failedMessageRetry.StageAttempts + 1;
var uniqueMessageId = transportOperation.Message.Headers["ServiceControl.Retry.UniqueMessageId"];
if (incrementedAttempts < MaxStagingAttempts)
{
Log.Warn($"Attempt {incrementedAttempts} of {MaxStagingAttempts} to stage a retry message {uniqueMessageId} failed", e);
await IncrementAttemptCounter(failedMessageRetry)
.ConfigureAwait(false);
}
else
{
Log.Error($"Retry message {uniqueMessageId} reached its staging retry limit ({MaxStagingAttempts}) and is going to be removed from the batch.", e);
await store.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null)
.ConfigureAwait(false);
await domainEvents.Raise(new MessageFailedInStaging
{
UniqueMessageId = uniqueMessageId
}).ConfigureAwait(false);
}
throw new RetryStagingException(e);
}
}
async Task TryStageMessage(FailedMessage message, string stagingId, FailedMessageRetry failedMessageRetry)
{
try
{
await StageMessage(message, stagingId).ConfigureAwait(false);
}
catch (Exception e)
{
var incrementedAttempts = failedMessageRetry.StageAttempts + 1;
if (incrementedAttempts < MaxStagingAttempts)
{
Log.Warn($"Attempt {incrementedAttempts} of {MaxStagingAttempts} to stage a retry message {message.UniqueMessageId} failed", e);
await IncrementAttemptCounter(failedMessageRetry)
.ConfigureAwait(false);
}
else
{
Log.Error($"Retry message {message.UniqueMessageId} reached its staging retry limit ({MaxStagingAttempts}) and is going to be removed from the batch.", e);
await store.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(message.UniqueMessageId), null)
.ConfigureAwait(false);
await domainEvents.Raise(new MessageFailedInStaging
{
UniqueMessageId = message.UniqueMessageId
}).ConfigureAwait(false);
}
throw new RetryStagingException();
}
}
public ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, string messageUniqueId)
{
var failureRetryId = FailedMessageRetry.MakeDocumentId(messageUniqueId);
return(new PatchCommandData
{
Patches = patchRequestsEmpty,
PatchesIfMissing = new[]
{
new PatchRequest
{
Name = "FailedMessageId",
Type = PatchCommandType.Set,
Value = FailedMessage.MakeDocumentId(messageUniqueId)
},
new PatchRequest
{
Name = "RetryBatchId",
Type = PatchCommandType.Set,
Value = batchDocumentId
}
},
Key = failureRetryId,
Metadata = defaultMetadata
});
}
public void ProcessingAlwaysFailsForMessage(TransportMessage message, Exception e)
{
try
{
var destination = message.Headers["ServiceControl.TargetEndpointAddress"];
var messageUniqueId = message.Headers["ServiceControl.Retry.UniqueMessageId"];
Log.Warn($"Failed to send '{messageUniqueId}' message to '{destination}' for retry. Attempting to revert message status to unresolved so it can be tried again.", e);
using (var session = store.OpenSession())
{
var failedMessage = session.Load <FailedMessage>(FailedMessage.MakeDocumentId(messageUniqueId));
if (failedMessage != null)
{
failedMessage.Status = FailedMessageStatus.Unresolved;
}
var failedMessageRetry = session.Load <FailedMessageRetry>(FailedMessageRetry.MakeDocumentId(messageUniqueId));
if (failedMessageRetry != null)
{
session.Delete(failedMessageRetry);
}
session.SaveChanges();
}
string reason;
try
{
reason = e.GetBaseException().Message;
}
catch (Exception)
{
reason = "Failed to retrieve reason!";
}
domainEvents.Raise(new MessagesSubmittedForRetryFailed
{
Reason = reason,
FailedMessageId = messageUniqueId,
Destination = destination
});
}
catch (Exception ex)
{
// If something goes wrong here we just ignore, not the end of the world!
Log.Error("A failure occurred when trying to handle a retry failure.", ex);
}
finally
{
executeOnFailure();
}
}
async Task IncrementAttemptCounter(FailedMessageRetry message)
{
try
{
await store.AsyncDatabaseCommands.PatchAsync(message.Id,
new[]
{
new PatchRequest
{
Type = PatchCommandType.Set,
Name = "StageAttempts",
Value = message.StageAttempts + 1
}
}).ConfigureAwait(false);
}
catch (ConcurrencyException)
{
Log.DebugFormat("Ignoring concurrency exception while incrementing staging attempt count for {0}", message.FailedMessageId);
}
}
public Task RemoveFailedMessageRetryDocument(string uniqueMessageId)
{
return(store.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null));
}
public async Task <ErrorHandleResult> OnError(IErrorHandlingPolicyContext handlingContext, IDispatchMessages dispatcher)
{
try
{
var message = handlingContext.Error.Message;
var destination = message.Headers["ServiceControl.TargetEndpointAddress"];
var messageUniqueId = message.Headers["ServiceControl.Retry.UniqueMessageId"];
Log.Warn($"Failed to send '{messageUniqueId}' message to '{destination}' for retry. Attempting to revert message status to unresolved so it can be tried again.", handlingContext.Error.Exception);
using (var session = store.OpenAsyncSession())
{
var failedMessage = await session.LoadAsync <FailedMessage>(FailedMessage.MakeDocumentId(messageUniqueId))
.ConfigureAwait(false);
if (failedMessage != null)
{
failedMessage.Status = FailedMessageStatus.Unresolved;
}
var failedMessageRetry = await session.LoadAsync <FailedMessageRetry>(FailedMessageRetry.MakeDocumentId(messageUniqueId))
.ConfigureAwait(false);
if (failedMessageRetry != null)
{
session.Delete(failedMessageRetry);
}
await session.SaveChangesAsync()
.ConfigureAwait(false);
}
string reason;
try
{
reason = handlingContext.Error.Exception.GetBaseException().Message;
}
catch (Exception)
{
reason = "Failed to retrieve reason!";
}
await domainEvents.Raise(new MessagesSubmittedForRetryFailed
{
Reason = reason,
FailedMessageId = messageUniqueId,
Destination = destination
}).ConfigureAwait(false);
}
catch (Exception ex)
{
// If something goes wrong here we just ignore, not the end of the world!
Log.Error("A failure occurred when trying to handle a retry failure.", ex);
}
finally
{
executeOnFailure();
}
return(ErrorHandleResult.Handled);
}
public void RemoveFailedMessageRetryDocument(string uniqueMessageId)
{
store.DatabaseCommands.Delete(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null);
}
