static bool HasReachedMaxTime(TransportMessage message)
{
var timestampHeader = TransportMessageHeaderHelper.GetHeader(message, SecondLevelRetriesHeaders.RetriesTimestamp);
if (String.IsNullOrEmpty(timestampHeader))
{
return(false);
}
try
{
var handledAt = DateTimeExtensions.ToUtcDateTime(timestampHeader);
if (DateTime.UtcNow > handledAt.AddDays(1))
{
return(true);
}
}
// ReSharper disable once EmptyGeneralCatchClause
// this code won't usually throw but in case a user has decided to hack a message/headers and for some bizarre reason
// they changed the date and that parse fails, we want to make sure that doesn't prevent the message from being
// forwarded to the error queue.
catch (Exception)
{
}
return(false);
}
void Defer(TimeSpan defer, TransportMessage message)
{
var retryMessageAt = DateTime.UtcNow + defer;
TransportMessageHeaderHelper.SetHeader(message, Headers.Retries,
(TransportMessageHeaderHelper.GetNumberOfRetries(message) + 1).ToString(CultureInfo.InvariantCulture));
var addressOfFaultingEndpoint = TransportMessageHeaderHelper.GetAddressOfFaultingEndpoint(message);
if (!TransportMessageHeaderHelper.HeaderExists(message, SecondLevelRetriesHeaders.RetriesTimestamp))
{
TransportMessageHeaderHelper.SetHeader(message, SecondLevelRetriesHeaders.RetriesTimestamp,
DateTimeExtensions.ToWireFormattedString(DateTime.UtcNow));
}
logger.DebugFormat("Defer message and send it to {0}", addressOfFaultingEndpoint);
var sendOptions = new SendOptions(addressOfFaultingEndpoint)
{
DeliverAt = retryMessageAt
};
MessageDeferrer.Defer(message, sendOptions);
}
public void A_message_should_only_be_able_to_retry_during_N_minutes()
{
TransportMessageHeaderHelper.SetHeader(message, FaultsHeaderKeys.FailedQ, "reply@address");
TransportMessageHeaderHelper.SetHeader(message, SecondLevelRetriesHeaders.RetriesTimestamp, DateTimeExtensions.ToWireFormattedString(DateTime.Now.AddDays(-2)));
satellite.Handle(message);
Assert.AreEqual(ERROR_QUEUE, messageSender.MessageSentTo);
}
public void Message_should_be_sent_to_real_errorQ_if_defer_timeSpan_is_less_than_zero()
{
TransportMessageHeaderHelper.SetHeader(message, FaultsHeaderKeys.FailedQ, "reply@address");
satellite.SecondLevelRetriesConfiguration.RetryPolicy = _ => TimeSpan.MinValue;
satellite.Handle(message);
Assert.AreEqual(ERROR_QUEUE, messageSender.MessageSentTo);
}
public void Message_should_be_routed_to_the_failing_endpoint_when_the_time_is_up()
{
TransportMessageHeaderHelper.SetHeader(message, FaultsHeaderKeys.FailedQ, ORIGINAL_QUEUE.ToString());
satellite.SecondLevelRetriesConfiguration.RetryPolicy = _ => TimeSpan.FromSeconds(1);
satellite.Handle(message);
Assert.AreEqual(ORIGINAL_QUEUE, deferrer.MessageRoutedTo);
}
public void The_default_time_out_should_be_1_day()
{
TransportMessageHeaderHelper.SetHeader(_message, SecondLevelRetriesHeaders.RetriesTimestamp, DateTimeExtensions.ToWireFormattedString(DateTime.UtcNow.AddDays(-1).AddSeconds(-1)));
var retriesProcessor = new SecondLevelRetriesProcessor();
var hasTimedOut = retriesProcessor.Validate(_message) == TimeSpan.MinValue;
Assert.IsTrue(hasTimedOut);
}
public void Message_should_be_sent_to_retryQ_if_defer_timeSpan_is_greater_than_zero()
{
TransportMessageHeaderHelper.SetHeader(message, FaultsHeaderKeys.FailedQ, "reply@address");
satellite.SecondLevelRetriesConfiguration.RetryPolicy = _ => TimeSpan.FromSeconds(1);
satellite.Handle(message);
Assert.AreEqual(message, deferrer.DeferredMessage);
}
void SendToRetriesQueue(TransportMessage message, Exception e, string flrPart)
{
message.TimeToBeReceived = TimeSpan.MaxValue;
sender.Send(message, new SendOptions(RetriesQueue));
var retryAttempt = TransportMessageHeaderHelper.GetNumberOfRetries(message) + 1;
Logger.WarnFormat("{0} will be handed over to SLR for retry attempt {1}.", flrPart, retryAttempt);
busNotifications.Errors.InvokeMessageHasBeenSentToSecondLevelRetries(retryAttempt, message, e);
}
public void Message_should_only_be_retried_X_times_when_using_the_defaultPolicy()
{
TransportMessageHeaderHelper.SetHeader(message, FaultsHeaderKeys.FailedQ, "reply@address");
for (var i = 0; i < satellite.SecondLevelRetriesConfiguration.NumberOfRetries + 1; i++)
{
satellite.Handle(message);
}
Assert.AreEqual(ERROR_QUEUE, messageSender.MessageSentTo);
}
bool MessageWasSentFromSLR(TransportMessage message)
{
if (RetriesQueue == null)
{
return(false);
}
// if the reply to address == ErrorQueue and RealErrorQueue is not null, the
// SecondLevelRetries sat is running and the error happened within that sat.
return(TransportMessageHeaderHelper.GetAddressOfFaultingEndpoint(message) == RetriesQueue);
}
public void Message_should_have_ReplyToAddress_set_to_original_sender_when_sent_to_real_error_queue_after_retries()
{
TransportMessageHeaderHelper.SetHeader(message, FaultsHeaderKeys.FailedQ, "reply@address");
for (var i = 0; i < satellite.SecondLevelRetriesConfiguration.NumberOfRetries + 1; i++)
{
satellite.Handle(message);
}
Assert.AreEqual(CLIENT_QUEUE, message.ReplyToAddress);
}
public void Message_retries_header_should_be_removed_before_being_sent_to_real_errorQ()
{
TransportMessageHeaderHelper.SetHeader(message, FaultsHeaderKeys.FailedQ, "reply@address");
satellite.Handle(message);
TransportMessageHeaderHelper.SetHeader(message, SecondLevelRetriesHeaders.RetriesTimestamp, DateTimeExtensions.ToWireFormattedString(DateTime.Now.AddDays(-2)));
satellite.Handle(message);
Assert.False(message.Headers.ContainsKey(Headers.Retries));
}
public void For_each_retry_the_NServiceBus_Retries_header_should_be_increased()
{
TransportMessageHeaderHelper.SetHeader(message, FaultsHeaderKeys.FailedQ, "reply@address");
satellite.SecondLevelRetriesConfiguration.RetryPolicy = _ => TimeSpan.FromSeconds(1);
for (var i = 0; i < 10; i++)
{
satellite.Handle(message);
}
Assert.AreEqual(10, TransportMessageHeaderHelper.GetNumberOfRetries(message));
}
void SendFailureMessage(TransportMessage message, Exception e, bool serializationException = false)
{
message.SetExceptionHeaders(e, localAddress ?? config.LocalAddress);
try
{
var destinationQ = RetriesErrorQueue ?? ErrorQueue;
// Intentionally service-locate ISendMessages to avoid circular
// resolution problem in the container
var sender = builder.Build <ISendMessages>();
if (serializationException || MessageWasSentFromSLR(message))
{
sender.Send(message, new SendOptions(ErrorQueue));
return;
}
sender.Send(message, new SendOptions(destinationQ));
//HACK: We need this hack here till we refactor the SLR to be a first class concept in the TransportReceiver
if (RetriesErrorQueue == null)
{
Logger.ErrorFormat("Message with '{0}' id has failed FLR and will be moved to the configured error queue.", message.Id);
}
else
{
var retryAttempt = TransportMessageHeaderHelper.GetNumberOfRetries(message) + 1;
Logger.WarnFormat("Message with '{0}' id has failed FLR and will be handed over to SLR for retry attempt {1}.", message.Id, retryAttempt);
}
}
catch (Exception exception)
{
var queueNotFoundException = exception as QueueNotFoundException;
string errorMessage;
if (queueNotFoundException != null)
{
errorMessage = string.Format("Could not forward failed message to error queue '{0}' as it could not be found.", queueNotFoundException.Queue);
Logger.Fatal(errorMessage);
}
else
{
errorMessage = "Could not forward failed message to error queue.";
Logger.Fatal(errorMessage, exception);
}
throw new InvalidOperationException(errorMessage, exception);
}
}
TimeSpan DefaultRetryPolicy(TransportMessage message)
{
if (HasReachedMaxTime(message))
{
return(TimeSpan.MinValue);
}
var numberOfRetries = TransportMessageHeaderHelper.GetNumberOfRetries(message);
var timeToIncreaseInTicks = TimeIncrease.Ticks * (numberOfRetries + 1);
var timeIncrease = TimeSpan.FromTicks(timeToIncreaseInTicks);
return(numberOfRetries >= NumberOfRetries ? TimeSpan.MinValue : timeIncrease);
}
void HandleProcessingAlwaysFailsForMessage(TransportMessage message, Exception e, int numberOfRetries)
{
message.SetExceptionHeaders(e, localAddress ?? config.LocalAddress);
if (MessageWasSentFromSLR(message))
{
sender.Send(message, new SendOptions(ErrorQueue));
busNotifications.Errors.InvokeMessageHasBeenSentToErrorQueue(message, e);
return;
}
var flrPart = numberOfRetries > 0
? string.Format("Message with '{0}' id has failed FLR and", message.Id)
: string.Format("FLR is disabled and the message '{0}'", message.Id);
//HACK: We need this hack here till we refactor the SLR to be a first class concept in the TransportReceiver
if (RetriesErrorQueue == null)
{
sender.Send(message, new SendOptions(ErrorQueue));
Logger.ErrorFormat("{0} will be moved to the configured error queue.", flrPart);
busNotifications.Errors.InvokeMessageHasBeenSentToErrorQueue(message, e);
return;
}
var defer = SecondLevelRetriesConfiguration.RetryPolicy.Invoke(message);
if (defer < TimeSpan.Zero)
{
SendToErrorQueue(message, e);
return;
}
sender.Send(message, new SendOptions(RetriesErrorQueue));
var retryAttempt = TransportMessageHeaderHelper.GetNumberOfRetries(message) + 1;
Logger.WarnFormat("{0} will be handed over to SLR for retry attempt {1}.", flrPart, retryAttempt);
busNotifications.Errors.InvokeMessageHasBeenSentToSecondLevelRetries(retryAttempt, message, e);
}
private void Defer()
{
TransportMessageHeaderHelper.SetHeader(_message, Headers.Retries, (TransportMessageHeaderHelper.GetNumberOfRetries(_message) + 1).ToString());
}
