void OnPoisonMessage(IBus bus, ReceivedTransportMessage message, PoisonMessageInfo poisonmessageinfo)
{
if (message.Headers.ContainsKey("command.id"))
{
Guid commandId = Guid.Parse(message.Headers["command.id"].ToString());
var description = message.Headers["command.description"].ToString();
var ex = poisonmessageinfo.Exceptions.Last();
string exMessage = description;
Exception exception = ex.Value;
while (exception is TargetInvocationException)
{
exception = exception.InnerException;
}
if (exception != null)
{
exMessage = exception.Message;
}
this.MessagesTracker.Failed(commandId, ex.Time, exception);
var command = GetCommandFromMessage(message);
if (command != null)
{
var notifyTo = command.GetContextData(MessagesConstants.ReplyToHeader);
if (!string.IsNullOrEmpty(notifyTo))
{
var commandHandled = new CommandHandled(
notifyTo,
commandId,
CommandHandled.CommandResult.Failed,
description,
exMessage
);
commandHandled.CopyHeaders(command);
bus.Advanced.Routing.Send(
message.Headers["rebus-return-address"].ToString(),
commandHandled);
}
}
}
}
/// <summary>
/// Inner function to handle the different type of exceptions
/// </summary>
/// <param name="ex"></param>
/// <param name="command"></param>
/// <param name="retryCount"></param>
/// <param name="retry"></param>
/// <param name="replyCommand"></param>
/// <param name="shouldContinue">true if the handler can continue execution, false if the execution of the command should stop
/// because there is no need to continue.</param>
/// <returns>Return always true because it is the very same routine used for aggregate exception. We always want not to throw in this
/// method, but allow the caller to throw.</returns>
private Boolean InnerHandle(Exception ex, ICommand command, Int32 retryCount, out Boolean retry, out CommandHandled replyCommand, out Boolean shouldContinue)
{
retry = false;
replyCommand = null;
shouldContinue = false;
switch (ex)
{
case ConcurrencyException cex:
SharedMetricsHelper.MarkConcurrencyException();
// retry
if (_logger.IsInfoEnabled)
{
_logger.InfoFormat(ex, "Handled {0} {1} [{2}], concurrency exception. Retry count: {3}", command.GetType().FullName, command.MessageId, command.Describe(), retryCount);
}
// increment the retries counter and maybe add a delay
if (retryCount++ > _numberOfConcurrencyExceptionBeforeRandomSleeping)
{
Thread.Sleep(new Random(DateTime.Now.Millisecond).Next(retryCount * 10));
}
retry = retryCount < _maxRetryOnConcurrencyException; //can retry
shouldContinue = true; //can proceed
break;
case DomainException dex:
SharedMetricsHelper.MarkDomainException(command, dex);
var notifyTo = command.GetContextData(MessagesConstants.ReplyToHeader);
if (notifyTo != null)
{
replyCommand = new CommandHandled(
notifyTo,
command.MessageId,
CommandHandled.CommandResult.Failed,
command.Describe(),
ex.Message,
isDomainException: true
);
replyCommand.CopyHeaders(command);
}
_logger.ErrorFormat(ex, "DomainException on command {0} [MessageId: {1}] : {2} : {3}", command.GetType(), command.MessageId, command.Describe(), ex.Message);
shouldContinue = true; //exception was handled, can proceed to the execution but retry is false
break;
case SecurityException sex:
SharedMetricsHelper.MarkSecurityException(command);
var notifySexTo = command.GetContextData(MessagesConstants.ReplyToHeader);
if (notifySexTo != null)
{
replyCommand = new CommandHandled(
notifySexTo,
command.MessageId,
CommandHandled.CommandResult.Failed,
command.Describe(),
$"Security exception: {ex.Message}",
isDomainException: false
);
replyCommand.CopyHeaders(command);
}
_logger.ErrorFormat(ex, "SecurityException on command {0} [MessageId: {1}] : {2} : {3}", command.GetType(), command.MessageId, command.Describe(), ex.Message);
shouldContinue = true; //exception was handled, can proceed to the execution but retry is false because security should not retry
break;
case Exception gex:
_logger.ErrorFormat(ex, "Generic Exception on command {0} [MessageId: {1}] : {2} : {3}", command.GetType(), command.MessageId, command.Describe(), ex.Message);
shouldContinue = false; //cannot proceed, this exception cannot be handled
break;
}
return(true);
}
public async Task Handle(T message)
{
LoggerThreadContextManager.MarkCommandExecution(message);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handling {0} {1}", message.GetType().FullName, message.MessageId);
}
int i = 0;
bool retry = false;
var notifyTo = message.GetContextData(MessagesConstants.ReplyToHeader);
CommandHandled replyCommandHandled = null;
Boolean success = false;
Exception lastException = null;
do
{
try
{
_messagesTracker.ElaborationStarted(message, DateTime.UtcNow);
_commandHandler.HandleAsync(message).Wait(); //need to wait, or you will free the worker rebus thread and you will dispatch many concurrent handler.
_messagesTracker.Completed(message, DateTime.UtcNow);
if (notifyTo != null && message.GetContextData("disable-success-reply", "false") != "true")
{
replyCommandHandled = new CommandHandled(
notifyTo,
message.MessageId,
CommandHandled.CommandResult.Handled,
message.Describe()
);
replyCommandHandled.CopyHeaders(message);
_bus.Reply(replyCommandHandled).Wait();
}
success = true;
retry = false;
}
catch (Exception ex)
{
lastException = ex;
if (!_commandExecutionExceptionHelper.Handle(ex, message, i, out retry, out replyCommandHandled))
{
//Handler is not able to handle the exception, simply retrhow
_messagesTracker.Failed(message, DateTime.UtcNow, ex);
LoggerThreadContextManager.ClearMarkCommandExecution();
throw;
}
}
i++; //if we reach here we need to increment the counter.
} while (retry);
if (!success)
{
_messagesTracker.Failed(message, DateTime.UtcNow, lastException);
if (notifyTo != null && replyCommandHandled != null)
{
await _bus.Reply(replyCommandHandled).ConfigureAwait(false);
}
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handled {0} {1} {2}", message.GetType().FullName, message.MessageId, message.Describe());
}
LoggerThreadContextManager.ClearMarkCommandExecution();
}
public async Task HandlePoisonMessage(TransportMessage transportMessage, ITransactionContext transactionContext, Exception exception)
{
try
{
if (transportMessage.Headers.ContainsKey("rbs2-msg-id"))
{
Guid commandId = Guid.Parse(transportMessage.Headers["rbs2-msg-id"]);
var description = transportMessage.Headers["rbs2-msg-type"];
String exMessage = description;
while (exception is TargetInvocationException)
{
exception = exception.InnerException;
}
var command = GetCommandFromMessage(transportMessage);
_lazyMessageTracker.Value.Failed(command, DateTime.UtcNow, exception);
if (exception != null)
{
exMessage = GetErrorMessage(exception);
_logger.ErrorFormat("HandlingPoisionMessage for {0}/{1} - {2}", commandId, description, command?.Describe());
}
if (command != null)
{
var notifyTo = command.GetContextData(MessagesConstants.ReplyToHeader);
if (!string.IsNullOrEmpty(notifyTo))
{
var commandHandled = new CommandHandled(
notifyTo,
commandId,
CommandHandled.CommandResult.Failed,
description,
exMessage
);
commandHandled.CopyHeaders(command);
Dictionary <String, String> headers = new Dictionary <string, string>
{
{ Headers.MessageId, Guid.NewGuid().ToString() }
};
//TODO: WIth new rebus I do not know how to resend header back. This will throw some unknown and obscure error in rebus.
await _lazyBus.Value.Advanced.Routing.Send(
transportMessage.Headers["rbs2-return-address"],
commandHandled,
headers).ConfigureAwait(false);
}
}
}
}
catch (Exception ex)
{
_logger.Error($"Error during HandlePoisonMessage of message: {ex.Message}", ex);
}
finally
{
var headers = transportMessage.Headers;
headers[Headers.ErrorDetails] = exception?.ToString();
headers[Headers.SourceQueue] = _transport.Address;
_logger.Error($"Moving message to error queue {_jarvisRebusConfiguration.ErrorQueue}", exception);
if (_transport == null)
{
_logger.Error("Error handler has no transport...this should be not possible. Problem in rebus initialization");
}
else
{
await _transport.Send(_jarvisRebusConfiguration.ErrorQueue, transportMessage, transactionContext).ConfigureAwait(false);
}
}
}
public void Handle(T message)
{
try
{
Logger.MarkCommandExecution(message);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handling {0} {1}", message.GetType().FullName, message.MessageId);
}
int i = 0;
bool done = false;
var notifyTo = message.GetContextData(MessagesConstants.ReplyToHeader);
while (!done && i < 100)
{
try
{
_messagesTracker.ElaborationStarted(message.MessageId, DateTime.UtcNow);
_commandHandler.Handle(message);
_messagesTracker.Completed(message.MessageId, DateTime.UtcNow);
done = true;
if (notifyTo != null && message.GetContextData("disable-success-reply", "false") != "true")
{
var replyCommand = new CommandHandled(
notifyTo,
message.MessageId,
CommandHandled.CommandResult.Handled,
message.Describe()
);
replyCommand.CopyHeaders(message);
_bus.Reply(replyCommand);
}
}
catch (ConflictingCommandException ex)
{
MetricsHelper.MarkConcurrencyException();
// retry
if (Logger.IsInfoEnabled)
{
Logger.InfoFormat(ex, "Handled {0} {1} [{2}], concurrency exception. Retry count: {3}", message.GetType().FullName, message.MessageId, message.Describe(), i);
}
if (i++ > 5)
{
Thread.Sleep(new Random(DateTime.Now.Millisecond).Next(i * 10));
}
}
catch (DomainException ex)
{
MetricsHelper.MarkDomainException();
done = true;
_messagesTracker.Failed(message.MessageId, DateTime.UtcNow, ex);
if (notifyTo != null)
{
var replyCommand = new CommandHandled(
notifyTo,
message.MessageId,
CommandHandled.CommandResult.Failed,
message.Describe(),
ex.Message,
true
);
replyCommand.CopyHeaders(message);
_bus.Reply(replyCommand);
}
_logger.ErrorFormat(ex, "DomainException on command {0} [MessageId: {1}] : {2} : {3}", message.GetType(), message.MessageId, message.Describe(), ex.Message);
}
catch (Exception ex)
{
_logger.ErrorFormat(ex, "Generic Exception on command {0} [MessageId: {1}] : {2} : {3}", message.GetType(), message.MessageId, message.Describe(), ex.Message);
_messagesTracker.Failed(message.MessageId, DateTime.UtcNow, ex);
throw; //rethrow exception.
}
}
if (done == false)
{
_logger.ErrorFormat("Too many conflict on command {0} [MessageId: {1}] : {2}", message.GetType(), message.MessageId, message.Describe());
var exception = new Exception("Command failed. Too many Conflicts");
_messagesTracker.Failed(message.MessageId, DateTime.UtcNow, exception);
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handled {0} {1} {3}", message.GetType().FullName, message.MessageId, message.Describe());
}
}
finally
{
Logger.ClearCommandExecution();
}
}
public void Handle(T message)
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handling {0} {1}", message.GetType().FullName, message.MessageId);
}
int i = 0;
bool done = false;
var notifyTo = message.GetContextData(MessagesConstants.ReplyToHeader);
while (!done && i < 100)
{
i++;
try
{
_commandHandler.Handle(message);
_messagesTracker.Completed(message.MessageId, DateTime.UtcNow);
done = true;
if (notifyTo != null && message.GetContextData("disable-success-reply", "false") != "true")
{
var replyCommand = new CommandHandled(
notifyTo,
message.MessageId,
CommandHandled.CommandResult.Handled,
message.Describe()
);
replyCommand.CopyHeaders(message);
_bus.Reply(replyCommand);
}
}
catch (ConflictingCommandException ex)
{
// retry
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handled {0} {1}, concurrency exception. Retry count: {2}", message.GetType().FullName, message.MessageId, i);
}
if (i++ > 5)
{
Thread.Sleep(new Random(DateTime.Now.Millisecond).Next(i * 10));
}
}
catch (DomainException ex)
{
done = true;
_messagesTracker.Failed(message.MessageId, DateTime.UtcNow, ex);
if (notifyTo != null)
{
var replyCommand = new CommandHandled(
notifyTo,
message.MessageId,
CommandHandled.CommandResult.Failed,
message.Describe(),
ex.Message,
true
);
replyCommand.CopyHeaders(message);
_bus.Reply(replyCommand);
}
}
}
if (done == false)
{
var exception = new Exception("Command failed. Too many Conflicts");
_messagesTracker.Failed(message.MessageId, DateTime.UtcNow, exception);
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handled {0} {1}", message.GetType().FullName, message.MessageId);
}
}