public void check_response_handled()
{
var sampleMessage = new SampleTestCommand(10);
sampleMessage.SetContextData(MessagesConstants.ReplyToHeader, "test");
_bus.Send(sampleMessage);
var handled = _handler.Reset.WaitOne(10000);
//cycle until we found handled message on tracking
List<TrackedMessageModel> tracks = null;
DateTime startTime = DateTime.Now;
do
{
Thread.Sleep(50);
tracks = _messages.FindAll().ToList();
}
while (
tracks.Count < 2 && //command tracking and commandhandled tracking
DateTime.Now.Subtract(startTime).TotalSeconds < 4
);
var track = tracks.Single(t => t.MessageId == sampleMessage.MessageId.ToString());
Assert.That(track.Description, Is.EqualTo(sampleMessage.Describe()));
Assert.That(track.StartedAt, Is.Not.Null);
Assert.That(track.CompletedAt, Is.Not.Null);
var handledTrack = tracks.Single(t => t.MessageId != sampleMessage.MessageId.ToString());
CommandHandled commandHandled = (CommandHandled)handledTrack.Message;
Assert.That(commandHandled.CommandId, Is.EqualTo(sampleMessage.MessageId));
Assert.That(handledTrack.Description, Is.StringContaining(sampleMessage.MessageId.ToString()));
Assert.That(handledTrack.Description, Is.StringContaining("Handled!"));
}
public void Consume(IConsumeContext <Command> context)
{
Console.WriteLine("Handling command...");
var @event = new CommandHandled {
CorrelationId = context.Message.CorrelationId
};
context.Bus.Publish(@event);
}
public async Task <IActionResult> SetLocation([FromBody] SetLocationModel model)
{
CommandHandled <SetLocationCommand, VoidResult> result = await HandleCommand <SetLocationCommand, VoidResult>(
new SetLocationCommand(model.UserId, model.Longitude, model.Latitude));
if (!result.ResultStatus.Successfully)
{
return(StatusCode(500, "Location cannot be set."));
}
return(Ok());
}
private void OnCommandHandled(
CommandHandled commandHandled,
Message request,
IMessageSender serverChannel)
{
// reply ok
var executeReplyPayload = new ExecuteReplyOk(executionCount: _executionCount);
// send to server
var executeReply = Message.CreateResponse(
executeReplyPayload,
request);
serverChannel.Send(executeReply);
}
void OnPoisonMessage(IBus bus, ReceivedTransportMessage message, PoisonMessageInfo poisonmessageinfo)
{
if (message.Headers.ContainsKey("command.id"))
{
Guid commandId = Guid.Parse(message.Headers["command.id"].ToString());
var description = message.Headers["command.description"].ToString();
var ex = poisonmessageinfo.Exceptions.Last();
string exMessage = description;
Exception exception = ex.Value;
while (exception is TargetInvocationException)
{
exception = exception.InnerException;
}
if (exception != null)
{
exMessage = exception.Message;
}
this.MessagesTracker.Failed(commandId, ex.Time, exception);
var command = GetCommandFromMessage(message);
if (command != null)
{
var notifyTo = command.GetContextData(MessagesConstants.ReplyToHeader);
if (!string.IsNullOrEmpty(notifyTo))
{
var commandHandled = new CommandHandled(
notifyTo,
commandId,
CommandHandled.CommandResult.Failed,
description,
exMessage
);
commandHandled.CopyHeaders(command);
bus.Advanced.Routing.Send(
message.Headers["rebus-return-address"].ToString(),
commandHandled);
}
}
}
}
private void OnCommandHandled(CommandHandled commandHandled)
{
if (!InFlightRequests.TryRemove(commandHandled.GetRootCommand(), out var openRequest))
{
return;
}
// reply ok
var executeReplyPayload = new ExecuteReplyOk(executionCount: openRequest.ExecutionCount);
// send to server
var executeReply = Message.CreateResponse(
executeReplyPayload,
openRequest.Context.Request);
openRequest.Context.ServerChannel.Send(executeReply);
openRequest.Context.RequestHandlerStatus.SetAsIdle();
openRequest.Dispose();
}
protected virtual async Task HandleAsync(ICommandHandler handler, ICommand command)
{
int i = 0;
Boolean success = false;
Boolean retry = false;
CommandHandled replyCommandHandled = null;
Exception lastException = null;
do
{
try
{
_messagesTracker.ElaborationStarted(command, DateTime.UtcNow);
await handler.HandleAsync(command).ConfigureAwait(false);
_messagesTracker.Completed(command, DateTime.UtcNow);
success = true;
retry = false;
}
catch (Exception ex)
{
lastException = ex;
if (!_commandExecutionExceptionHelper.Handle(ex, command, i, out retry, out replyCommandHandled))
{
//Handler is not able to handle the exception, simply retrhow
_messagesTracker.Failed(command, DateTime.UtcNow, ex);
throw;
}
}
i++; //if we reach here we need to increment the counter.
} while (retry);
if (!success)
{
_messagesTracker.Failed(command, DateTime.UtcNow, lastException);
throw lastException;
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Command {0} executed: {1}", command.MessageId, command.Describe());
}
}
public CommandReceptionistActor()
{
commandHandler = Context.ActorSelection(ActorSystemValues.COMMAND_HANDLER_ROUTER_PATH);
Receive <TCommand>(command => {
originalSender = Sender;
commandHandler.Tell(command);
SetReceiveTimeout(TimeSpan.FromSeconds(3));
Become(() => {
Receive <CommandHandled <TCommand, TCommandResult> >(x => {
originalSender.Tell(x);
Self.Tell(PoisonPill.Instance);
});
Receive <ReceiveTimeout>(x => {
originalSender.Tell(CommandHandled <TCommand, TCommandResult> .CreateUnsuccessfulResult(command, CommandResultStatus.GENERAL_ERROR));
});
});
});
}
private void WaitForAnswer()
{
Receive <ReceiveTimeout>(x => {
commandSender.Tell(CommandHandled <TCommand, TCommandResult> .CreateUnsuccessfulResult(command));
Self.Tell(PoisonPill.Instance);
});
Receive <CommandNotSuccessfulResult>(x => {
commandSender.Tell(CommandHandled <TCommand, TCommandResult> .CreateUnsuccessfulResult(command, x.CommandStatus));
Self.Tell(PoisonPill.Instance);
});
Receive <CommandHandled <TCommand, TCommandResult> >(x => {
commandSender.Tell(x);
Self.Tell(PoisonPill.Instance);
});
Receive <TCommandResult>(x => {
commandSender.Tell(CommandHandled <TCommand, TCommandResult> .CreateSuccessfulResult(command, x));
Self.Tell(PoisonPill.Instance);
});
}
public Boolean Handle(Exception ex, ICommand command, Int32 retryCount, out Boolean retry, out CommandHandled replyCommand)
{
retry = false;
replyCommand = null;
switch (ex)
{
case AggregateException aex:
aex = aex.Flatten();
Boolean aexRetry = false;
CommandHandled aexReplyCommand = null;
Boolean aexHandled = false;
aex.Handle(e => InnerHandle(e, command, retryCount, out aexRetry, out aexReplyCommand, out aexHandled));
//if we reach here, all exception were correctly handled.
retry = aexRetry;
replyCommand = aexReplyCommand;
return(aexHandled);
default:
Boolean handled;
InnerHandle(ex, command, retryCount, out retry, out replyCommand, out handled);
return(handled);
}
}
public void Handled(CommandHandled callback)
{
((ICanProcessCommandProcess)Proxy).AddHandled(callback);
}
/// <summary>
/// Inner function to handle the different type of exceptions
/// </summary>
/// <param name="ex"></param>
/// <param name="command"></param>
/// <param name="retryCount"></param>
/// <param name="retry"></param>
/// <param name="replyCommand"></param>
/// <param name="shouldContinue">true if the handler can continue execution, false if the execution of the command should stop
/// because there is no need to continue.</param>
/// <returns>Return always true because it is the very same routine used for aggregate exception. We always want not to throw in this
/// method, but allow the caller to throw.</returns>
private Boolean InnerHandle(Exception ex, ICommand command, Int32 retryCount, out Boolean retry, out CommandHandled replyCommand, out Boolean shouldContinue)
{
retry = false;
replyCommand = null;
shouldContinue = false;
switch (ex)
{
case ConcurrencyException cex:
SharedMetricsHelper.MarkConcurrencyException();
// retry
if (_logger.IsInfoEnabled)
{
_logger.InfoFormat(ex, "Handled {0} {1} [{2}], concurrency exception. Retry count: {3}", command.GetType().FullName, command.MessageId, command.Describe(), retryCount);
}
// increment the retries counter and maybe add a delay
if (retryCount++ > _numberOfConcurrencyExceptionBeforeRandomSleeping)
{
Thread.Sleep(new Random(DateTime.Now.Millisecond).Next(retryCount * 10));
}
retry = retryCount < _maxRetryOnConcurrencyException; //can retry
shouldContinue = true; //can proceed
break;
case DomainException dex:
SharedMetricsHelper.MarkDomainException(command, dex);
var notifyTo = command.GetContextData(MessagesConstants.ReplyToHeader);
if (notifyTo != null)
{
replyCommand = new CommandHandled(
notifyTo,
command.MessageId,
CommandHandled.CommandResult.Failed,
command.Describe(),
ex.Message,
isDomainException: true
);
replyCommand.CopyHeaders(command);
}
_logger.ErrorFormat(ex, "DomainException on command {0} [MessageId: {1}] : {2} : {3}", command.GetType(), command.MessageId, command.Describe(), ex.Message);
shouldContinue = true; //exception was handled, can proceed to the execution but retry is false
break;
case SecurityException sex:
SharedMetricsHelper.MarkSecurityException(command);
var notifySexTo = command.GetContextData(MessagesConstants.ReplyToHeader);
if (notifySexTo != null)
{
replyCommand = new CommandHandled(
notifySexTo,
command.MessageId,
CommandHandled.CommandResult.Failed,
command.Describe(),
$"Security exception: {ex.Message}",
isDomainException: false
);
replyCommand.CopyHeaders(command);
}
_logger.ErrorFormat(ex, "SecurityException on command {0} [MessageId: {1}] : {2} : {3}", command.GetType(), command.MessageId, command.Describe(), ex.Message);
shouldContinue = true; //exception was handled, can proceed to the execution but retry is false because security should not retry
break;
case Exception gex:
_logger.ErrorFormat(ex, "Generic Exception on command {0} [MessageId: {1}] : {2} : {3}", command.GetType(), command.MessageId, command.Describe(), ex.Message);
shouldContinue = false; //cannot proceed, this exception cannot be handled
break;
}
return(true);
}
public void AddHandled(CommandHandled callback)
{
_commandHandledCallbacks.Add(new WeakDelegate(callback));
}
public void Handled(CommandHandled callback)
{
throw new System.NotImplementedException();
}
public void Handle(T message)
{
try
{
Logger.MarkCommandExecution(message);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handling {0} {1}", message.GetType().FullName, message.MessageId);
}
int i = 0;
bool done = false;
var notifyTo = message.GetContextData(MessagesConstants.ReplyToHeader);
while (!done && i < 100)
{
try
{
_messagesTracker.ElaborationStarted(message.MessageId, DateTime.UtcNow);
_commandHandler.Handle(message);
_messagesTracker.Completed(message.MessageId, DateTime.UtcNow);
done = true;
if (notifyTo != null && message.GetContextData("disable-success-reply", "false") != "true")
{
var replyCommand = new CommandHandled(
notifyTo,
message.MessageId,
CommandHandled.CommandResult.Handled,
message.Describe()
);
replyCommand.CopyHeaders(message);
_bus.Reply(replyCommand);
}
}
catch (ConflictingCommandException ex)
{
MetricsHelper.MarkConcurrencyException();
// retry
if (Logger.IsInfoEnabled)
{
Logger.InfoFormat(ex, "Handled {0} {1} [{2}], concurrency exception. Retry count: {3}", message.GetType().FullName, message.MessageId, message.Describe(), i);
}
if (i++ > 5)
{
Thread.Sleep(new Random(DateTime.Now.Millisecond).Next(i * 10));
}
}
catch (DomainException ex)
{
MetricsHelper.MarkDomainException();
done = true;
_messagesTracker.Failed(message.MessageId, DateTime.UtcNow, ex);
if (notifyTo != null)
{
var replyCommand = new CommandHandled(
notifyTo,
message.MessageId,
CommandHandled.CommandResult.Failed,
message.Describe(),
ex.Message,
true
);
replyCommand.CopyHeaders(message);
_bus.Reply(replyCommand);
}
_logger.ErrorFormat(ex, "DomainException on command {0} [MessageId: {1}] : {2} : {3}", message.GetType(), message.MessageId, message.Describe(), ex.Message);
}
catch (Exception ex)
{
_logger.ErrorFormat(ex, "Generic Exception on command {0} [MessageId: {1}] : {2} : {3}", message.GetType(), message.MessageId, message.Describe(), ex.Message);
_messagesTracker.Failed(message.MessageId, DateTime.UtcNow, ex);
throw; //rethrow exception.
}
}
if (done == false)
{
_logger.ErrorFormat("Too many conflict on command {0} [MessageId: {1}] : {2}", message.GetType(), message.MessageId, message.Describe());
var exception = new Exception("Command failed. Too many Conflicts");
_messagesTracker.Failed(message.MessageId, DateTime.UtcNow, exception);
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handled {0} {1} {3}", message.GetType().FullName, message.MessageId, message.Describe());
}
}
finally
{
Logger.ClearCommandExecution();
}
}
public void Handle(T message)
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handling {0} {1}", message.GetType().FullName, message.MessageId);
}
int i = 0;
bool done = false;
var notifyTo = message.GetContextData(MessagesConstants.ReplyToHeader);
while (!done && i < 100)
{
i++;
try
{
_commandHandler.Handle(message);
_messagesTracker.Completed(message.MessageId, DateTime.UtcNow);
done = true;
if (notifyTo != null && message.GetContextData("disable-success-reply", "false") != "true")
{
var replyCommand = new CommandHandled(
notifyTo,
message.MessageId,
CommandHandled.CommandResult.Handled,
message.Describe()
);
replyCommand.CopyHeaders(message);
_bus.Reply(replyCommand);
}
}
catch (ConflictingCommandException ex)
{
// retry
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handled {0} {1}, concurrency exception. Retry count: {2}", message.GetType().FullName, message.MessageId, i);
}
if (i++ > 5)
{
Thread.Sleep(new Random(DateTime.Now.Millisecond).Next(i * 10));
}
}
catch (DomainException ex)
{
done = true;
_messagesTracker.Failed(message.MessageId, DateTime.UtcNow, ex);
if (notifyTo != null)
{
var replyCommand = new CommandHandled(
notifyTo,
message.MessageId,
CommandHandled.CommandResult.Failed,
message.Describe(),
ex.Message,
true
);
replyCommand.CopyHeaders(message);
_bus.Reply(replyCommand);
}
}
}
if (done == false)
{
var exception = new Exception("Command failed. Too many Conflicts");
_messagesTracker.Failed(message.MessageId, DateTime.UtcNow, exception);
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handled {0} {1}", message.GetType().FullName, message.MessageId);
}
}
public async Task Handle(T message)
{
LoggerThreadContextManager.MarkCommandExecution(message);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handling {0} {1}", message.GetType().FullName, message.MessageId);
}
int i = 0;
bool retry = false;
var notifyTo = message.GetContextData(MessagesConstants.ReplyToHeader);
CommandHandled replyCommandHandled = null;
Boolean success = false;
Exception lastException = null;
do
{
try
{
_messagesTracker.ElaborationStarted(message, DateTime.UtcNow);
_commandHandler.HandleAsync(message).Wait(); //need to wait, or you will free the worker rebus thread and you will dispatch many concurrent handler.
_messagesTracker.Completed(message, DateTime.UtcNow);
if (notifyTo != null && message.GetContextData("disable-success-reply", "false") != "true")
{
replyCommandHandled = new CommandHandled(
notifyTo,
message.MessageId,
CommandHandled.CommandResult.Handled,
message.Describe()
);
replyCommandHandled.CopyHeaders(message);
_bus.Reply(replyCommandHandled).Wait();
}
success = true;
retry = false;
}
catch (Exception ex)
{
lastException = ex;
if (!_commandExecutionExceptionHelper.Handle(ex, message, i, out retry, out replyCommandHandled))
{
//Handler is not able to handle the exception, simply retrhow
_messagesTracker.Failed(message, DateTime.UtcNow, ex);
LoggerThreadContextManager.ClearMarkCommandExecution();
throw;
}
}
i++; //if we reach here we need to increment the counter.
} while (retry);
if (!success)
{
_messagesTracker.Failed(message, DateTime.UtcNow, lastException);
if (notifyTo != null && replyCommandHandled != null)
{
await _bus.Reply(replyCommandHandled).ConfigureAwait(false);
}
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Handled {0} {1} {2}", message.GetType().FullName, message.MessageId, message.Describe());
}
LoggerThreadContextManager.ClearMarkCommandExecution();
}
public void Handled(CommandHandled callback)
{
((ICanProcessCommandProcess)Proxy).AddHandled(callback);
}
public async Task HandlePoisonMessage(TransportMessage transportMessage, ITransactionContext transactionContext, Exception exception)
{
try
{
if (transportMessage.Headers.ContainsKey("rbs2-msg-id"))
{
Guid commandId = Guid.Parse(transportMessage.Headers["rbs2-msg-id"]);
var description = transportMessage.Headers["rbs2-msg-type"];
String exMessage = description;
while (exception is TargetInvocationException)
{
exception = exception.InnerException;
}
var command = GetCommandFromMessage(transportMessage);
_lazyMessageTracker.Value.Failed(command, DateTime.UtcNow, exception);
if (exception != null)
{
exMessage = GetErrorMessage(exception);
_logger.ErrorFormat("HandlingPoisionMessage for {0}/{1} - {2}", commandId, description, command?.Describe());
}
if (command != null)
{
var notifyTo = command.GetContextData(MessagesConstants.ReplyToHeader);
if (!string.IsNullOrEmpty(notifyTo))
{
var commandHandled = new CommandHandled(
notifyTo,
commandId,
CommandHandled.CommandResult.Failed,
description,
exMessage
);
commandHandled.CopyHeaders(command);
Dictionary <String, String> headers = new Dictionary <string, string>
{
{ Headers.MessageId, Guid.NewGuid().ToString() }
};
//TODO: WIth new rebus I do not know how to resend header back. This will throw some unknown and obscure error in rebus.
await _lazyBus.Value.Advanced.Routing.Send(
transportMessage.Headers["rbs2-return-address"],
commandHandled,
headers).ConfigureAwait(false);
}
}
}
}
catch (Exception ex)
{
_logger.Error($"Error during HandlePoisonMessage of message: {ex.Message}", ex);
}
finally
{
var headers = transportMessage.Headers;
headers[Headers.ErrorDetails] = exception?.ToString();
headers[Headers.SourceQueue] = _transport.Address;
_logger.Error($"Moving message to error queue {_jarvisRebusConfiguration.ErrorQueue}", exception);
if (_transport == null)
{
_logger.Error("Error handler has no transport...this should be not possible. Problem in rebus initialization");
}
else
{
await _transport.Send(_jarvisRebusConfiguration.ErrorQueue, transportMessage, transactionContext).ConfigureAwait(false);
}
}
}
