/// <summary>
/// Returns true if the max number of retries for the message have been exceeded, and removes the message Id at the same time. Also sets the last Exception
/// raised. Otherwise returns false.
/// </summary>
public bool HasMaxRetriesExceeded(IncomingTransportMessage message, out Exception lastException)
{
string messageId = message.MessageId;
failuresPerMessageLocker.EnterReadLock();
if (failuresPerMessage.ContainsKey(messageId) &&
(failuresPerMessage[messageId] >= maxRetries))
{
failuresPerMessageLocker.ExitReadLock();
failuresPerMessageLocker.EnterWriteLock();
lastException = lastExceptionsForMessage[messageId];
failuresPerMessage.Remove(messageId);
lastExceptionsForMessage.Remove(messageId);
messageFailureExpiration.Remove(messageId);
failuresPerMessageLocker.ExitWriteLock();
return(true);
}
lastException = null;
failuresPerMessageLocker.ExitReadLock();
return(false);
}
/// <summary>
/// Returns true if the max number of retries for the message have been exceeded, and removes the message Id at the same time. Also sets the last Exception
/// raised. Otherwise returns false.
/// </summary>
public bool HasMaxRetriesExceeded(IncomingTransportMessage message, out Exception lastException)
{
string messageId = message.MessageId;
failuresPerMessageLocker.EnterReadLock();
if (failuresPerMessage.ContainsKey(messageId) &&
(failuresPerMessage[messageId] >= maxRetries))
{
failuresPerMessageLocker.ExitReadLock();
failuresPerMessageLocker.EnterWriteLock();
lastException = lastExceptionsForMessage[messageId];
failuresPerMessage.Remove(messageId);
lastExceptionsForMessage.Remove(messageId);
messageFailureExpiration.Remove(messageId);
failuresPerMessageLocker.ExitWriteLock();
return true;
}
lastException = null;
failuresPerMessageLocker.ExitReadLock();
return false;
}
private void TryHandleMessage(object message)
{
var messageID = MessageId(message).ToString();
logger.Debug("Received message {0} from queue {1}", messageID, QueueName);
// NOTE this method _should not_ throw an exception!
try
{
var transportMessage = new IncomingTransportMessage(messageID, new Dictionary<string, string>(), message);
Exception lastException = null;
if (faultManager.HasMaxRetriesExceeded(transportMessage, out lastException))
{
logger.Debug("MaxRetriesExceeded. Will not re-enque.", messageID.ToString(), QueueName);
OnPoisonMessageDetected(new PoisonMessageDetectedEventArgs()
{
QueueName = QueueName,
Retries = MaxRetries,
Exception = lastException,
MessageId = messageID,
ErrorCode = "MaxRetriesExceeded",
});
return; // return without error to commit transaction
}
logger.Debug("Notifying observers of new TransportMessage for message {0} from queue {1}.", messageID.ToString(), QueueName);
OnMessageAvailable(transportMessage);
faultManager.ClearFailuresForMessage(messageID);
logger.Debug("Committed message {0} from queue {1}", messageID.ToString(), QueueName);
}
catch (Exception e)
{
faultManager.IncrementFailuresForMessage(messageID.ToString(), e);
logger.Error(e, "Exception caught handling message {0} from queue {1}. Re-enqueing.", messageID, QueueName);
Thread.Sleep(1000); // TODO possibly implement a backoff with the fault manager based on number of retries?
queue.Enqueue(message);
queueNotifier.Set();
}
}
private void OnMessageAvailable(IncomingTransportMessage transportMessage)
{
MessageAvailable?.Invoke(this, new MessageAvailableEventArgs(transportMessage));
}
private void TryHandleMessage(ReceiverLink receiverLink, Message amqpMessage)
{
// NOTE this method _should not_ throw an exception!
var threadName = receiverLink.Name;
var messageID = amqpMessage.Properties?.MessageId ?? "?";
try
{
logger.Debug("{0}: Received Message with MessageId={1}", threadName, messageID);
var headers = new Dictionary<string, string>();
if (amqpMessage.ApplicationProperties != null)
{
foreach (var key in amqpMessage.ApplicationProperties.Map.Keys)
{
headers[key.ToString()] = amqpMessage.ApplicationProperties.Map[key].ToString();
}
}
object message = null;
try
{
message = DecodeMessage(amqpMessage);
}
catch (Exception e)
{
logger.Error(e, "{0}: Cannot Decode Message. Will not re-enqueue.", threadName);
OnPoisonMessageDetected(new PoisonMessageDetectedEventArgs()
{
QueueName = ReceiverLinkAddress,
Retries = MaxRetries,
Exception = e,
MessageId = messageID,
ErrorCode = "DecodeException",
});
}
if (message != null)
{
var transportMessage = new IncomingTransportMessage(messageID, headers, message);
Exception lastException = null;
if (faultManager.HasMaxRetriesExceeded(transportMessage, out lastException))
{
logger.Error(lastException, "{0}: MaxRetriesExceeded for MessageId={1}. Will not re-enqueue.", threadName, messageID.ToString());
OnPoisonMessageDetected(new PoisonMessageDetectedEventArgs()
{
QueueName = ReceiverLinkAddress,
Retries = MaxRetries,
Exception = lastException,
MessageId = messageID,
ErrorCode = "MaxRetriesExceeded",
});
}
else
{
logger.Debug("{0}: Notifying observers of new TransportMessage for MessageId={1}.", threadName, messageID.ToString());
OnMessageAvailable(transportMessage);
faultManager.ClearFailuresForMessage(messageID);
}
}
receiverLink.Accept(amqpMessage);
logger.Debug("{0}: Accepting MessageId={1}", threadName, messageID.ToString());
}
catch (Exception e)
{
faultManager.IncrementFailuresForMessage(messageID.ToString(), e);
receiverLink.Reject(amqpMessage);
logger.Error(e, "{0}: Exception caught handling MessageId={1}. Rejecting.", threadName, messageID.ToString());
Thread.Sleep(1000); // TODO possibly implement a backoff with the fault manager based on number of retries?
}
}
private void OnMessageAvailable(IncomingTransportMessage transportMessage)
{
var callback = MessageAvailable;
if (callback != null)
{
callback(this, new MessageAvailableEventArgs(transportMessage));
}
}
private void MoveToPoisonMessage(SqlTransaction transaction
, ServiceBrokerMessage serviceBrokerMessage
, IncomingTransportMessage transportMessage
, Exception exception
, string errorCode
, int retries)
{
var origin_service_name = "";
if (transportMessage != null && transportMessage.Headers.ContainsKey(StandardHeaders.OriginatingAddress))
{
origin_service_name = transportMessage.Headers[StandardHeaders.OriginatingAddress];
}
else if (transportMessage != null && transportMessage.Headers.ContainsKey(StandardHeaders.ReplyToAddress))
{
origin_service_name = transportMessage.Headers[StandardHeaders.ReplyToAddress];
}
else
{
origin_service_name = ServiceBrokerWrapper.TryGetConversationFarService(transaction, serviceBrokerMessage.ConversationHandle);
}
var enqueuedDateTime = DateTime.MinValue;
if (transportMessage != null && transportMessage.Headers.ContainsKey(StandardHeaders.TimeSent))
{
DateTime.TryParse(transportMessage.Headers[StandardHeaders.TimeSent], out enqueuedDateTime);
}
try
{
PoisonMessageSqlHelper.WriteToPoisonMessageTable(
transaction
, serviceBrokerMessage.ConversationHandle
, origin_service_name
, enqueuedDateTime
, ServiceBrokerService
, ServiceBrokerQueue
, serviceBrokerMessage.Body
, MaxRetries
, errorCode
, exception);
}
catch (Exception e)
{
logger.Fatal(e, "Failed to write PoisonMessage for message {0} from queue {1}", serviceBrokerMessage.ConversationHandle, ServiceBrokerQueue);
// suppress -- don't let this exception take down the process
}
PoisonMessageDetected?.Invoke(this, new PoisonMessageDetectedEventArgs()
{
MessageId = serviceBrokerMessage.ConversationHandle.ToString(),
OriginServiceName = origin_service_name,
ServiceName = this.ServiceBrokerService,
QueueName = this.ServiceBrokerQueue,
MessageBody = serviceBrokerMessage.Body,
Retries = retries,
ErrorCode = errorCode,
Exception = exception,
});
}
public MessageAvailableEventArgs(IncomingTransportMessage transportMessage)
{
this.TransportMessage = transportMessage;
}