protected override void OnMessageSerializationFailure(Message msg, Exception exc)
{
// we only get here if we failed to serialize the msg (or any other catastrophic failure).
// Request msg fails to serialize on the sending silo, so we just enqueue a rejection msg.
// Response msg fails to serialize on the responding silo, so we try to send an error response back.
Log.Warn(ErrorCode.MessagingUnexpectedSendError, String.Format("Unexpected error sending message {0}", msg.ToString()), exc);
msg.ReleaseBodyAndHeaderBuffers();
MessagingStatisticsGroup.OnFailedSentMessage(msg);
if (msg.Direction == Message.Directions.Request)
{
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, exc.ToString());
}
else if (msg.Direction == Message.Directions.Response && msg.Result != Message.ResponseTypes.Error)
{
// if we failed sending an original response, turn the response body into an error and reply with it.
// unless the response was already an error response (so we don't loop forever).
msg.Result = Message.ResponseTypes.Error;
msg.BodyObject = Response.ExceptionResponse(exc);
messageCenter.SendMessage(msg);
}
else
{
MessagingStatisticsGroup.OnDroppedSentMessage(msg);
}
}
protected override void OnMessageSerializationFailure(Message msg, Exception exc)
{
// we only get here if we failed to serialize the msg (or any other catastrophic failure).
// Request msg fails to serialize on the sending silo, so we just enqueue a rejection msg.
// Response msg fails to serialize on the responding silo, so we try to send an error response back.
this.Log.LogWarning(
(int)ErrorCode.MessagingUnexpectedSendError,
"Unexpected error serializing message {Message}: {Exception}",
msg,
exc);
msg.ReleaseBodyAndHeaderBuffers();
MessagingStatisticsGroup.OnFailedSentMessage(msg);
var retryCount = msg.RetryCount ?? 0;
if (msg.Direction == Message.Directions.Request)
{
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, exc.ToString());
}
else if (msg.Direction == Message.Directions.Response && retryCount < 1)
{
// if we failed sending an original response, turn the response body into an error and reply with it.
// unless we have already tried sending the response multiple times.
msg.Result = Message.ResponseTypes.Error;
msg.BodyObject = Response.ExceptionResponse(exc);
msg.RetryCount = retryCount + 1;
this.messageCenter.SendMessage(msg);
}
else
{
this.Log.LogWarning(
(int)ErrorCode.Messaging_OutgoingMS_DroppingMessage,
"Silo {SiloAddress} is dropping message which failed during serialization: {Message}. Exception = {Exception}",
this.messageCenter.MyAddress,
msg,
exc);
MessagingStatisticsGroup.OnDroppedSentMessage(msg);
}
}
public void SendMessage(Message msg)
{
if (msg == null)
{
throw new ArgumentNullException("msg", "Can't send a null message.");
}
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100112, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// Don't process messages that have already timed out
if (msg.IsExpired)
{
msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Send);
return;
}
if (!msg.QueuedTime.HasValue)
{
msg.QueuedTime = DateTime.UtcNow;
}
// First check to see if it's really destined for a proxied client, instead of a local grain.
if (messageCenter.IsProxying && messageCenter.TryDeliverToProxy(msg))
{
return;
}
if (msg.TargetSilo == null)
{
logger.Error(ErrorCode.Runtime_Error_100113, "Message does not have a target silo: " + msg + " -- Call stack is: " + Utils.GetStackTrace());
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message to be sent does not have a target silo");
return;
}
if (!messageCenter.TrySendLocal(msg))
{
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100115, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// check for simulation of lost messages
if (messageCenter.ShouldDrop?.Invoke(msg) == true)
{
logger.Info(ErrorCode.Messaging_SimulatedMessageLoss, "Message blocked by test");
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message blocked by test");
return;
}
if (this.siloStatusOracle.IsDeadSilo(msg.TargetSilo))
{
MessagingStatisticsGroup.OnFailedSentMessage(msg);
var reason = $"Target {msg.TargetSilo.ToLongString()} silo is known to be dead";
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug(
(int)ErrorCode.MessagingSendingRejection,
"Silo {siloAddress} is rejecting message: {message}. Reason = {reason}",
messageCenter.MyAddress,
msg,
reason);
}
this.messageCenter.SendRejection(msg, Message.RejectionTypes.Transient, reason);
return;
}
var senderTask = this.connectionManager.GetConnection(msg.TargetSilo);
if (senderTask.IsCompletedSuccessfully)
{
var sender = senderTask.Result;
sender.Send(msg);
}
else
{
_ = SendAsync(senderTask, msg);
async Task SendAsync(ValueTask <Connection> c, Message m)
{
try
{
var sender = await c;
sender.Send(m);
}
catch (Exception exception)
{
this.messageCenter.SendRejection(m, Message.RejectionTypes.Transient, $"Exception while sending message: {exception}");
}
}
}
}
}
public void SendMessage(Message msg)
{
if (msg == null)
{
throw new ArgumentNullException("msg", "Can't send a null message.");
}
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100112, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// Don't process messages that have already timed out
if (msg.IsExpired)
{
msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Send);
return;
}
if (!msg.QueuedTime.HasValue)
{
msg.QueuedTime = DateTime.UtcNow;
}
// First check to see if it's really destined for a proxied client, instead of a local grain.
if (messageCenter.IsProxying && messageCenter.TryDeliverToProxy(msg))
{
return;
}
if (msg.TargetSilo == null)
{
logger.Error(ErrorCode.Runtime_Error_100113, "Message does not have a target silo: " + msg + " -- Call stack is: " + Utils.GetStackTrace());
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message to be sent does not have a target silo");
return;
}
if (!messageCenter.TrySendLocal(msg))
{
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100115, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// check for simulation of lost messages
if (messageCenter?.ShouldDrop?.Invoke(msg) == true)
{
logger.Info(ErrorCode.Messaging_SimulatedMessageLoss, "Message blocked by test");
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message blocked by test");
return;
}
// Prioritize system messages
switch (msg.Category)
{
case Message.Categories.Ping:
pingSender.QueueRequest(msg);
break;
case Message.Categories.System:
systemSender.QueueRequest(msg);
break;
default:
{
int index = Math.Abs(msg.TargetSilo.GetConsistentHashCode()) % senders.Length;
senders[index].Value.QueueRequest(msg);
break;
}
}
}
}
public void SendMessage(Message msg)
{
if (msg is null)
{
throw new ArgumentNullException("msg", "Can't send a null message.");
}
if (stopped)
{
logger.LogInformation((int)ErrorCode.Runtime_Error_100115, "Message was queued for sending after outbound queue was stopped: {Message}", msg);
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message was queued for sending after outbound queue was stopped");
return;
}
// Don't process messages that have already timed out
if (msg.IsExpired)
{
this.messagingTrace.OnDropExpiredMessage(msg, MessagingStatisticsGroup.Phase.Send);
return;
}
// First check to see if it's really destined for a proxied client, instead of a local grain.
if (messageCenter.TryDeliverToProxy(msg))
{
// Message was successfully delivered to the proxy.
return;
}
if (msg.TargetSilo == null)
{
logger.LogError((int)ErrorCode.Runtime_Error_100113, "Message does not have a target silo: " + msg + " -- Call stack is: " + Utils.GetStackTrace());
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message to be sent does not have a target silo");
return;
}
messagingTrace.OnSendMessage(msg);
if (!messageCenter.TrySendLocal(msg))
{
if (stopped)
{
logger.LogInformation((int)ErrorCode.Runtime_Error_100115, "Message was queued for sending after outbound queue was stopped: {Message}", msg);
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message was queued for sending after outbound queue was stopped");
return;
}
// check for simulation of lost messages
if (messageCenter.ShouldDrop?.Invoke(msg) == true)
{
logger.LogInformation((int)ErrorCode.Messaging_SimulatedMessageLoss, "Message blocked by test");
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message blocked by test");
return;
}
if (this.connectionManager.TryGetConnection(msg.TargetSilo, out var existingConnection))
{
existingConnection.Send(msg);
return;
}
else if (this.siloStatusOracle.IsDeadSilo(msg.TargetSilo))
{
// Do not try to establish
this.messagingTrace.OnRejectSendMessageToDeadSilo(this.messageCenter.MyAddress, msg);
this.messageCenter.SendRejection(msg, Message.RejectionTypes.Transient, "Target silo is known to be dead");
return;
}
else
{
var senderTask = this.connectionManager.GetConnection(msg.TargetSilo);
if (senderTask.IsCompletedSuccessfully)
{
var sender = senderTask.Result;
sender.Send(msg);
}
else
{
_ = SendAsync(senderTask, msg);
async Task SendAsync(ValueTask <Connection> c, Message m)
{
try
{
var sender = await c;
sender.Send(m);
}
catch (Exception exception)
{
this.messageCenter.SendRejection(m, Message.RejectionTypes.Transient, $"Exception while sending message: {exception}");
}
}
}
}
}
}
public void SendMessage(Message msg)
{
if (msg == null)
{
throw new ArgumentNullException("msg", "Can't send a null message.");
}
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100112, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// Don't process messages that have already timed out
if (msg.IsExpired)
{
msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Send);
return;
}
if (!msg.ContainsMetadata(QUEUED_TIME_METADATA))
{
msg.SetMetadata(QUEUED_TIME_METADATA, DateTime.UtcNow);
}
// First check to see if it's really destined for a proxied client, instead of a local grain.
if (messageCenter.IsProxying && messageCenter.TryDeliverToProxy(msg))
{
return;
}
if (!msg.ContainsHeader(Message.Header.TARGET_SILO))
{
logger.Error(ErrorCode.Runtime_Error_100113, "Message does not have a target silo: " + msg + " -- Call stack is: " + (new System.Diagnostics.StackTrace()));
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message to be sent does not have a target silo");
return;
}
if (Message.WriteMessagingTraces)
{
msg.AddTimestamp(Message.LifecycleTag.EnqueueOutgoing);
}
// Shortcut messages to this silo
if (msg.TargetSilo.Equals(messageCenter.MyAddress))
{
if (logger.IsVerbose3)
{
logger.Verbose3("Message has been looped back to this silo: {0}", msg);
}
MessagingStatisticsGroup.LocalMessagesSent.Increment();
messageCenter.InboundQueue.PostMessage(msg);
}
else
{
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100115, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// check for simulation of lost messages
if (Silo.CurrentSilo.TestHook.ShouldDrop(msg))
{
logger.Info(ErrorCode.Messaging_SimulatedMessageLoss, "Message blocked by test");
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message blocked by test");
}
// Prioritize system messages
switch (msg.Category)
{
case Message.Categories.Ping:
pingSender.QueueRequest(msg);
break;
case Message.Categories.System:
systemSender.QueueRequest(msg);
break;
default:
{
int index = Math.Abs(msg.TargetSilo.GetConsistentHashCode()) % senders.Length;
senders[index].Value.QueueRequest(msg);
break;
}
}
}
}