protected override void OnReceivedMessage(Message msg)
{
// See it's a Ping message, and if so, short-circuit it
var requestContext = msg.RequestContextData;
if (requestContext != null &&
requestContext.TryGetValue(RequestContext.PING_APPLICATION_HEADER, out var pingObj) &&
pingObj is bool &&
(bool)pingObj)
{
MessagingStatisticsGroup.OnPingReceive(msg.SendingSilo);
if (this.Log.IsEnabled(LogLevel.Trace))
{
this.Log.Trace("Responding to Ping from {0}", msg.SendingSilo);
}
if (!msg.TargetSilo.Equals(messageCenter.MyAddress)) // got ping that is not destined to me. For example, got a ping to my older incarnation.
{
MessagingStatisticsGroup.OnRejectedMessage(msg);
Message rejection = this.messageFactory.CreateRejectionResponse(msg, Message.RejectionTypes.Unrecoverable,
$"The target silo is no longer active: target was {msg.TargetSilo.ToLongString()}, but this silo is {messageCenter.MyAddress.ToLongString()}. " +
$"The rejected ping message is {msg}.");
messageCenter.OutboundQueue.SendMessage(rejection);
}
else
{
var response = this.messageFactory.CreateResponseMessage(msg);
response.BodyObject = Response.Done;
this.messageCenter.SendMessage(response);
}
return;
}
// sniff message headers for directory cache management
this.messageCenter.SniffIncomingMessage?.Invoke(msg);
// Don't process messages that have already timed out
if (msg.IsExpired)
{
msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Receive);
return;
}
// If we've stopped application message processing, then filter those out now
// Note that if we identify or add other grains that are required for proper stopping, we will need to treat them as we do the membership table grain here.
if (messageCenter.IsBlockingApplicationMessages && (msg.Category == Message.Categories.Application) && !Constants.SystemMembershipTableId.Equals(msg.SendingGrain))
{
// We reject new requests, and drop all other messages
if (msg.Direction != Message.Directions.Request)
{
return;
}
MessagingStatisticsGroup.OnRejectedMessage(msg);
var reject = this.messageFactory.CreateRejectionResponse(msg, Message.RejectionTypes.Unrecoverable, "Silo stopping");
this.messageCenter.SendMessage(reject);
return;
}
// Make sure the message is for us. Note that some control messages may have no target
// information, so a null target silo is OK.
if ((msg.TargetSilo == null) || msg.TargetSilo.Matches(messageCenter.MyAddress))
{
// See if it's a message for a client we're proxying.
if (messageCenter.IsProxying && messageCenter.TryDeliverToProxy(msg))
{
return;
}
// Nope, it's for us
messageCenter.OnReceivedMessage(msg);
return;
}
if (!msg.TargetSilo.Endpoint.Equals(messageCenter.MyAddress.Endpoint))
{
// If the message is for some other silo altogether, then we need to forward it.
if (this.Log.IsEnabled(LogLevel.Trace))
{
this.Log.Trace("Forwarding message {0} from {1} to silo {2}", msg.Id, msg.SendingSilo, msg.TargetSilo);
}
messageCenter.OutboundQueue.SendMessage(msg);
return;
}
// If the message was for this endpoint but an older epoch, then reject the message
// (if it was a request), or drop it on the floor if it was a response or one-way.
if (msg.Direction == Message.Directions.Request)
{
MessagingStatisticsGroup.OnRejectedMessage(msg);
Message rejection = this.messageFactory.CreateRejectionResponse(msg, Message.RejectionTypes.Transient,
string.Format("The target silo is no longer active: target was {0}, but this silo is {1}. The rejected message is {2}.",
msg.TargetSilo.ToLongString(), messageCenter.MyAddress.ToLongString(), msg));
// Invalidate the remote caller's activation cache entry.
if (msg.TargetAddress != null)
{
rejection.AddToCacheInvalidationHeader(msg.TargetAddress);
}
messageCenter.OutboundQueue.SendMessage(rejection);
if (this.Log.IsEnabled(LogLevel.Debug))
{
this.Log.Debug("Rejecting an obsolete request; target was {0}, but this silo is {1}. The rejected message is {2}.",
msg.TargetSilo.ToLongString(), messageCenter.MyAddress.ToLongString(), msg);
}
}
}
public void SendMessage(Message msg)
{
if (msg is null)
{
throw new ArgumentNullException("msg", "Can't send a null message.");
}
if (stopped)
{
logger.LogInformation((int)ErrorCode.Runtime_Error_100115, "Message was queued for sending after outbound queue was stopped: {Message}", msg);
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message was queued for sending after outbound queue was stopped");
return;
}
// Don't process messages that have already timed out
if (msg.IsExpired)
{
this.messagingTrace.OnDropExpiredMessage(msg, MessagingStatisticsGroup.Phase.Send);
return;
}
// First check to see if it's really destined for a proxied client, instead of a local grain.
if (messageCenter.TryDeliverToProxy(msg))
{
// Message was successfully delivered to the proxy.
return;
}
if (msg.TargetSilo == null)
{
logger.LogError((int)ErrorCode.Runtime_Error_100113, "Message does not have a target silo: " + msg + " -- Call stack is: " + Utils.GetStackTrace());
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message to be sent does not have a target silo");
return;
}
messagingTrace.OnSendMessage(msg);
if (!messageCenter.TrySendLocal(msg))
{
if (stopped)
{
logger.LogInformation((int)ErrorCode.Runtime_Error_100115, "Message was queued for sending after outbound queue was stopped: {Message}", msg);
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message was queued for sending after outbound queue was stopped");
return;
}
// check for simulation of lost messages
if (messageCenter.ShouldDrop?.Invoke(msg) == true)
{
logger.LogInformation((int)ErrorCode.Messaging_SimulatedMessageLoss, "Message blocked by test");
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message blocked by test");
return;
}
if (this.connectionManager.TryGetConnection(msg.TargetSilo, out var existingConnection))
{
existingConnection.Send(msg);
return;
}
else if (this.siloStatusOracle.IsDeadSilo(msg.TargetSilo))
{
// Do not try to establish
this.messagingTrace.OnRejectSendMessageToDeadSilo(this.messageCenter.MyAddress, msg);
this.messageCenter.SendRejection(msg, Message.RejectionTypes.Transient, "Target silo is known to be dead");
return;
}
else
{
var senderTask = this.connectionManager.GetConnection(msg.TargetSilo);
if (senderTask.IsCompletedSuccessfully)
{
var sender = senderTask.Result;
sender.Send(msg);
}
else
{
_ = SendAsync(senderTask, msg);
async Task SendAsync(ValueTask <Connection> c, Message m)
{
try
{
var sender = await c;
sender.Send(m);
}
catch (Exception exception)
{
this.messageCenter.SendRejection(m, Message.RejectionTypes.Transient, $"Exception while sending message: {exception}");
}
}
}
}
}
}
/// <summary>
/// Handles an incoming (proxied) message by rerouting it immediately and unconditionally,
/// after some header massaging.
/// </summary>
/// <param name="msg"></param>
/// <param name="receivedOnSocket"></param>
protected override void HandleMessage(Message msg, Socket receivedOnSocket)
{
// Don't process messages that have already timed out
if (msg.IsExpired)
{
msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Receive);
return;
}
if (Message.WriteMessagingTraces)
{
msg.AddTimestamp(Message.LifecycleTag.ReceiveIncoming);
}
gatewayTrafficCounter.Increment();
// Are we overloaded?
if ((MessageCenter.Metrics != null) && MessageCenter.Metrics.IsOverloaded)
{
MessagingStatisticsGroup.OnRejectedMessage(msg);
Message rejection = msg.CreateRejectionResponse(Message.RejectionTypes.GatewayTooBusy, "Shedding load");
MessageCenter.TryDeliverToProxy(rejection);
if (Log.IsVerbose)
{
Log.Verbose("Rejecting a request due to overloading: {0}", msg.ToString());
}
loadSheddingCounter.Increment();
return;
}
gateway.RecordSendingProxiedGrain(msg.SendingGrain, receivedOnSocket);
SiloAddress targetAddress = gateway.TryToReroute(msg);
msg.SendingSilo = MessageCenter.MyAddress;
if (targetAddress == null)
{
// reroute via Dispatcher
msg.RemoveHeader(Message.Header.TARGET_SILO);
msg.RemoveHeader(Message.Header.TARGET_ACTIVATION);
if (msg.TargetGrain.IsSystemTarget)
{
msg.TargetSilo = MessageCenter.MyAddress;
msg.TargetActivation = ActivationId.GetSystemActivation(msg.TargetGrain, MessageCenter.MyAddress);
}
if (Message.WriteMessagingTraces)
{
msg.AddTimestamp(Message.LifecycleTag.RerouteIncoming);
}
MessagingStatisticsGroup.OnMessageReRoute(msg);
MessageCenter.RerouteMessage(msg);
}
else
{
// send directly
msg.TargetSilo = targetAddress;
MessageCenter.SendMessage(msg);
}
}
public void SendMessage(Message msg)
{
if (msg == null)
{
throw new ArgumentNullException("msg", "Can't send a null message.");
}
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100112, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// Don't process messages that have already timed out
if (msg.IsExpired)
{
msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Send);
return;
}
if (!msg.ContainsMetadata(QUEUED_TIME_METADATA))
{
msg.SetMetadata(QUEUED_TIME_METADATA, DateTime.UtcNow);
}
// First check to see if it's really destined for a proxied client, instead of a local grain.
if (messageCenter.IsProxying && messageCenter.TryDeliverToProxy(msg))
{
return;
}
if (!msg.ContainsHeader(Message.Header.TARGET_SILO))
{
logger.Error(ErrorCode.Runtime_Error_100113, "Message does not have a target silo: " + msg + " -- Call stack is: " + (new System.Diagnostics.StackTrace()));
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message to be sent does not have a target silo");
return;
}
if (Message.WriteMessagingTraces)
{
msg.AddTimestamp(Message.LifecycleTag.EnqueueOutgoing);
}
// Shortcut messages to this silo
if (msg.TargetSilo.Equals(messageCenter.MyAddress))
{
if (logger.IsVerbose3)
{
logger.Verbose3("Message has been looped back to this silo: {0}", msg);
}
MessagingStatisticsGroup.LocalMessagesSent.Increment();
messageCenter.InboundQueue.PostMessage(msg);
}
else
{
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100115, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// check for simulation of lost messages
if (Silo.CurrentSilo.TestHook.ShouldDrop(msg))
{
logger.Info(ErrorCode.Messaging_SimulatedMessageLoss, "Message blocked by test");
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message blocked by test");
}
// Prioritize system messages
switch (msg.Category)
{
case Message.Categories.Ping:
pingSender.QueueRequest(msg);
break;
case Message.Categories.System:
systemSender.QueueRequest(msg);
break;
default:
{
int index = Math.Abs(msg.TargetSilo.GetConsistentHashCode()) % senders.Length;
senders[index].Value.QueueRequest(msg);
break;
}
}
}
}
public void SendMessage(Message msg)
{
if (msg == null)
{
throw new ArgumentNullException("msg", "Can't send a null message.");
}
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100112, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// Don't process messages that have already timed out
if (msg.IsExpired)
{
msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Send);
return;
}
if (!msg.QueuedTime.HasValue)
{
msg.QueuedTime = DateTime.UtcNow;
}
// First check to see if it's really destined for a proxied client, instead of a local grain.
if (messageCenter.IsProxying && messageCenter.TryDeliverToProxy(msg))
{
return;
}
if (msg.TargetSilo == null)
{
logger.Error(ErrorCode.Runtime_Error_100113, "Message does not have a target silo: " + msg + " -- Call stack is: " + Utils.GetStackTrace());
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message to be sent does not have a target silo");
return;
}
if (!messageCenter.TrySendLocal(msg))
{
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100115, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// check for simulation of lost messages
if (messageCenter.ShouldDrop?.Invoke(msg) == true)
{
logger.Info(ErrorCode.Messaging_SimulatedMessageLoss, "Message blocked by test");
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message blocked by test");
return;
}
var senderTask = this.connectionManager.GetConnection(msg.TargetSilo);
if (senderTask.IsCompletedSuccessfully)
{
var sender = senderTask.Result;
sender.Send(msg);
}
else
{
_ = SendAsync(senderTask, msg);
async Task SendAsync(ValueTask <Connection> c, Message m)
{
try
{
var sender = await c;
sender.Send(m);
}
catch (Exception exception)
{
this.messageCenter.SendRejection(m, Message.RejectionTypes.Transient, $"Exception while sending message: {exception}");
}
}
}
}
}
protected override void OnReceivedMessage(Message msg)
{
// See it's a Ping message, and if so, short-circuit it
if (msg.IsPing())
{
this.HandlePingMessage(msg);
return;
}
// sniff message headers for directory cache management
this.messageCenter.SniffIncomingMessage?.Invoke(msg);
// Don't process messages that have already timed out
if (msg.IsExpired)
{
this.MessagingTrace.OnDropExpiredMessage(msg, MessagingStatisticsGroup.Phase.Receive);
return;
}
// If we've stopped application message processing, then filter those out now
// Note that if we identify or add other grains that are required for proper stopping, we will need to treat them as we do the membership table grain here.
if (messageCenter.IsBlockingApplicationMessages && (msg.Category == Message.Categories.Application))
{
// We reject new requests, and drop all other messages
if (msg.Direction != Message.Directions.Request)
{
this.MessagingTrace.OnDropBlockedApplicationMessage(msg);
return;
}
MessagingStatisticsGroup.OnRejectedMessage(msg);
var rejection = this.MessageFactory.CreateRejectionResponse(msg, Message.RejectionTypes.Unrecoverable, "Silo stopping");
this.Send(rejection);
return;
}
// Make sure the message is for us. Note that some control messages may have no target
// information, so a null target silo is OK.
if (msg.TargetSilo == null || msg.TargetSilo.Matches(this.LocalSiloAddress))
{
// See if it's a message for a client we're proxying.
if (messageCenter.TryDeliverToProxy(msg))
{
return;
}
// Nope, it's for us
messageCenter.DispatchLocalMessage(msg);
return;
}
if (!msg.TargetSilo.Endpoint.Equals(this.LocalSiloAddress.Endpoint))
{
// If the message is for some other silo altogether, then we need to forward it.
if (this.Log.IsEnabled(LogLevel.Trace))
{
this.Log.Trace("Forwarding message {0} from {1} to silo {2}", msg.Id, msg.SendingSilo, msg.TargetSilo);
}
messageCenter.SendMessage(msg);
return;
}
// If the message was for this endpoint but an older epoch, then reject the message
// (if it was a request), or drop it on the floor if it was a response or one-way.
if (msg.Direction == Message.Directions.Request)
{
MessagingStatisticsGroup.OnRejectedMessage(msg);
var rejection = this.MessageFactory.CreateRejectionResponse(
msg,
Message.RejectionTypes.Transient,
$"The target silo is no longer active: target was {msg.TargetSilo.ToLongString()}, but this silo is {this.LocalSiloAddress.ToLongString()}. The rejected message is {msg}.");
// Invalidate the remote caller's activation cache entry.
if (msg.TargetAddress != null)
{
rejection.AddToCacheInvalidationHeader(msg.TargetAddress);
}
this.Send(rejection);
if (this.Log.IsEnabled(LogLevel.Debug))
{
this.Log.Debug(
"Rejecting an obsolete request; target was {0}, but this silo is {1}. The rejected message is {2}.",
msg.TargetSilo.ToLongString(),
this.LocalSiloAddress.ToLongString(),
msg);
}
}
}
public void SendMessage(Message msg)
{
if (msg == null)
{
throw new ArgumentNullException("msg", "Can't send a null message.");
}
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100112, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// Don't process messages that have already timed out
if (msg.IsExpired)
{
msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Send);
return;
}
if (!msg.QueuedTime.HasValue)
{
msg.QueuedTime = DateTime.UtcNow;
}
// First check to see if it's really destined for a proxied client, instead of a local grain.
if (messageCenter.IsProxying && messageCenter.TryDeliverToProxy(msg))
{
return;
}
if (msg.TargetSilo == null)
{
logger.Error(ErrorCode.Runtime_Error_100113, "Message does not have a target silo: " + msg + " -- Call stack is: " + Utils.GetStackTrace());
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message to be sent does not have a target silo");
return;
}
if (!messageCenter.TrySendLocal(msg))
{
if (stopped)
{
logger.Info(ErrorCode.Runtime_Error_100115, "Message was queued for sending after outbound queue was stopped: {0}", msg);
return;
}
// check for simulation of lost messages
if (messageCenter?.ShouldDrop?.Invoke(msg) == true)
{
logger.Info(ErrorCode.Messaging_SimulatedMessageLoss, "Message blocked by test");
messageCenter.SendRejection(msg, Message.RejectionTypes.Unrecoverable, "Message blocked by test");
return;
}
// Prioritize system messages
switch (msg.Category)
{
case Message.Categories.Ping:
pingSender.QueueRequest(msg);
break;
case Message.Categories.System:
systemSender.QueueRequest(msg);
break;
default:
{
int index = Math.Abs(msg.TargetSilo.GetConsistentHashCode()) % senders.Length;
senders[index].Value.QueueRequest(msg);
break;
}
}
}
}