public Bcl.IEnumerable <PeerConnection> FindRoutes(ExternalRouteLookupRequest lookupRequest)
{
var peers = new Bcl.List <PeerConnection>();
if (lookupRequest.ReceiverNodeIdentity.IsSet())
{
if (nodeToConnectionMap.TryGetValue(lookupRequest.ReceiverNodeIdentity, out var peerConnection))
{
peers.Add(peerConnection);
}
}
else
{
if (messageToNodeMap.TryGetValue(lookupRequest.Message, out var nodes))
{
if (lookupRequest.Distribution == DistributionPattern.Unicast)
{
peers.Add(nodeToConnectionMap[nodes.RoundRobinGet()]);
}
else
{
if (lookupRequest.Distribution == DistributionPattern.Broadcast)
{
foreach (var node in nodes)
{
peers.Add(nodeToConnectionMap[node]);
}
}
}
}
}
return(peers);
}
private bool SendToExactReceiver(Message message, ISocket scaleOutBackend, ExternalRouteLookupRequest lookupRequest)
{
if (Unsafe.ArraysEqual(message.ReceiverNodeIdentity, thisNodeIdentity))
{
var localDestinations = internalRoutingTable.FindRoutes(lookupRequest);
return(SendMessageLocally(localDestinations, message));
}
var remoteDestinations = externalRoutingTable.FindRoutes(lookupRequest);
return(SendMessageAway(remoteDestinations, message, scaleOutBackend));
}
private bool HandleOperationMessage(Message message, ISocket scaleOutBackend)
{
var lookupRequest = new ExternalRouteLookupRequest
{
ReceiverIdentity = new ReceiverIdentifier(message.ReceiverIdentity),
Message = new MessageIdentifier(message),
Distribution = message.Distribution,
ReceiverNodeIdentity = new ReceiverIdentifier(message.ReceiverNodeIdentity ?? IdentityExtensions.Empty)
};
var handled = message.ReceiverNodeIdentity.IsSet()
? SendToExactReceiver(message, scaleOutBackend, lookupRequest)
: SelectReceiverAndSend(message, scaleOutBackend, lookupRequest);
return(handled || ProcessUnhandledMessage(message, lookupRequest));
}
private bool ForwardMessageAway(ExternalRouteLookupRequest lookupRequest, Message message, ISocket scaleOutBackend)
{
var routes = message.Distribution == DistributionPattern.Broadcast && !MessageCameFromLocalActor(message)
? Enumerable.Empty <PeerConnection>()
: externalRoutingTable.FindRoutes(lookupRequest);
foreach (var route in routes)
{
try
{
if (!route.Connected)
{
scaleOutBackend.Connect(route.Node.Uri, waitUntilConnected: true);
route.Connected = true;
clusterServices.GetClusterHealthMonitor()
.StartPeerMonitoring(new Node(route.Node.Uri, route.Node.SocketIdentity), route.Health);
}
message.SetSocketIdentity(route.Node.SocketIdentity);
message.AddHop();
message.PushRouterAddress(scaleOutConfigurationProvider.GetScaleOutAddress());
message.SignMessage(securityProvider);
scaleOutBackend.SendMessage(message);
ForwardedToOtherNode(message);
}
catch (TimeoutException err)
{
clusterServices.GetClusterHealthMonitor()
.ScheduleConnectivityCheck(new ReceiverIdentifier(route.Node.SocketIdentity));
logger.Error(err);
}
catch (HostUnreachableException err)
{
var unregMessage = new UnregisterUnreachableNodeMessage {
ReceiverNodeIdentity = route.Node.SocketIdentity
};
TryHandleServiceMessage(Message.Create(unregMessage).As <Message>(), scaleOutBackend);
logger.Error(err);
}
}
return(routes.Any());
}
private bool HandleOperationMessage(Message message, ISocket scaleOutBackend)
{
var lookupRequest = new ExternalRouteLookupRequest
{
ReceiverIdentity = new ReceiverIdentifier(message.ReceiverIdentity),
Message = new MessageIdentifier(message),
Distribution = message.Distribution,
ReceiverNodeIdentity = new ReceiverIdentifier(message.ReceiverNodeIdentity ?? IdentityExtensions.Empty)
};
var handleMessageLocally = !message.ReceiverNodeIdentity.IsSet() ||
Unsafe.ArraysEqual(message.ReceiverNodeIdentity, thisNodeIdentity);
var handled = handleMessageLocally && HandleMessageLocally(lookupRequest, message);
if (!handled || message.Distribution == DistributionPattern.Broadcast)
{
handled = ForwardMessageAway(lookupRequest, message, scaleOutBackend) || handled;
}
return(handled || ProcessUnhandledMessage(message, lookupRequest));
}
private bool ProcessUnhandledMessage(IMessage message, ExternalRouteLookupRequest lookupRequest)
{
var messageRoute = new Cluster.MessageRoute
{
Receiver = lookupRequest.ReceiverIdentity,
Message = lookupRequest.Message
};
clusterServices.GetClusterMonitor().DiscoverMessageRoute(messageRoute);
if (MessageCameFromOtherNode(message))
{
clusterServices.GetClusterMonitor().UnregisterSelf(messageRoute.ToEnumerable());
}
var route = message.As <Message>().GetMessageRouting().FirstOrDefault();
logger.Warn($"Route not found for Message:{lookupRequest.Message} lookup by " +
$"[{nameof(lookupRequest.ReceiverNodeIdentity)}:{lookupRequest.ReceiverNodeIdentity}]-" +
$"[{nameof(lookupRequest.ReceiverIdentity)}:{lookupRequest.ReceiverIdentity}]-" +
$"[{lookupRequest.Distribution}] " +
$"Sent by:[{route?.Identity.GetAnyString()}@{route?.Uri}]");
return(true);
}
private bool SelectReceiverAndSend(Message message, ISocket scaleOutBackend, ExternalRouteLookupRequest lookupRequest)
{
var handled = false;
if (message.Distribution == DistributionPattern.Broadcast)
{
handled = SendMessageLocally(internalRoutingTable.FindRoutes(lookupRequest), message);
handled = MessageCameFromLocalActor(message) &&
SendMessageAway(externalRoutingTable.FindRoutes(lookupRequest), message, scaleOutBackend) ||
handled;
}
else
{
var localDestinations = internalRoutingTable.FindRoutes(lookupRequest);
var remoteDestinations = externalRoutingTable.FindRoutes(lookupRequest);
var local = localDestinations.FirstOrDefault()?.As <IDestination>();
var remote = remoteDestinations.FirstOrDefault()?.Node.As <IDestination>();
var destination = (local != null && remote != null)
? roundRobinDestinationList.SelectNextDestination(local, remote)
: (local ?? remote);
if (destination != null)
{
if (MessageCameFromOtherNode(message) || destination.Equals(local))
{
handled = SendMessageLocally(localDestinations, message);
}
if (!handled)
{
handled = SendMessageAway(remoteDestinations, message, scaleOutBackend);
}
}
}
return(handled);
}
