private void Test_GatewaySelection(IGatewayListProvider listProvider)
{
IList<Uri> gatewayUris = listProvider.GetGateways().GetResult();
Assert.IsTrue(gatewayUris.Count > 0, "Found some gateways. Data = {0}", Utils.EnumerableToString(gatewayUris));
var gatewayEndpoints = gatewayUris.Select(uri =>
{
return new IPEndPoint(IPAddress.Parse(uri.Host), uri.Port);
}).ToList();
var cfg = new ClientConfiguration
{
Gateways = gatewayEndpoints
};
var gatewayManager = new GatewayManager(cfg, listProvider);
var counts = new int[4];
for (int i = 0; i < 2300; i++)
{
var ip = gatewayManager.GetLiveGateway();
var addr = IPAddress.Parse(ip.Host);
Assert.AreEqual(IPAddress.Loopback, addr, "Incorrect IP address returned for gateway");
Assert.IsTrue((0 < ip.Port) && (ip.Port < 5), "Incorrect IP port returned for gateway");
counts[ip.Port - 1]++;
}
// The following needed to be changed as the gateway manager now round-robins through the available gateways, rather than
// selecting randomly based on load numbers.
//Assert.IsTrue((500 < counts[0]) && (counts[0] < 1500), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((500 < counts[1]) && (counts[1] < 1500), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((125 < counts[2]) && (counts[2] < 375), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((25 < counts[3]) && (counts[3] < 75), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((287 < counts[0]) && (counts[0] < 1150), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((287 < counts[1]) && (counts[1] < 1150), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((287 < counts[2]) && (counts[2] < 1150), "Gateway selection is incorrectly skewed");
//Assert.IsTrue((287 < counts[3]) && (counts[3] < 1150), "Gateway selection is incorrectly skewed");
int low = 2300 / 4;
int up = 2300 / 4;
Assert.IsTrue((low <= counts[0]) && (counts[0] <= up), "Gateway selection is incorrectly skewed. " + counts[0]);
Assert.IsTrue((low <= counts[1]) && (counts[1] <= up), "Gateway selection is incorrectly skewed. " + counts[1]);
Assert.IsTrue((low <= counts[2]) && (counts[2] <= up), "Gateway selection is incorrectly skewed. " + counts[2]);
Assert.IsTrue((low <= counts[3]) && (counts[3] <= up), "Gateway selection is incorrectly skewed. " + counts[3]);
}
private System.Threading.Tasks.ValueTask <Connection> GetGatewayConnection(Message msg)
{
// If there's a specific gateway specified, use it
if (msg.TargetSilo != null && gatewayManager.GetLiveGateways().Contains(msg.TargetSilo))
{
var siloAddress = SiloAddress.New(msg.TargetSilo.Endpoint, 0);
var connectionTask = this.connectionManager.GetConnection(siloAddress);
if (connectionTask.IsCompletedSuccessfully)
{
return(connectionTask);
}
return(ConnectAsync(msg.TargetSilo, connectionTask, msg, directGatewayMessage: true));
}
// For untargeted messages to system targets, and for unordered messages, pick a next connection in round robin fashion.
if (msg.TargetGrain.IsSystemTarget || msg.IsUnordered)
{
// Get the cached list of live gateways.
// Pick a next gateway name in a round robin fashion.
// See if we have a live connection to it.
// If Yes, use it.
// If not, create a new GatewayConnection and start it.
// If start fails, we will mark this connection as dead and remove it from the GetCachedLiveGatewayNames.
int msgNumber = Interlocked.Increment(ref numMessages);
var gatewayAddresses = gatewayManager.GetLiveGateways();
int numGateways = gatewayAddresses.Count;
if (numGateways == 0)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend, "Unable to send message {0}; gateway manager state is {1}", msg, gatewayManager);
return(new System.Threading.Tasks.ValueTask <Connection>(default(Connection)));
}
var gatewayAddress = gatewayAddresses[msgNumber % numGateways];
var connectionTask = this.connectionManager.GetConnection(gatewayAddress);
if (connectionTask.IsCompletedSuccessfully)
{
return(connectionTask);
}
return(ConnectAsync(gatewayAddress, connectionTask, msg, directGatewayMessage: false));
}
// Otherwise, use the buckets to ensure ordering.
var index = msg.TargetGrain.GetHashCode_Modulo((uint)grainBuckets.Length);
// Repeated from above, at the declaration of the grainBuckets array:
// Requests are bucketed by GrainID, so that all requests to a grain get routed through the same bucket.
// Each bucket holds a (possibly null) weak reference to a GatewayConnection object. That connection instance is used
// if the WeakReference is non-null, is alive, and points to a live gateway connection. If any of these conditions is
// false, then a new gateway is selected using the gateway manager, and a new connection established if necessary.
WeakReference <Connection> weakRef = grainBuckets[index];
if (weakRef != null && weakRef.TryGetTarget(out var existingConnection) && existingConnection.IsValid)
{
return(new System.Threading.Tasks.ValueTask <Connection>(existingConnection));
}
var addr = gatewayManager.GetLiveGateway();
if (addr == null)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend_NoGateway, "Unable to send message {0}; gateway manager state is {1}", msg, gatewayManager);
return(new System.Threading.Tasks.ValueTask <Connection>(default(Connection)));
}
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace(ErrorCode.ProxyClient_NewBucketIndex, "Starting new bucket index {0} for ordered messages to grain {1}", index, msg.TargetGrain);
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(
ErrorCode.ProxyClient_CreatedGatewayToGrain,
"Creating gateway to {0} for message to grain {1}, bucket {2}, grain id hash code {3}X",
addr,
msg.TargetGrain,
index,
msg.TargetGrain.GetHashCode().ToString("x"));
}
var gatewayConnection = this.connectionManager.GetConnection(addr);
if (gatewayConnection.IsCompletedSuccessfully)
{
this.UpdateBucket(index, gatewayConnection.Result);
return(gatewayConnection);
}
return(AddToBucketAsync(index, gatewayConnection, addr));
async System.Threading.Tasks.ValueTask <Connection> AddToBucketAsync(
uint bucketIndex,
System.Threading.Tasks.ValueTask <Connection> connectionTask,
SiloAddress gatewayAddress)
{
try
{
var connection = await connectionTask.ConfigureAwait(false);
this.UpdateBucket(bucketIndex, connection);
return(connection);
}
catch
{
this.gatewayManager.MarkAsDead(gatewayAddress);
this.UpdateBucket(bucketIndex, null);
throw;
}
}
async System.Threading.Tasks.ValueTask <Connection> ConnectAsync(
SiloAddress gateway,
System.Threading.Tasks.ValueTask <Connection> connectionTask,
Message message,
bool directGatewayMessage)
{
Connection result = default;
try
{
return(result = await connectionTask);
}
catch (Exception exception) when(directGatewayMessage)
{
RejectMessage(message, string.Format("Target silo {0} is unavailable", message.TargetSilo), exception);
return(null);
}
finally
{
if (result is null)
{
this.gatewayManager.MarkAsDead(gateway);
}
}
}
}
public void SendMessage(Message msg)
{
GatewayConnection gatewayConnection = null;
bool startRequired = false;
// If there's a specific gateway specified, use it
if (msg.TargetSilo != null)
{
Uri addr = msg.TargetSilo.ToGatewayUri();
lock (lockable)
{
if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive)
{
gatewayConnection = new GatewayConnection(addr, this);
gatewayConnections[addr] = gatewayConnection;
if (logger.IsVerbose)
{
logger.Verbose("Creating gateway to {0} for pre-addressed message", addr);
}
startRequired = true;
}
}
}
// For untargeted messages to system targets, and for unordered messages, pick a next connection in round robin fashion.
else if (msg.TargetGrain.IsSystemTarget || msg.IsUnordered)
{
// Get the cached list of live gateways.
// Pick a next gateway name in a round robin fashion.
// See if we have a live connection to it.
// If Yes, use it.
// If not, create a new GatewayConnection and start it.
// If start fails, we will mark this connection as dead and remove it from the GetCachedLiveGatewayNames.
lock (lockable)
{
int msgNumber = numMessages;
numMessages = unchecked (numMessages + 1);
List <Uri> gatewayNames = GatewayManager.GetLiveGateways();
int numGateways = gatewayNames.Count;
if (numGateways == 0)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend, "Unable to send message {0}; gateway manager state is {1}", msg, GatewayManager);
return;
}
Uri addr = gatewayNames[msgNumber % numGateways];
if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive)
{
gatewayConnection = new GatewayConnection(addr, this);
gatewayConnections[addr] = gatewayConnection;
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.ProxyClient_CreatedGatewayUnordered, "Creating gateway to {0} for unordered message to grain {1}", addr, msg.TargetGrain);
}
startRequired = true;
}
// else - Fast path - we've got a live gatewayConnection to use
}
}
// Otherwise, use the buckets to ensure ordering.
else
{
var index = msg.TargetGrain.GetHashCode_Modulo((uint)grainBuckets.Length);
lock (lockable)
{
// Repeated from above, at the declaration of the grainBuckets array:
// Requests are bucketed by GrainID, so that all requests to a grain get routed through the same bucket.
// Each bucket holds a (possibly null) weak reference to a GatewayConnection object. That connection instance is used
// if the WeakReference is non-null, is alive, and points to a live gateway connection. If any of these conditions is
// false, then a new gateway is selected using the gateway manager, and a new connection established if necessary.
var weakRef = grainBuckets[index];
if ((weakRef != null) && weakRef.IsAlive)
{
gatewayConnection = weakRef.Target as GatewayConnection;
}
if ((gatewayConnection == null) || !gatewayConnection.IsLive)
{
var addr = GatewayManager.GetLiveGateway();
if (addr == null)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend_NoGateway, "Unable to send message {0}; gateway manager state is {1}", msg, GatewayManager);
return;
}
if (logger.IsVerbose2)
{
logger.Verbose2(ErrorCode.ProxyClient_NewBucketIndex, "Starting new bucket index {0} for ordered messages to grain {1}", index, msg.TargetGrain);
}
if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive)
{
gatewayConnection = new GatewayConnection(addr, this);
gatewayConnections[addr] = gatewayConnection;
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.ProxyClient_CreatedGatewayToGrain, "Creating gateway to {0} for message to grain {1}, bucket {2}, grain id hash code {3}X", addr, msg.TargetGrain, index,
msg.TargetGrain.GetHashCode().ToString("x"));
}
startRequired = true;
}
grainBuckets[index] = new WeakReference(gatewayConnection);
}
}
}
if (startRequired)
{
gatewayConnection.Start();
if (gatewayConnection.IsLive)
{
// Register existing client observers with the new gateway
List <GrainId> localObjects;
lock (lockable)
{
localObjects = registeredLocalObjects.ToList();
}
var registrar = GetRegistrar(gatewayConnection.Silo);
foreach (var obj in localObjects)
{
registrar.RegisterClientObserver(obj, ClientId).Ignore();
}
}
else
{
// if failed to start Gateway connection (failed to connect), try sending this msg to another Gateway.
RejectOrResend(msg);
return;
}
}
try
{
gatewayConnection.QueueRequest(msg);
if (logger.IsVerbose2)
{
logger.Verbose2(ErrorCode.ProxyClient_QueueRequest, "Sending message {0} via gateway {1}", msg, gatewayConnection.Address);
}
}
catch (InvalidOperationException)
{
// This exception can be thrown if the gateway connection we selected was closed since we checked (i.e., we lost the race)
// If this happens, we reject if the message is targeted to a specific silo, or try again if not
RejectOrResend(msg);
}
}
