private void Test_GatewaySelection(IGatewayListProvider listProvider) { IList<Uri> gatewayUris = listProvider.GetGateways().GetResult(); Assert.IsTrue(gatewayUris.Count > 0, "Found some gateways. Data = {0}", Utils.EnumerableToString(gatewayUris)); var gatewayEndpoints = gatewayUris.Select(uri => { return new IPEndPoint(IPAddress.Parse(uri.Host), uri.Port); }).ToList(); var cfg = new ClientConfiguration { Gateways = gatewayEndpoints }; var gatewayManager = new GatewayManager(cfg, listProvider); var counts = new int[4]; for (int i = 0; i < 2300; i++) { var ip = gatewayManager.GetLiveGateway(); var addr = IPAddress.Parse(ip.Host); Assert.AreEqual(IPAddress.Loopback, addr, "Incorrect IP address returned for gateway"); Assert.IsTrue((0 < ip.Port) && (ip.Port < 5), "Incorrect IP port returned for gateway"); counts[ip.Port - 1]++; } // The following needed to be changed as the gateway manager now round-robins through the available gateways, rather than // selecting randomly based on load numbers. //Assert.IsTrue((500 < counts[0]) && (counts[0] < 1500), "Gateway selection is incorrectly skewed"); //Assert.IsTrue((500 < counts[1]) && (counts[1] < 1500), "Gateway selection is incorrectly skewed"); //Assert.IsTrue((125 < counts[2]) && (counts[2] < 375), "Gateway selection is incorrectly skewed"); //Assert.IsTrue((25 < counts[3]) && (counts[3] < 75), "Gateway selection is incorrectly skewed"); //Assert.IsTrue((287 < counts[0]) && (counts[0] < 1150), "Gateway selection is incorrectly skewed"); //Assert.IsTrue((287 < counts[1]) && (counts[1] < 1150), "Gateway selection is incorrectly skewed"); //Assert.IsTrue((287 < counts[2]) && (counts[2] < 1150), "Gateway selection is incorrectly skewed"); //Assert.IsTrue((287 < counts[3]) && (counts[3] < 1150), "Gateway selection is incorrectly skewed"); int low = 2300 / 4; int up = 2300 / 4; Assert.IsTrue((low <= counts[0]) && (counts[0] <= up), "Gateway selection is incorrectly skewed. " + counts[0]); Assert.IsTrue((low <= counts[1]) && (counts[1] <= up), "Gateway selection is incorrectly skewed. " + counts[1]); Assert.IsTrue((low <= counts[2]) && (counts[2] <= up), "Gateway selection is incorrectly skewed. " + counts[2]); Assert.IsTrue((low <= counts[3]) && (counts[3] <= up), "Gateway selection is incorrectly skewed. " + counts[3]); }
private System.Threading.Tasks.ValueTask <Connection> GetGatewayConnection(Message msg) { // If there's a specific gateway specified, use it if (msg.TargetSilo != null && gatewayManager.GetLiveGateways().Contains(msg.TargetSilo)) { var siloAddress = SiloAddress.New(msg.TargetSilo.Endpoint, 0); var connectionTask = this.connectionManager.GetConnection(siloAddress); if (connectionTask.IsCompletedSuccessfully) { return(connectionTask); } return(ConnectAsync(msg.TargetSilo, connectionTask, msg, directGatewayMessage: true)); } // For untargeted messages to system targets, and for unordered messages, pick a next connection in round robin fashion. if (msg.TargetGrain.IsSystemTarget || msg.IsUnordered) { // Get the cached list of live gateways. // Pick a next gateway name in a round robin fashion. // See if we have a live connection to it. // If Yes, use it. // If not, create a new GatewayConnection and start it. // If start fails, we will mark this connection as dead and remove it from the GetCachedLiveGatewayNames. int msgNumber = Interlocked.Increment(ref numMessages); var gatewayAddresses = gatewayManager.GetLiveGateways(); int numGateways = gatewayAddresses.Count; if (numGateways == 0) { RejectMessage(msg, "No gateways available"); logger.Warn(ErrorCode.ProxyClient_CannotSend, "Unable to send message {0}; gateway manager state is {1}", msg, gatewayManager); return(new System.Threading.Tasks.ValueTask <Connection>(default(Connection))); } var gatewayAddress = gatewayAddresses[msgNumber % numGateways]; var connectionTask = this.connectionManager.GetConnection(gatewayAddress); if (connectionTask.IsCompletedSuccessfully) { return(connectionTask); } return(ConnectAsync(gatewayAddress, connectionTask, msg, directGatewayMessage: false)); } // Otherwise, use the buckets to ensure ordering. var index = msg.TargetGrain.GetHashCode_Modulo((uint)grainBuckets.Length); // Repeated from above, at the declaration of the grainBuckets array: // Requests are bucketed by GrainID, so that all requests to a grain get routed through the same bucket. // Each bucket holds a (possibly null) weak reference to a GatewayConnection object. That connection instance is used // if the WeakReference is non-null, is alive, and points to a live gateway connection. If any of these conditions is // false, then a new gateway is selected using the gateway manager, and a new connection established if necessary. WeakReference <Connection> weakRef = grainBuckets[index]; if (weakRef != null && weakRef.TryGetTarget(out var existingConnection) && existingConnection.IsValid) { return(new System.Threading.Tasks.ValueTask <Connection>(existingConnection)); } var addr = gatewayManager.GetLiveGateway(); if (addr == null) { RejectMessage(msg, "No gateways available"); logger.Warn(ErrorCode.ProxyClient_CannotSend_NoGateway, "Unable to send message {0}; gateway manager state is {1}", msg, gatewayManager); return(new System.Threading.Tasks.ValueTask <Connection>(default(Connection))); } if (logger.IsEnabled(LogLevel.Trace)) { logger.Trace(ErrorCode.ProxyClient_NewBucketIndex, "Starting new bucket index {0} for ordered messages to grain {1}", index, msg.TargetGrain); } if (logger.IsEnabled(LogLevel.Debug)) { logger.Debug( ErrorCode.ProxyClient_CreatedGatewayToGrain, "Creating gateway to {0} for message to grain {1}, bucket {2}, grain id hash code {3}X", addr, msg.TargetGrain, index, msg.TargetGrain.GetHashCode().ToString("x")); } var gatewayConnection = this.connectionManager.GetConnection(addr); if (gatewayConnection.IsCompletedSuccessfully) { this.UpdateBucket(index, gatewayConnection.Result); return(gatewayConnection); } return(AddToBucketAsync(index, gatewayConnection, addr)); async System.Threading.Tasks.ValueTask <Connection> AddToBucketAsync( uint bucketIndex, System.Threading.Tasks.ValueTask <Connection> connectionTask, SiloAddress gatewayAddress) { try { var connection = await connectionTask.ConfigureAwait(false); this.UpdateBucket(bucketIndex, connection); return(connection); } catch { this.gatewayManager.MarkAsDead(gatewayAddress); this.UpdateBucket(bucketIndex, null); throw; } } async System.Threading.Tasks.ValueTask <Connection> ConnectAsync( SiloAddress gateway, System.Threading.Tasks.ValueTask <Connection> connectionTask, Message message, bool directGatewayMessage) { Connection result = default; try { return(result = await connectionTask); } catch (Exception exception) when(directGatewayMessage) { RejectMessage(message, string.Format("Target silo {0} is unavailable", message.TargetSilo), exception); return(null); } finally { if (result is null) { this.gatewayManager.MarkAsDead(gateway); } } } }
public void SendMessage(Message msg) { GatewayConnection gatewayConnection = null; bool startRequired = false; // If there's a specific gateway specified, use it if (msg.TargetSilo != null) { Uri addr = msg.TargetSilo.ToGatewayUri(); lock (lockable) { if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive) { gatewayConnection = new GatewayConnection(addr, this); gatewayConnections[addr] = gatewayConnection; if (logger.IsVerbose) { logger.Verbose("Creating gateway to {0} for pre-addressed message", addr); } startRequired = true; } } } // For untargeted messages to system targets, and for unordered messages, pick a next connection in round robin fashion. else if (msg.TargetGrain.IsSystemTarget || msg.IsUnordered) { // Get the cached list of live gateways. // Pick a next gateway name in a round robin fashion. // See if we have a live connection to it. // If Yes, use it. // If not, create a new GatewayConnection and start it. // If start fails, we will mark this connection as dead and remove it from the GetCachedLiveGatewayNames. lock (lockable) { int msgNumber = numMessages; numMessages = unchecked (numMessages + 1); List <Uri> gatewayNames = GatewayManager.GetLiveGateways(); int numGateways = gatewayNames.Count; if (numGateways == 0) { RejectMessage(msg, "No gateways available"); logger.Warn(ErrorCode.ProxyClient_CannotSend, "Unable to send message {0}; gateway manager state is {1}", msg, GatewayManager); return; } Uri addr = gatewayNames[msgNumber % numGateways]; if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive) { gatewayConnection = new GatewayConnection(addr, this); gatewayConnections[addr] = gatewayConnection; if (logger.IsVerbose) { logger.Verbose(ErrorCode.ProxyClient_CreatedGatewayUnordered, "Creating gateway to {0} for unordered message to grain {1}", addr, msg.TargetGrain); } startRequired = true; } // else - Fast path - we've got a live gatewayConnection to use } } // Otherwise, use the buckets to ensure ordering. else { var index = msg.TargetGrain.GetHashCode_Modulo((uint)grainBuckets.Length); lock (lockable) { // Repeated from above, at the declaration of the grainBuckets array: // Requests are bucketed by GrainID, so that all requests to a grain get routed through the same bucket. // Each bucket holds a (possibly null) weak reference to a GatewayConnection object. That connection instance is used // if the WeakReference is non-null, is alive, and points to a live gateway connection. If any of these conditions is // false, then a new gateway is selected using the gateway manager, and a new connection established if necessary. var weakRef = grainBuckets[index]; if ((weakRef != null) && weakRef.IsAlive) { gatewayConnection = weakRef.Target as GatewayConnection; } if ((gatewayConnection == null) || !gatewayConnection.IsLive) { var addr = GatewayManager.GetLiveGateway(); if (addr == null) { RejectMessage(msg, "No gateways available"); logger.Warn(ErrorCode.ProxyClient_CannotSend_NoGateway, "Unable to send message {0}; gateway manager state is {1}", msg, GatewayManager); return; } if (logger.IsVerbose2) { logger.Verbose2(ErrorCode.ProxyClient_NewBucketIndex, "Starting new bucket index {0} for ordered messages to grain {1}", index, msg.TargetGrain); } if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive) { gatewayConnection = new GatewayConnection(addr, this); gatewayConnections[addr] = gatewayConnection; if (logger.IsVerbose) { logger.Verbose(ErrorCode.ProxyClient_CreatedGatewayToGrain, "Creating gateway to {0} for message to grain {1}, bucket {2}, grain id hash code {3}X", addr, msg.TargetGrain, index, msg.TargetGrain.GetHashCode().ToString("x")); } startRequired = true; } grainBuckets[index] = new WeakReference(gatewayConnection); } } } if (startRequired) { gatewayConnection.Start(); if (gatewayConnection.IsLive) { // Register existing client observers with the new gateway List <GrainId> localObjects; lock (lockable) { localObjects = registeredLocalObjects.ToList(); } var registrar = GetRegistrar(gatewayConnection.Silo); foreach (var obj in localObjects) { registrar.RegisterClientObserver(obj, ClientId).Ignore(); } } else { // if failed to start Gateway connection (failed to connect), try sending this msg to another Gateway. RejectOrResend(msg); return; } } try { gatewayConnection.QueueRequest(msg); if (logger.IsVerbose2) { logger.Verbose2(ErrorCode.ProxyClient_QueueRequest, "Sending message {0} via gateway {1}", msg, gatewayConnection.Address); } } catch (InvalidOperationException) { // This exception can be thrown if the gateway connection we selected was closed since we checked (i.e., we lost the race) // If this happens, we reject if the message is targeted to a specific silo, or try again if not RejectOrResend(msg); } }