private async Task EstablishInitialConnection(CancellationToken cancellationToken)
{
var cancellationTask = cancellationToken.WhenCancelled();
var liveGateways = gatewayManager.GetLiveGateways();
if (liveGateways.Count == 0)
{
throw new ConnectionFailedException("There are no available gateways");
}
var pendingTasks = new List <Task>(liveGateways.Count + 1);
pendingTasks.Add(cancellationTask);
foreach (var gateway in liveGateways)
{
pendingTasks.Add(connectionManager.GetConnection(gateway).AsTask());
}
try
{
// There will always be one task to represent cancellation.
while (pendingTasks.Count > 1)
{
var completedTask = await Task.WhenAny(pendingTasks);
pendingTasks.Remove(completedTask);
cancellationToken.ThrowIfCancellationRequested();
// If at least one gateway connection has been established, break out of the loop and continue startup.
if (completedTask.IsCompletedSuccessfully)
{
break;
}
// If there are no more gateways, observe the most recent exception and bail out.
if (pendingTasks.Count == 1)
{
await completedTask;
}
}
}
catch (Exception exception)
{
throw new ConnectionFailedException(
$"Unable to connect to any of the {liveGateways.Count} available gateways.",
exception);
}
}
private System.Threading.Tasks.ValueTask <Connection> GetGatewayConnection(Message msg)
{
// If there's a specific gateway specified, use it
if (msg.TargetSilo != null && gatewayManager.GetLiveGateways().Contains(msg.TargetSilo))
{
var siloAddress = SiloAddress.New(msg.TargetSilo.Endpoint, 0);
var connectionTask = this.connectionManager.GetConnection(siloAddress);
if (connectionTask.IsCompletedSuccessfully)
{
return(connectionTask);
}
return(ConnectAsync(msg.TargetSilo, connectionTask, msg, directGatewayMessage: true));
}
// For untargeted messages to system targets, and for unordered messages, pick a next connection in round robin fashion.
if (msg.TargetGrain.IsSystemTarget || msg.IsUnordered)
{
// Get the cached list of live gateways.
// Pick a next gateway name in a round robin fashion.
// See if we have a live connection to it.
// If Yes, use it.
// If not, create a new GatewayConnection and start it.
// If start fails, we will mark this connection as dead and remove it from the GetCachedLiveGatewayNames.
int msgNumber = Interlocked.Increment(ref numMessages);
var gatewayAddresses = gatewayManager.GetLiveGateways();
int numGateways = gatewayAddresses.Count;
if (numGateways == 0)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend, "Unable to send message {0}; gateway manager state is {1}", msg, gatewayManager);
return(new System.Threading.Tasks.ValueTask <Connection>(default(Connection)));
}
var gatewayAddress = gatewayAddresses[msgNumber % numGateways];
var connectionTask = this.connectionManager.GetConnection(gatewayAddress);
if (connectionTask.IsCompletedSuccessfully)
{
return(connectionTask);
}
return(ConnectAsync(gatewayAddress, connectionTask, msg, directGatewayMessage: false));
}
// Otherwise, use the buckets to ensure ordering.
var index = msg.TargetGrain.GetHashCode_Modulo((uint)grainBuckets.Length);
// Repeated from above, at the declaration of the grainBuckets array:
// Requests are bucketed by GrainID, so that all requests to a grain get routed through the same bucket.
// Each bucket holds a (possibly null) weak reference to a GatewayConnection object. That connection instance is used
// if the WeakReference is non-null, is alive, and points to a live gateway connection. If any of these conditions is
// false, then a new gateway is selected using the gateway manager, and a new connection established if necessary.
WeakReference <Connection> weakRef = grainBuckets[index];
if (weakRef != null && weakRef.TryGetTarget(out var existingConnection) && existingConnection.IsValid)
{
return(new System.Threading.Tasks.ValueTask <Connection>(existingConnection));
}
var addr = gatewayManager.GetLiveGateway();
if (addr == null)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend_NoGateway, "Unable to send message {0}; gateway manager state is {1}", msg, gatewayManager);
return(new System.Threading.Tasks.ValueTask <Connection>(default(Connection)));
}
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace(ErrorCode.ProxyClient_NewBucketIndex, "Starting new bucket index {0} for ordered messages to grain {1}", index, msg.TargetGrain);
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(
ErrorCode.ProxyClient_CreatedGatewayToGrain,
"Creating gateway to {0} for message to grain {1}, bucket {2}, grain id hash code {3}X",
addr,
msg.TargetGrain,
index,
msg.TargetGrain.GetHashCode().ToString("x"));
}
var gatewayConnection = this.connectionManager.GetConnection(addr);
if (gatewayConnection.IsCompletedSuccessfully)
{
this.UpdateBucket(index, gatewayConnection.Result);
return(gatewayConnection);
}
return(AddToBucketAsync(index, gatewayConnection, addr));
async System.Threading.Tasks.ValueTask <Connection> AddToBucketAsync(
uint bucketIndex,
System.Threading.Tasks.ValueTask <Connection> connectionTask,
SiloAddress gatewayAddress)
{
try
{
var connection = await connectionTask.ConfigureAwait(false);
this.UpdateBucket(bucketIndex, connection);
return(connection);
}
catch
{
this.gatewayManager.MarkAsDead(gatewayAddress);
this.UpdateBucket(bucketIndex, null);
throw;
}
}
async System.Threading.Tasks.ValueTask <Connection> ConnectAsync(
SiloAddress gateway,
System.Threading.Tasks.ValueTask <Connection> connectionTask,
Message message,
bool directGatewayMessage)
{
Connection result = default;
try
{
return(result = await connectionTask);
}
catch (Exception exception) when(directGatewayMessage)
{
RejectMessage(message, string.Format("Target silo {0} is unavailable", message.TargetSilo), exception);
return(null);
}
finally
{
if (result is null)
{
this.gatewayManager.MarkAsDead(gateway);
}
}
}
}
public void SendMessage(Message msg)
{
GatewayConnection gatewayConnection = null;
bool startRequired = false;
// If there's a specific gateway specified, use it
if (msg.TargetSilo != null)
{
Uri addr = msg.TargetSilo.ToGatewayUri();
lock (lockable)
{
if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive)
{
gatewayConnection = new GatewayConnection(addr, this);
gatewayConnections[addr] = gatewayConnection;
if (logger.IsVerbose)
{
logger.Verbose("Creating gateway to {0} for pre-addressed message", addr);
}
startRequired = true;
}
}
}
// For untargeted messages to system targets, and for unordered messages, pick a next connection in round robin fashion.
else if (msg.TargetGrain.IsSystemTarget || msg.IsUnordered)
{
// Get the cached list of live gateways.
// Pick a next gateway name in a round robin fashion.
// See if we have a live connection to it.
// If Yes, use it.
// If not, create a new GatewayConnection and start it.
// If start fails, we will mark this connection as dead and remove it from the GetCachedLiveGatewayNames.
lock (lockable)
{
int msgNumber = numMessages;
numMessages = unchecked (numMessages + 1);
List <Uri> gatewayNames = GatewayManager.GetLiveGateways();
int numGateways = gatewayNames.Count;
if (numGateways == 0)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend, "Unable to send message {0}; gateway manager state is {1}", msg, GatewayManager);
return;
}
Uri addr = gatewayNames[msgNumber % numGateways];
if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive)
{
gatewayConnection = new GatewayConnection(addr, this);
gatewayConnections[addr] = gatewayConnection;
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.ProxyClient_CreatedGatewayUnordered, "Creating gateway to {0} for unordered message to grain {1}", addr, msg.TargetGrain);
}
startRequired = true;
}
// else - Fast path - we've got a live gatewayConnection to use
}
}
// Otherwise, use the buckets to ensure ordering.
else
{
var index = msg.TargetGrain.GetHashCode_Modulo((uint)grainBuckets.Length);
lock (lockable)
{
// Repeated from above, at the declaration of the grainBuckets array:
// Requests are bucketed by GrainID, so that all requests to a grain get routed through the same bucket.
// Each bucket holds a (possibly null) weak reference to a GatewayConnection object. That connection instance is used
// if the WeakReference is non-null, is alive, and points to a live gateway connection. If any of these conditions is
// false, then a new gateway is selected using the gateway manager, and a new connection established if necessary.
var weakRef = grainBuckets[index];
if ((weakRef != null) && weakRef.IsAlive)
{
gatewayConnection = weakRef.Target as GatewayConnection;
}
if ((gatewayConnection == null) || !gatewayConnection.IsLive)
{
var addr = GatewayManager.GetLiveGateway();
if (addr == null)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend_NoGateway, "Unable to send message {0}; gateway manager state is {1}", msg, GatewayManager);
return;
}
if (logger.IsVerbose2)
{
logger.Verbose2(ErrorCode.ProxyClient_NewBucketIndex, "Starting new bucket index {0} for ordered messages to grain {1}", index, msg.TargetGrain);
}
if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive)
{
gatewayConnection = new GatewayConnection(addr, this);
gatewayConnections[addr] = gatewayConnection;
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.ProxyClient_CreatedGatewayToGrain, "Creating gateway to {0} for message to grain {1}, bucket {2}, grain id hash code {3}X", addr, msg.TargetGrain, index,
msg.TargetGrain.GetHashCode().ToString("x"));
}
startRequired = true;
}
grainBuckets[index] = new WeakReference(gatewayConnection);
}
}
}
if (startRequired)
{
gatewayConnection.Start();
if (gatewayConnection.IsLive)
{
// Register existing client observers with the new gateway
List <GrainId> localObjects;
lock (lockable)
{
localObjects = registeredLocalObjects.ToList();
}
var registrar = GetRegistrar(gatewayConnection.Silo);
foreach (var obj in localObjects)
{
registrar.RegisterClientObserver(obj, ClientId).Ignore();
}
}
else
{
// if failed to start Gateway connection (failed to connect), try sending this msg to another Gateway.
RejectOrResend(msg);
return;
}
}
try
{
gatewayConnection.QueueRequest(msg);
if (logger.IsVerbose2)
{
logger.Verbose2(ErrorCode.ProxyClient_QueueRequest, "Sending message {0} via gateway {1}", msg, gatewayConnection.Address);
}
}
catch (InvalidOperationException)
{
// This exception can be thrown if the gateway connection we selected was closed since we checked (i.e., we lost the race)
// If this happens, we reject if the message is targeted to a specific silo, or try again if not
RejectOrResend(msg);
}
}
