/// <summary>
/// Gets all nodes that make up the cluster
/// </summary>
/// <param name="seed"> The reference. </param>
/// <param name="logger"> logger used to log progress </param>
/// <param name="token"> The token. </param>
/// <returns> </returns>
/// <exception cref="CqlException">
/// Could not detect datacenter or rack information from the reference specified in the
/// config section!
/// </exception>
private async Task GetClusterInfoAsync(Node seed, Logger logger, CancellationToken token)
{
Connection c;
using(logger.ThreadBinding())
{
//get a connection
if(seed != null && seed.IsUp)
c = seed.GetOrCreateConnection();
else if(_maintenanceConnection != null && _maintenanceConnection.IsAvailable)
{
c = _maintenanceConnection;
seed = c.Node;
}
else
{
c = _connectionStrategy.GetOrCreateConnection(ConnectionScope.Infrastructure, null);
seed = c.Node;
}
}
//get local information
string partitioner;
using(var result = await ExecQuery(c,
"select cluster_name, cql_version, release_version, partitioner, data_center, rack, tokens from system.local",
logger,
token
).AutoConfigureAwait())
{
if(! await result.ReadAsyncInternal(token).AutoConfigureAwait())
throw new CqlException("Could not fetch configuration data from seed");
_name = result.GetString(0);
_cqlVersion = result.GetString(1);
_release = result.GetString(2);
partitioner = result.GetString(3);
_dataCenter = seed.DataCenter = result.GetString(4);
_rack = seed.Rack = result.GetString(5);
seed.Tokens = result.GetSet<string>(6) ?? new HashSet<string>();
}
logger.LogInfo(
"Reconfigured cluster {0}: based on Cassandra Release {1}, supporting CqlVersion {2}, using partitioner '{3}'",
_name, _release, _cqlVersion, partitioner);
//create list of nodes that make up the cluster, and add the seed
var found = new List<Node> {seed};
//get the peers
using(
var result =
await
ExecQuery(c,
"select peer, rpc_address, data_center, rack, tokens, release_version from system.peers",
logger,
token).AutoConfigureAwait())
{
//iterate over the peers
while(await result.ReadAsyncInternal(token).AutoConfigureAwait())
{
var newNode = GetNodeFromDataReader(result, logger);
//add it if it is in scope
if(InDiscoveryScope(seed, newNode, _config.DiscoveryScope))
found.Add(newNode);
}
}
//set the new Ring of nodes
_nodes.Update(found, partitioner, logger);
//check if all tokens are received
if(_nodes.Any(n => n.Tokens.Count == 0))
{
//wait and retry the fetch later...
Scheduler.RunOnThreadPool(async () =>
{
try
{
logger.LogInfo("Cluster info incomplete scheduling new retrieval in 1 minute");
await Task.Delay(TimeSpan.FromMinutes(1)).AutoConfigureAwait();
await GetClusterInfoAsync(null, logger, CancellationToken.None).AutoConfigureAwait();
}
catch(Exception ex)
{
logger.LogCritical("Critical error occured while updating cluster info: {0}", ex);
}
});
}
}
/// <summary>
/// Opens the cluster for queries. Contains actual implementation and will be called only once per cluster
/// </summary>
/// <returns></returns>
/// <exception cref="CqlException">Cannot construct ring from provided seeds!</exception>
private async Task OpenAsyncInternal(Logger logger)
{
logger.LogInfo("Opening Cluster with parameters: {0}", _config.ToString());
//try to connect to the seeds in turn
foreach (IPAddress seedAddress in _config.NodeAddresses)
{
try
{
var seed = new Node(seedAddress, this);
_nodes = await DiscoverNodesAsync(seed, logger).ConfigureAwait(false);
}
catch (Exception ex)
{
//seed not reachable, try next
logger.LogWarning("Could not discover nodes via seed {0}: {1}", seedAddress, ex);
}
}
if (_nodes == null)
{
var ex = new CqlException("Cannot construct ring from provided seeds!");
logger.LogCritical("Unable to setup Cluster based on given configuration: {0}", ex);
throw ex;
}
logger.LogInfo("Nodes detected: " + string.Join(", ", _nodes.Select(n => n.Address)));
//setup cluster connection strategy
switch (_config.ConnectionStrategy)
{
case ConnectionStrategy.Balanced:
_connectionSelector = new BalancedConnectionStrategy(_nodes, _config);
break;
case ConnectionStrategy.Random:
_connectionSelector = new RandomConnectionStrategy(_nodes, _config);
break;
case ConnectionStrategy.Exclusive:
_connectionSelector = new ExclusiveConnectionStrategy(_nodes, _config);
break;
case ConnectionStrategy.PartitionAware:
_connectionSelector = new PartitionAwareConnectionStrategy(_nodes, _config);
if (_config.DiscoveryScope != DiscoveryScope.Cluster || _config.DiscoveryScope != DiscoveryScope.DataCenter)
logger.LogWarning("PartitionAware connection strategy performs best if DiscoveryScope is set to cluster or datacenter");
break;
}
//setup throttle
int concurrent = _config.MaxConcurrentQueries <= 0
? _nodes.Count * _config.MaxConnectionsPerNode * 256
: _config.MaxConcurrentQueries;
logger.LogInfo("Cluster is configured to allow {0} parallel queries", concurrent);
_throttle = new SemaphoreSlim(concurrent, concurrent);
//setup prepared query cache
_prepareResultCache = new ConcurrentDictionary<string, ConcurrentDictionary<IPAddress, ResultFrame>>();
//setup maintenance connection
SetupMaintenanceConnection(logger);
}
/// <summary>
/// Opens the cluster for queries. Contains actual implementation and will be called only once per cluster
/// </summary>
/// <returns> </returns>
/// <exception cref="CqlException">Cannot construct ring from provided seeds!</exception>
private async Task OpenAsyncInternal(Logger logger, CancellationToken token)
{
logger.LogInfo("Opening Cluster with parameters: {0}", _config.ToString());
//initialize the ring
_nodes = new Ring();
//retry a few times to deal with bad network conditions
for(int attempt = 0; attempt <= _config.MaxQueryRetries && _nodes.Count == 0; attempt++)
{
//try to connect to the seeds in turn
foreach(IPAddress seedAddress in _config.ServerAddresses)
{
try
{
var seed = new Node(seedAddress, this);
await GetClusterInfoAsync(seed, logger, token).AutoConfigureAwait();
//break from the loop as it seems we are done
if(_nodes.Count > 0)
break;
}
catch(OperationCanceledException)
{
logger.LogWarning("Opening connection to cluster was cancelled");
throw;
}
catch(ProtocolException pex)
{
logger.LogWarning("Could not open cluster via {0}: {1}", seedAddress, pex.Message);
//node is not available, or starting up, try next, otherwise throw error
if(pex.Code != ErrorCode.Overloaded && pex.Code != ErrorCode.IsBootstrapping)
throw;
}
catch(SocketException ex)
{
//seed not reachable, try next
logger.LogWarning("Could not open TCP connection to seed {0}: {1}", seedAddress, ex.Message);
}
catch(IOException ex)
{
//seed not reachable, try next
logger.LogWarning("Could not discover nodes via seed {0}: {1}", seedAddress, ex);
}
}
}
//check if not disposed while opening
if(_disposed)
{
foreach(var node in _nodes)
node.Dispose();
throw new ObjectDisposedException("Cluster", "Cluster was disposed while opening");
}
//check if we found any nodes
if(_nodes.Count == 0)
{
var ex = new CqlException("Unable to connect to the cluster as none of the provided seeds is reachable.");
logger.LogCritical("Unable to setup Cluster based on given configuration: {0}", ex);
throw ex;
}
//setup cluster connection strategy
switch(_config.ConnectionStrategy)
{
case CqlSharp.ConnectionStrategy.Balanced:
_connectionStrategy = new BalancedConnectionStrategy(_nodes, _config);
break;
case CqlSharp.ConnectionStrategy.Random:
_connectionStrategy = new RandomConnectionStrategy(_nodes, _config);
break;
case CqlSharp.ConnectionStrategy.Exclusive:
_connectionStrategy = new ExclusiveConnectionStrategy(_nodes, _config);
break;
case CqlSharp.ConnectionStrategy.PartitionAware:
_connectionStrategy = new PartitionAwareConnectionStrategy(_nodes, _config);
if(_config.DiscoveryScope != DiscoveryScope.Cluster &&
_config.DiscoveryScope != DiscoveryScope.DataCenter)
{
logger.LogWarning(
"PartitionAware connection strategy performs best if DiscoveryScope is set to cluster or datacenter");
}
break;
}
//setup prepared query cache
PreparedQueryCache = new ConcurrentDictionary<string, ResultFrame>();
//setup maintenance connection
Scheduler.RunOnThreadPool(() => SetupMaintenanceConnection(logger));
}