/// <summary>
/// Opens the cluster for queries. Contains actual implementation and will be called only once per cluster
/// </summary>
/// <returns></returns>
/// <exception cref="CqlException">Cannot construct ring from provided seeds!</exception>
private async Task OpenAsyncInternal(Logger logger)
{
logger.LogInfo("Opening Cluster with parameters: {0}", _config.ToString());
//try to connect to the seeds in turn
foreach (IPAddress seedAddress in _config.NodeAddresses)
{
try
{
var seed = new Node(seedAddress, this);
_nodes = await DiscoverNodesAsync(seed, logger).ConfigureAwait(false);
}
catch (Exception ex)
{
//seed not reachable, try next
logger.LogWarning("Could not discover nodes via seed {0}: {1}", seedAddress, ex);
}
}
if (_nodes == null)
{
var ex = new CqlException("Cannot construct ring from provided seeds!");
logger.LogCritical("Unable to setup Cluster based on given configuration: {0}", ex);
throw ex;
}
logger.LogInfo("Nodes detected: " + string.Join(", ", _nodes.Select(n => n.Address)));
//setup cluster connection strategy
switch (_config.ConnectionStrategy)
{
case ConnectionStrategy.Balanced:
_connectionSelector = new BalancedConnectionStrategy(_nodes, _config);
break;
case ConnectionStrategy.Random:
_connectionSelector = new RandomConnectionStrategy(_nodes, _config);
break;
case ConnectionStrategy.Exclusive:
_connectionSelector = new ExclusiveConnectionStrategy(_nodes, _config);
break;
case ConnectionStrategy.PartitionAware:
_connectionSelector = new PartitionAwareConnectionStrategy(_nodes, _config);
if (_config.DiscoveryScope != DiscoveryScope.Cluster || _config.DiscoveryScope != DiscoveryScope.DataCenter)
logger.LogWarning("PartitionAware connection strategy performs best if DiscoveryScope is set to cluster or datacenter");
break;
}
//setup throttle
int concurrent = _config.MaxConcurrentQueries <= 0
? _nodes.Count * _config.MaxConnectionsPerNode * 256
: _config.MaxConcurrentQueries;
logger.LogInfo("Cluster is configured to allow {0} parallel queries", concurrent);
_throttle = new SemaphoreSlim(concurrent, concurrent);
//setup prepared query cache
_prepareResultCache = new ConcurrentDictionary<string, ConcurrentDictionary<IPAddress, ResultFrame>>();
//setup maintenance connection
SetupMaintenanceConnection(logger);
}
/// <summary>
/// Setups the maintenance channel.
/// </summary>
private async void SetupMaintenanceConnection(Logger logger)
{
try
{
if (_maintenanceConnection == null || !_maintenanceConnection.IsConnected)
{
//setup maintenance connection
logger.LogVerbose("Creating new maintenance connection");
//pick a random node from the list
var strategy = new RandomConnectionStrategy(_nodes, _config);
//get or create a connection
var connection = strategy.GetOrCreateConnection(null);
//allow this connection to be used by others as well
_connectionSelector.ReturnConnection(connection);
//setup event handlers
connection.OnConnectionChange += (src, ev) => SetupMaintenanceConnection(logger);
connection.OnClusterChange += OnClusterChange;
//store the new connection
_maintenanceConnection = connection;
//register for events
await connection.RegisterForClusterChangesAsync(logger).ConfigureAwait(false);
logger.LogInfo("Registered for cluster changes using {0}", connection);
}
//all seems right, we're done
return;
}
catch (Exception ex)
{
logger.LogWarning("Failed to setup maintenance connection: {0}", ex);
//temporary disconnect or registration failed, reset maintenance connection
_maintenanceConnection = null;
}
//wait a moment, try again
logger.LogVerbose("Waiting 2secs before retrying setup maintenance connection");
await Task.Delay(2000).ConfigureAwait(false);
SetupMaintenanceConnection(logger);
}
/// <summary>
/// Gets all nodes that make up the cluster
/// </summary>
/// <param name="seed">The reference.</param>
/// <param name="logger">logger used to log progress</param>
/// <returns></returns>
/// <exception cref="CqlException">Could not detect datacenter or rack information from the reference specified in the config section!</exception>
private async Task<Ring> DiscoverNodesAsync(Node seed, Logger logger)
{
Connection c;
using (logger.ThreadBinding())
{
//get a connection
c = seed.GetOrCreateConnection(null);
}
//get partitioner
string partitioner;
using (var result = await ExecQuery(c, "select partitioner from system.local", logger).ConfigureAwait(false))
{
if (!await result.ReadAsync().ConfigureAwait(false))
throw new CqlException("Could not detect the cluster partitioner");
partitioner = (string)result[0];
}
logger.LogInfo("Partitioner in use: {0}", partitioner);
//get the "local" data center, rack and token
using (var result = await ExecQuery(c, "select data_center, rack, tokens from system.local", logger).ConfigureAwait(false))
{
if (await result.ReadAsync().ConfigureAwait(false))
{
seed.DataCenter = (string)result["data_center"];
seed.Rack = (string)result["rack"];
seed.Tokens = (ISet<string>)result["tokens"];
logger.LogVerbose("Seed info - Address:{0} DataCenter:{1} Rack:{2}", seed.Address, seed.DataCenter, seed.Rack);
}
else
{
//strange, no local info found?!
throw new CqlException("Could not detect datacenter or rack information from the reference specified in the config section!");
}
}
//create list of nodes that make up the cluster, and add the seed
var found = new List<Node> { seed };
//get the peers
using (var result = await ExecQuery(c, "select rpc_address, data_center, rack, tokens from system.peers", logger).ConfigureAwait(false))
{
//iterate over the peers
while (await result.ReadAsync().ConfigureAwait(false))
{
//create a new node
var newNode = new Node((IPAddress)result["rpc_address"], this)
{
DataCenter = (string)result["data_center"],
Rack = (string)result["rack"],
Tokens = (ISet<string>)result["tokens"]
};
//add it if it is in scope
if (InDiscoveryScope(seed, newNode, _config.DiscoveryScope))
found.Add(newNode);
}
}
//return a new Ring of nodes
return new Ring(found, partitioner);
}
/// <summary>
/// Opens the connection
/// </summary>
private async Task OpenAsyncInternal(Logger logger)
{
//switch state to connecting if not done so
int state = Interlocked.CompareExchange(ref _connectionState, 1, 0);
if (state == 1)
return;
if (state == 2)
throw new ObjectDisposedException("Connection disposed before opening!");
try
{
//create TCP connection
_client = new TcpClient();
await _client.ConnectAsync(_address, _cluster.Config.Port).ConfigureAwait(false);
_writeStream = _client.GetStream();
_readStream = _client.GetStream();
logger.LogVerbose("TCP connection to {0} is opened", Address);
//start readloop
StartReadingAsync();
//get compression option
_allowCompression = false; //assume false unless
if (_cluster.Config.AllowCompression)
{
//check wether compression is supported by getting compression options from server
var options = new OptionsFrame();
var supported =
await SendRequestAsync(options, logger, 1, true).ConfigureAwait(false) as SupportedFrame;
if (supported == null)
throw new ProtocolException(0, "Expected Supported frame not received");
IList<string> compressionOptions;
//check if options contain compression
if (supported.SupportedOptions.TryGetValue("COMPRESSION", out compressionOptions))
{
//check wether snappy is supported
_allowCompression = compressionOptions.Contains("snappy");
}
//dispose supported frame
supported.Dispose();
}
//submit startup frame
var startup = new StartupFrame(_cluster.Config.CqlVersion);
if (_allowCompression)
{
logger.LogVerbose("Enabling Snappy Compression.");
startup.Options["COMPRESSION"] = "snappy";
}
Frame response = await SendRequestAsync(startup, logger, 1, true).ConfigureAwait(false);
//authenticate if required
var auth = response as AuthenticateFrame;
if (auth != null)
{
logger.LogVerbose("Authentication requested, attempting to provide credentials", Address);
//check if _username is actually set
if (_cluster.Config.Username == null || _cluster.Config.Password == null)
throw new UnauthorizedException("No credentials provided");
//dispose AuthenticateFrame
response.Dispose();
var cred = new CredentialsFrame(_cluster.Config.Username, _cluster.Config.Password);
response = await SendRequestAsync(cred, logger, 1, true).ConfigureAwait(false);
}
//check if ready
if (!(response is ReadyFrame))
throw new ProtocolException(0, "Expected Ready frame not received");
//dispose ready frame
response.Dispose();
using (logger.ThreadBinding())
{
if (OnConnectionChange != null)
OnConnectionChange(this, new ConnectionChangeEvent { Connected = true });
}
logger.LogInfo("{0} is opened and ready for use", this);
}
catch (Exception ex)
{
using (logger.ThreadBinding())
{
Dispose(true, ex);
throw;
}
}
}
/// <summary>
/// Opens the connection. Called once per connection only
/// </summary>
private async Task OpenAsyncInternal(Logger logger)
{
//set state to connecting
int previousState = Interlocked.CompareExchange(ref _connectionState, ConnectionState.Connecting, ConnectionState.Created);
if (previousState == ConnectionState.Closed)
throw new ObjectDisposedException(ToString());
if (previousState != ConnectionState.Created)
throw new InvalidOperationException("Opening a connection that is already connected!");
try
{
while(true)
{
//connect
await ConnectAsync().AutoConfigureAwait();
//get streams
Stream tcpStream = _client.GetStream();
_writeStream = tcpStream;
_readStream = tcpStream;
logger.LogVerbose("TCP connection for {0} is opened", this);
//start readloop
Scheduler.RunOnIOThread((Action)ReadFramesAsync);
try
{
logger.LogVerbose("Attempting to connect using protocol version {0}", Node.ProtocolVersion);
await NegotiateConnectionOptionsAsync(logger).AutoConfigureAwait();
break;
}
catch(ProtocolException pex)
{
//In case of a protocol version mismatch, Cassandra will reply with an error
//using the supported protocol version. If we are using the correct version
//something else is wrong, and it is no use to retry with a different version,
//so rethrow
if(Node.ProtocolVersion == pex.ProtocolVersion)
throw;
logger.LogVerbose(
"Failed connecting using protocol version {0}, retrying with protocol version {1}...",
Node.ProtocolVersion, pex.ProtocolVersion);
//set protocol version to the one received
Node.ProtocolVersion = pex.ProtocolVersion;
//close the connection (required as protocols are not backwards compatible, so stream may be corrupt now)
using(logger.ThreadBinding())
Disconnect();
//wait until the readloop has stopped
_readLoopCompleted.Wait();
}
}
//run the startup message exchange
await StartupAsync(logger).AutoConfigureAwait();
//yeah, connected
previousState = Interlocked.CompareExchange(ref _connectionState, ConnectionState.Connected, ConnectionState.Connecting);
if(previousState!=ConnectionState.Connecting)
throw new ObjectDisposedException(ToString(), "Connection closed while opening");
//notify connection changed
using(logger.ThreadBinding())
{
if(OnConnectionChange != null)
OnConnectionChange(this, new ConnectionChangeEvent {Connected = true});
}
logger.LogInfo("{0} is opened using Cql Protocol v{1}", this, Node.ProtocolVersion);
}
catch (Exception)
{
using (logger.ThreadBinding())
{
Close(true);
throw;
}
}
}
/// <summary>
/// Gets all nodes that make up the cluster
/// </summary>
/// <param name="seed"> The reference. </param>
/// <param name="logger"> logger used to log progress </param>
/// <param name="token"> The token. </param>
/// <returns> </returns>
/// <exception cref="CqlException">
/// Could not detect datacenter or rack information from the reference specified in the
/// config section!
/// </exception>
private async Task GetClusterInfoAsync(Node seed, Logger logger, CancellationToken token)
{
Connection c;
using(logger.ThreadBinding())
{
//get a connection
if(seed != null && seed.IsUp)
c = seed.GetOrCreateConnection();
else if(_maintenanceConnection != null && _maintenanceConnection.IsAvailable)
{
c = _maintenanceConnection;
seed = c.Node;
}
else
{
c = _connectionStrategy.GetOrCreateConnection(ConnectionScope.Infrastructure, null);
seed = c.Node;
}
}
//get local information
string partitioner;
using(var result = await ExecQuery(c,
"select cluster_name, cql_version, release_version, partitioner, data_center, rack, tokens from system.local",
logger,
token
).AutoConfigureAwait())
{
if(! await result.ReadAsyncInternal(token).AutoConfigureAwait())
throw new CqlException("Could not fetch configuration data from seed");
_name = result.GetString(0);
_cqlVersion = result.GetString(1);
_release = result.GetString(2);
partitioner = result.GetString(3);
_dataCenter = seed.DataCenter = result.GetString(4);
_rack = seed.Rack = result.GetString(5);
seed.Tokens = result.GetSet<string>(6) ?? new HashSet<string>();
}
logger.LogInfo(
"Reconfigured cluster {0}: based on Cassandra Release {1}, supporting CqlVersion {2}, using partitioner '{3}'",
_name, _release, _cqlVersion, partitioner);
//create list of nodes that make up the cluster, and add the seed
var found = new List<Node> {seed};
//get the peers
using(
var result =
await
ExecQuery(c,
"select peer, rpc_address, data_center, rack, tokens, release_version from system.peers",
logger,
token).AutoConfigureAwait())
{
//iterate over the peers
while(await result.ReadAsyncInternal(token).AutoConfigureAwait())
{
var newNode = GetNodeFromDataReader(result, logger);
//add it if it is in scope
if(InDiscoveryScope(seed, newNode, _config.DiscoveryScope))
found.Add(newNode);
}
}
//set the new Ring of nodes
_nodes.Update(found, partitioner, logger);
//check if all tokens are received
if(_nodes.Any(n => n.Tokens.Count == 0))
{
//wait and retry the fetch later...
Scheduler.RunOnThreadPool(async () =>
{
try
{
logger.LogInfo("Cluster info incomplete scheduling new retrieval in 1 minute");
await Task.Delay(TimeSpan.FromMinutes(1)).AutoConfigureAwait();
await GetClusterInfoAsync(null, logger, CancellationToken.None).AutoConfigureAwait();
}
catch(Exception ex)
{
logger.LogCritical("Critical error occured while updating cluster info: {0}", ex);
}
});
}
}
/// <summary>
/// Setups the maintenance channel.
/// </summary>
private async void SetupMaintenanceConnection(Logger logger)
{
//skip if disposed
if(_disposed)
return;
try
{
if(_maintenanceConnection == null || !_maintenanceConnection.IsAvailable)
{
//setup maintenance connection
logger.LogVerbose("Creating new maintenance connection");
//get or create a connection
Connection connection;
using(logger.ThreadBinding())
{
connection = _connectionStrategy.GetOrCreateConnection(ConnectionScope.Infrastructure, null);
}
//check if we really got a connection
if(connection == null)
throw new CqlException("Can not obtain connection for maintenance channel");
//register for events
await connection.RegisterForClusterChangesAsync(logger).AutoConfigureAwait();
//setup event handlers
connection.OnConnectionChange +=
(src, ev) => Scheduler.RunOnThreadPool(() => SetupMaintenanceConnection(logger));
connection.OnClusterChange += OnClusterChange;
//store the new connection
_maintenanceConnection = connection;
logger.LogInfo("Registered for cluster changes using {0}", connection);
}
//all seems right, we're done
return;
}
catch(Exception ex)
{
logger.LogWarning("Failed to setup maintenance connection: {0}", ex);
//temporary disconnect or registration failed, reset maintenance connection
_maintenanceConnection = null;
}
//don't retry if disposed
if(_disposed)
return;
//wait a moment, try again
logger.LogVerbose("Waiting 5secs before retrying setup maintenance connection");
await Task.Delay(5000).AutoConfigureAwait();
SetupMaintenanceConnection(logger);
}
/// <summary>
/// Opens the cluster for queries. Contains actual implementation and will be called only once per cluster
/// </summary>
/// <returns> </returns>
/// <exception cref="CqlException">Cannot construct ring from provided seeds!</exception>
private async Task OpenAsyncInternal(Logger logger, CancellationToken token)
{
logger.LogInfo("Opening Cluster with parameters: {0}", _config.ToString());
//initialize the ring
_nodes = new Ring();
//retry a few times to deal with bad network conditions
for(int attempt = 0; attempt <= _config.MaxQueryRetries && _nodes.Count == 0; attempt++)
{
//try to connect to the seeds in turn
foreach(IPAddress seedAddress in _config.ServerAddresses)
{
try
{
var seed = new Node(seedAddress, this);
await GetClusterInfoAsync(seed, logger, token).AutoConfigureAwait();
//break from the loop as it seems we are done
if(_nodes.Count > 0)
break;
}
catch(OperationCanceledException)
{
logger.LogWarning("Opening connection to cluster was cancelled");
throw;
}
catch(ProtocolException pex)
{
logger.LogWarning("Could not open cluster via {0}: {1}", seedAddress, pex.Message);
//node is not available, or starting up, try next, otherwise throw error
if(pex.Code != ErrorCode.Overloaded && pex.Code != ErrorCode.IsBootstrapping)
throw;
}
catch(SocketException ex)
{
//seed not reachable, try next
logger.LogWarning("Could not open TCP connection to seed {0}: {1}", seedAddress, ex.Message);
}
catch(IOException ex)
{
//seed not reachable, try next
logger.LogWarning("Could not discover nodes via seed {0}: {1}", seedAddress, ex);
}
}
}
//check if not disposed while opening
if(_disposed)
{
foreach(var node in _nodes)
node.Dispose();
throw new ObjectDisposedException("Cluster", "Cluster was disposed while opening");
}
//check if we found any nodes
if(_nodes.Count == 0)
{
var ex = new CqlException("Unable to connect to the cluster as none of the provided seeds is reachable.");
logger.LogCritical("Unable to setup Cluster based on given configuration: {0}", ex);
throw ex;
}
//setup cluster connection strategy
switch(_config.ConnectionStrategy)
{
case CqlSharp.ConnectionStrategy.Balanced:
_connectionStrategy = new BalancedConnectionStrategy(_nodes, _config);
break;
case CqlSharp.ConnectionStrategy.Random:
_connectionStrategy = new RandomConnectionStrategy(_nodes, _config);
break;
case CqlSharp.ConnectionStrategy.Exclusive:
_connectionStrategy = new ExclusiveConnectionStrategy(_nodes, _config);
break;
case CqlSharp.ConnectionStrategy.PartitionAware:
_connectionStrategy = new PartitionAwareConnectionStrategy(_nodes, _config);
if(_config.DiscoveryScope != DiscoveryScope.Cluster &&
_config.DiscoveryScope != DiscoveryScope.DataCenter)
{
logger.LogWarning(
"PartitionAware connection strategy performs best if DiscoveryScope is set to cluster or datacenter");
}
break;
}
//setup prepared query cache
PreparedQueryCache = new ConcurrentDictionary<string, ResultFrame>();
//setup maintenance connection
Scheduler.RunOnThreadPool(() => SetupMaintenanceConnection(logger));
}
