/// <summary>
/// Gets all nodes that make up the cluster
/// </summary>
/// <param name="seed">The reference.</param>
/// <param name="logger">logger used to log progress</param>
/// <returns></returns>
/// <exception cref="CqlException">Could not detect datacenter or rack information from the reference specified in the config section!</exception>
private async Task<Ring> DiscoverNodesAsync(Node seed, Logger logger)
{
Connection c;
using (logger.ThreadBinding())
{
//get a connection
c = seed.GetOrCreateConnection(null);
}
//get partitioner
string partitioner;
using (var result = await ExecQuery(c, "select partitioner from system.local", logger).ConfigureAwait(false))
{
if (!await result.ReadAsync().ConfigureAwait(false))
throw new CqlException("Could not detect the cluster partitioner");
partitioner = (string)result[0];
}
logger.LogInfo("Partitioner in use: {0}", partitioner);
//get the "local" data center, rack and token
using (var result = await ExecQuery(c, "select data_center, rack, tokens from system.local", logger).ConfigureAwait(false))
{
if (await result.ReadAsync().ConfigureAwait(false))
{
seed.DataCenter = (string)result["data_center"];
seed.Rack = (string)result["rack"];
seed.Tokens = (ISet<string>)result["tokens"];
logger.LogVerbose("Seed info - Address:{0} DataCenter:{1} Rack:{2}", seed.Address, seed.DataCenter, seed.Rack);
}
else
{
//strange, no local info found?!
throw new CqlException("Could not detect datacenter or rack information from the reference specified in the config section!");
}
}
//create list of nodes that make up the cluster, and add the seed
var found = new List<Node> { seed };
//get the peers
using (var result = await ExecQuery(c, "select rpc_address, data_center, rack, tokens from system.peers", logger).ConfigureAwait(false))
{
//iterate over the peers
while (await result.ReadAsync().ConfigureAwait(false))
{
//create a new node
var newNode = new Node((IPAddress)result["rpc_address"], this)
{
DataCenter = (string)result["data_center"],
Rack = (string)result["rack"],
Tokens = (ISet<string>)result["tokens"]
};
//add it if it is in scope
if (InDiscoveryScope(seed, newNode, _config.DiscoveryScope))
found.Add(newNode);
}
}
//return a new Ring of nodes
return new Ring(found, partitioner);
}
/// <summary>
/// Opens the connection
/// </summary>
private async Task OpenAsyncInternal(Logger logger)
{
//switch state to connecting if not done so
int state = Interlocked.CompareExchange(ref _connectionState, 1, 0);
if (state == 1)
return;
if (state == 2)
throw new ObjectDisposedException("Connection disposed before opening!");
try
{
//create TCP connection
_client = new TcpClient();
await _client.ConnectAsync(_address, _cluster.Config.Port).ConfigureAwait(false);
_writeStream = _client.GetStream();
_readStream = _client.GetStream();
logger.LogVerbose("TCP connection to {0} is opened", Address);
//start readloop
StartReadingAsync();
//get compression option
_allowCompression = false; //assume false unless
if (_cluster.Config.AllowCompression)
{
//check wether compression is supported by getting compression options from server
var options = new OptionsFrame();
var supported =
await SendRequestAsync(options, logger, 1, true).ConfigureAwait(false) as SupportedFrame;
if (supported == null)
throw new ProtocolException(0, "Expected Supported frame not received");
IList<string> compressionOptions;
//check if options contain compression
if (supported.SupportedOptions.TryGetValue("COMPRESSION", out compressionOptions))
{
//check wether snappy is supported
_allowCompression = compressionOptions.Contains("snappy");
}
//dispose supported frame
supported.Dispose();
}
//submit startup frame
var startup = new StartupFrame(_cluster.Config.CqlVersion);
if (_allowCompression)
{
logger.LogVerbose("Enabling Snappy Compression.");
startup.Options["COMPRESSION"] = "snappy";
}
Frame response = await SendRequestAsync(startup, logger, 1, true).ConfigureAwait(false);
//authenticate if required
var auth = response as AuthenticateFrame;
if (auth != null)
{
logger.LogVerbose("Authentication requested, attempting to provide credentials", Address);
//check if _username is actually set
if (_cluster.Config.Username == null || _cluster.Config.Password == null)
throw new UnauthorizedException("No credentials provided");
//dispose AuthenticateFrame
response.Dispose();
var cred = new CredentialsFrame(_cluster.Config.Username, _cluster.Config.Password);
response = await SendRequestAsync(cred, logger, 1, true).ConfigureAwait(false);
}
//check if ready
if (!(response is ReadyFrame))
throw new ProtocolException(0, "Expected Ready frame not received");
//dispose ready frame
response.Dispose();
using (logger.ThreadBinding())
{
if (OnConnectionChange != null)
OnConnectionChange(this, new ConnectionChangeEvent { Connected = true });
}
logger.LogInfo("{0} is opened and ready for use", this);
}
catch (Exception ex)
{
using (logger.ThreadBinding())
{
Dispose(true, ex);
throw;
}
}
}
/// <summary>
/// Submits a frame, and waits until response is received
/// </summary>
/// <param name="frame"> The frame to send. </param>
/// <param name="logger">logger to write progress to</param>
/// <param name="load"> the load indication of the request. Used for balancing queries over nodes and connections </param>
/// <param name="isConnecting">indicates if this request is send as part of connection setup protocol</param>
/// <returns> </returns>
internal async Task<Frame> SendRequestAsync(Frame frame, Logger logger, int load = 1, bool isConnecting = false)
{
try
{
//make sure we're already connected
if (!isConnecting)
await OpenAsync(logger).ConfigureAwait(false);
//make sure we are connected
if (!IsConnected)
throw new IOException("Not connected");
//count the operation
Interlocked.Increment(ref _activeRequests);
//increase the load
UpdateLoad(load, logger);
logger.LogVerbose("Waiting for connection lock on {0}...", this);
//wait until allowed to submit a frame
await _frameSubmitLock.WaitAsync().ConfigureAwait(false);
//get a task that gets completed when a response is received
var waitTask = new TaskCompletionSource<Frame>();
//get a stream id, and store wait task under that id
sbyte id;
lock (_availableQueryIds)
{
id = _availableQueryIds.Dequeue();
_openRequests.Add(id, waitTask);
}
try
{
//send frame
frame.Stream = id;
//serialize frame outside lock
Stream frameBytes = frame.GetFrameBytes(_allowCompression && !isConnecting, _cluster.Config.CompressionTreshold);
await _writeLock.WaitAsync().ConfigureAwait(false);
try
{
//final check to make sure we're connected
if (_connectionState != 1)
throw new IOException("Not connected");
logger.LogVerbose("Sending {0} Frame with Id {1}, to {2}", frame.OpCode, id, this);
await frameBytes.CopyToAsync(_writeStream).ConfigureAwait(false);
}
finally
{
_writeLock.Release();
frameBytes.Dispose();
}
//wait until response is received
Frame response = await waitTask.Task.ConfigureAwait(false);
logger.LogVerbose("{0} response for frame with Id {1} received from {2}", response.OpCode, id, Address);
//throw error if result is an error
var error = response as ErrorFrame;
if (error != null)
{
throw error.Exception;
}
//return response
return response;
}
finally
{
//return request slot to the pool
lock (_availableQueryIds)
{
_openRequests.Remove(id);
_availableQueryIds.Enqueue(id);
}
//allow another frame to be send
_frameSubmitLock.Release();
//reduce load, we are done
Interlocked.Decrement(ref _activeRequests);
UpdateLoad(-load, logger);
}
}
catch (ProtocolException pex)
{
switch (pex.Code)
{
case ErrorCode.IsBootstrapping:
case ErrorCode.Overloaded:
using (logger.ThreadBinding())
{
//IO or node status related error, dispose this connection
Dispose(true, pex);
throw;
}
default:
//some other Cql error (syntax ok?), simply rethrow
throw;
}
}
catch (Exception ex)
{
using (logger.ThreadBinding())
{
//connection collapsed, dispose this connection
Dispose(true, ex);
throw;
}
}
}
/// <summary>
/// Sends the request async internal. Cancellation supported until request is send, after which answer must be handled
/// to avoid connection corruption.
/// </summary>
/// <param name="frame"> The frame. </param>
/// <param name="logger"> The logger. </param>
/// <param name="load"> The load. </param>
/// <param name="token"> The token. </param>
/// <returns> </returns>
/// <exception cref="System.IO.IOException">Not connected</exception>
private async Task<Frame> SendRequestAsyncInternal(Frame frame, Logger logger, int load, CancellationToken token)
{
try
{
//make sure we aren't disposed
if(_connectionState == ConnectionState.Closed)
throw new ObjectDisposedException(ToString());
//count the operation
Interlocked.Increment(ref _activeRequests);
if (_connectionState == ConnectionState.Connected)
{
//increase the load
UpdateLoad(load, logger);
//wait until frame id is available to submit a frame
logger.LogVerbose("Waiting for connection lock on {0}...", this);
if (Scheduler.RunningSynchronously)
_frameSubmitLock.Wait(token);
else
await _frameSubmitLock.WaitAsync(token).AutoConfigureAwait();
}
//get a task that gets completed when a response is received
var waitTask = new TaskCompletionSource<Frame>();
//get a stream id, and store wait task under that id
short id;
lock (_availableQueryIds)
{
id = _availableQueryIds.Count > 0 ? _availableQueryIds.Dequeue() : _usedQueryIds++;
_openRequests.Add(id, waitTask);
}
try
{
//send frame
frame.Stream = id;
//set protocol version in use
frame.ProtocolVersion = Node.ProtocolVersion;
//serialize frame outside lock
PoolMemoryStream frameBytes = frame.GetFrameBytes(_allowCompression && (_connectionState!=ConnectionState.Connecting),
_config.CompressionTreshold);
//wait to get access to stream
if (Scheduler.RunningSynchronously)
_writeLock.Wait(token);
else
await _writeLock.WaitAsync(token).AutoConfigureAwait();
try
{
//make very sure we aren't disposed
if (_connectionState == ConnectionState.Closed)
throw new ObjectDisposedException(ToString());
logger.LogVerbose("Sending {0} Frame with Id {1} over {2}", frame.OpCode, id, this);
//write frame to stream, don't use cancelToken to prevent half-written frames
if (Scheduler.RunningSynchronously)
frameBytes.CopyTo(_writeStream);
else
await frameBytes.CopyToAsync(_writeStream).AutoConfigureAwait();
//unblock readloop to read result
_readLock.Release();
}
finally
{
_writeLock.Release();
frameBytes.Dispose();
}
//wait until response is received
Frame response = await waitTask.Task.AutoConfigureAwait();
logger.LogVerbose("Received {0} Frame with Id {1} on {2}", response.OpCode, id,
this);
//read frame content
await response.ReadFrameContentAsync().AutoConfigureAwait();
//throw error if result is an error
var error = response as ErrorFrame;
if (error != null)
{
//dispose error frame
error.Dispose();
//throw exception
throw error.Exception;
}
//check for keyspace change
var keyspaceChange = response as ResultFrame;
if (keyspaceChange != null && keyspaceChange.CqlResultType == CqlResultType.SetKeyspace)
{
logger.LogVerbose("{0} changed KeySpace to \"{1}\"", this, keyspaceChange.Keyspace);
CurrentKeySpace = keyspaceChange.Keyspace;
}
//dispose frame, when cancellation requested
if (token.IsCancellationRequested)
{
response.Dispose();
throw new OperationCanceledException(token);
}
//return response
return response;
}
finally
{
//return request slot to the pool
lock (_availableQueryIds)
{
_openRequests.Remove(id);
_availableQueryIds.Enqueue(id);
}
if (_connectionState == ConnectionState.Connected)
{
//allow another frame to be send
_frameSubmitLock.Release();
//reduce load, we are done
UpdateLoad(-load, logger);
}
//decrease the amount of operations
Interlocked.Decrement(ref _activeRequests);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (ProtocolException pex)
{
switch (pex.Code)
{
case ErrorCode.IsBootstrapping:
case ErrorCode.Overloaded:
using (logger.ThreadBinding())
{
//IO or node status related error, dispose this connection
Close(true);
throw;
}
default:
//some other Cql error (syntax ok?), simply rethrow
throw;
}
}
catch (ObjectDisposedException odex)
{
throw new IOException("Connection closed while processing request", odex);
}
catch (Exception)
{
using (logger.ThreadBinding())
{
//connection collapsed, dispose this connection
Close(true);
throw;
}
}
}
/// <summary>
/// Opens the connection. Called once per connection only
/// </summary>
private async Task OpenAsyncInternal(Logger logger)
{
//set state to connecting
int previousState = Interlocked.CompareExchange(ref _connectionState, ConnectionState.Connecting, ConnectionState.Created);
if (previousState == ConnectionState.Closed)
throw new ObjectDisposedException(ToString());
if (previousState != ConnectionState.Created)
throw new InvalidOperationException("Opening a connection that is already connected!");
try
{
while(true)
{
//connect
await ConnectAsync().AutoConfigureAwait();
//get streams
Stream tcpStream = _client.GetStream();
_writeStream = tcpStream;
_readStream = tcpStream;
logger.LogVerbose("TCP connection for {0} is opened", this);
//start readloop
Scheduler.RunOnIOThread((Action)ReadFramesAsync);
try
{
logger.LogVerbose("Attempting to connect using protocol version {0}", Node.ProtocolVersion);
await NegotiateConnectionOptionsAsync(logger).AutoConfigureAwait();
break;
}
catch(ProtocolException pex)
{
//In case of a protocol version mismatch, Cassandra will reply with an error
//using the supported protocol version. If we are using the correct version
//something else is wrong, and it is no use to retry with a different version,
//so rethrow
if(Node.ProtocolVersion == pex.ProtocolVersion)
throw;
logger.LogVerbose(
"Failed connecting using protocol version {0}, retrying with protocol version {1}...",
Node.ProtocolVersion, pex.ProtocolVersion);
//set protocol version to the one received
Node.ProtocolVersion = pex.ProtocolVersion;
//close the connection (required as protocols are not backwards compatible, so stream may be corrupt now)
using(logger.ThreadBinding())
Disconnect();
//wait until the readloop has stopped
_readLoopCompleted.Wait();
}
}
//run the startup message exchange
await StartupAsync(logger).AutoConfigureAwait();
//yeah, connected
previousState = Interlocked.CompareExchange(ref _connectionState, ConnectionState.Connected, ConnectionState.Connecting);
if(previousState!=ConnectionState.Connecting)
throw new ObjectDisposedException(ToString(), "Connection closed while opening");
//notify connection changed
using(logger.ThreadBinding())
{
if(OnConnectionChange != null)
OnConnectionChange(this, new ConnectionChangeEvent {Connected = true});
}
logger.LogInfo("{0} is opened using Cql Protocol v{1}", this, Node.ProtocolVersion);
}
catch (Exception)
{
using (logger.ThreadBinding())
{
Close(true);
throw;
}
}
}
/// <summary>
/// Gets all nodes that make up the cluster
/// </summary>
/// <param name="seed"> The reference. </param>
/// <param name="logger"> logger used to log progress </param>
/// <param name="token"> The token. </param>
/// <returns> </returns>
/// <exception cref="CqlException">
/// Could not detect datacenter or rack information from the reference specified in the
/// config section!
/// </exception>
private async Task GetClusterInfoAsync(Node seed, Logger logger, CancellationToken token)
{
Connection c;
using(logger.ThreadBinding())
{
//get a connection
if(seed != null && seed.IsUp)
c = seed.GetOrCreateConnection();
else if(_maintenanceConnection != null && _maintenanceConnection.IsAvailable)
{
c = _maintenanceConnection;
seed = c.Node;
}
else
{
c = _connectionStrategy.GetOrCreateConnection(ConnectionScope.Infrastructure, null);
seed = c.Node;
}
}
//get local information
string partitioner;
using(var result = await ExecQuery(c,
"select cluster_name, cql_version, release_version, partitioner, data_center, rack, tokens from system.local",
logger,
token
).AutoConfigureAwait())
{
if(! await result.ReadAsyncInternal(token).AutoConfigureAwait())
throw new CqlException("Could not fetch configuration data from seed");
_name = result.GetString(0);
_cqlVersion = result.GetString(1);
_release = result.GetString(2);
partitioner = result.GetString(3);
_dataCenter = seed.DataCenter = result.GetString(4);
_rack = seed.Rack = result.GetString(5);
seed.Tokens = result.GetSet<string>(6) ?? new HashSet<string>();
}
logger.LogInfo(
"Reconfigured cluster {0}: based on Cassandra Release {1}, supporting CqlVersion {2}, using partitioner '{3}'",
_name, _release, _cqlVersion, partitioner);
//create list of nodes that make up the cluster, and add the seed
var found = new List<Node> {seed};
//get the peers
using(
var result =
await
ExecQuery(c,
"select peer, rpc_address, data_center, rack, tokens, release_version from system.peers",
logger,
token).AutoConfigureAwait())
{
//iterate over the peers
while(await result.ReadAsyncInternal(token).AutoConfigureAwait())
{
var newNode = GetNodeFromDataReader(result, logger);
//add it if it is in scope
if(InDiscoveryScope(seed, newNode, _config.DiscoveryScope))
found.Add(newNode);
}
}
//set the new Ring of nodes
_nodes.Update(found, partitioner, logger);
//check if all tokens are received
if(_nodes.Any(n => n.Tokens.Count == 0))
{
//wait and retry the fetch later...
Scheduler.RunOnThreadPool(async () =>
{
try
{
logger.LogInfo("Cluster info incomplete scheduling new retrieval in 1 minute");
await Task.Delay(TimeSpan.FromMinutes(1)).AutoConfigureAwait();
await GetClusterInfoAsync(null, logger, CancellationToken.None).AutoConfigureAwait();
}
catch(Exception ex)
{
logger.LogCritical("Critical error occured while updating cluster info: {0}", ex);
}
});
}
}
/// <summary>
/// Setups the maintenance channel.
/// </summary>
private async void SetupMaintenanceConnection(Logger logger)
{
//skip if disposed
if(_disposed)
return;
try
{
if(_maintenanceConnection == null || !_maintenanceConnection.IsAvailable)
{
//setup maintenance connection
logger.LogVerbose("Creating new maintenance connection");
//get or create a connection
Connection connection;
using(logger.ThreadBinding())
{
connection = _connectionStrategy.GetOrCreateConnection(ConnectionScope.Infrastructure, null);
}
//check if we really got a connection
if(connection == null)
throw new CqlException("Can not obtain connection for maintenance channel");
//register for events
await connection.RegisterForClusterChangesAsync(logger).AutoConfigureAwait();
//setup event handlers
connection.OnConnectionChange +=
(src, ev) => Scheduler.RunOnThreadPool(() => SetupMaintenanceConnection(logger));
connection.OnClusterChange += OnClusterChange;
//store the new connection
_maintenanceConnection = connection;
logger.LogInfo("Registered for cluster changes using {0}", connection);
}
//all seems right, we're done
return;
}
catch(Exception ex)
{
logger.LogWarning("Failed to setup maintenance connection: {0}", ex);
//temporary disconnect or registration failed, reset maintenance connection
_maintenanceConnection = null;
}
//don't retry if disposed
if(_disposed)
return;
//wait a moment, try again
logger.LogVerbose("Waiting 5secs before retrying setup maintenance connection");
await Task.Delay(5000).AutoConfigureAwait();
SetupMaintenanceConnection(logger);
}
private void CompleteOpen(Logger logger)
{
//get a connection
using(logger.ThreadBinding())
{
_connection = Cluster.ConnectionStrategy.GetOrCreateConnection(ConnectionScope.Connection,
PartitionKey.None);
}
//set database to its default
_database = Cluster.Config.Database;
}