/// <summary>
/// Executes a query.
/// </summary>
/// <param name="connection">The connection.</param>
/// <param name="cql">The CQL.</param>
/// <param name="logger">The logger.</param>
/// <returns>
/// A CqlDataReader that can be used to access the query results
/// </returns>
private async Task<CqlDataReader> ExecQuery(Connection connection, string cql, Logger logger)
{
logger.LogVerbose("Excuting query {0} on {1}", cql, connection);
var query = new QueryFrame(cql, CqlConsistency.One);
var result = (ResultFrame)await connection.SendRequestAsync(query, logger).ConfigureAwait(false);
var reader = new CqlDataReader(result);
logger.LogVerbose("Query {0} returned {1} results", cql, reader.Count);
return reader;
}
/// <summary>
/// Setups the maintenance channel.
/// </summary>
private async void SetupMaintenanceConnection(Logger logger)
{
try
{
if (_maintenanceConnection == null || !_maintenanceConnection.IsConnected)
{
//setup maintenance connection
logger.LogVerbose("Creating new maintenance connection");
//pick a random node from the list
var strategy = new RandomConnectionStrategy(_nodes, _config);
//get or create a connection
var connection = strategy.GetOrCreateConnection(null);
//allow this connection to be used by others as well
_connectionSelector.ReturnConnection(connection);
//setup event handlers
connection.OnConnectionChange += (src, ev) => SetupMaintenanceConnection(logger);
connection.OnClusterChange += OnClusterChange;
//store the new connection
_maintenanceConnection = connection;
//register for events
await connection.RegisterForClusterChangesAsync(logger).ConfigureAwait(false);
logger.LogInfo("Registered for cluster changes using {0}", connection);
}
//all seems right, we're done
return;
}
catch (Exception ex)
{
logger.LogWarning("Failed to setup maintenance connection: {0}", ex);
//temporary disconnect or registration failed, reset maintenance connection
_maintenanceConnection = null;
}
//wait a moment, try again
logger.LogVerbose("Waiting 2secs before retrying setup maintenance connection");
await Task.Delay(2000).ConfigureAwait(false);
SetupMaintenanceConnection(logger);
}
/// <summary>
/// Gets all nodes that make up the cluster
/// </summary>
/// <param name="seed">The reference.</param>
/// <param name="logger">logger used to log progress</param>
/// <returns></returns>
/// <exception cref="CqlException">Could not detect datacenter or rack information from the reference specified in the config section!</exception>
private async Task<Ring> DiscoverNodesAsync(Node seed, Logger logger)
{
Connection c;
using (logger.ThreadBinding())
{
//get a connection
c = seed.GetOrCreateConnection(null);
}
//get partitioner
string partitioner;
using (var result = await ExecQuery(c, "select partitioner from system.local", logger).ConfigureAwait(false))
{
if (!await result.ReadAsync().ConfigureAwait(false))
throw new CqlException("Could not detect the cluster partitioner");
partitioner = (string)result[0];
}
logger.LogInfo("Partitioner in use: {0}", partitioner);
//get the "local" data center, rack and token
using (var result = await ExecQuery(c, "select data_center, rack, tokens from system.local", logger).ConfigureAwait(false))
{
if (await result.ReadAsync().ConfigureAwait(false))
{
seed.DataCenter = (string)result["data_center"];
seed.Rack = (string)result["rack"];
seed.Tokens = (ISet<string>)result["tokens"];
logger.LogVerbose("Seed info - Address:{0} DataCenter:{1} Rack:{2}", seed.Address, seed.DataCenter, seed.Rack);
}
else
{
//strange, no local info found?!
throw new CqlException("Could not detect datacenter or rack information from the reference specified in the config section!");
}
}
//create list of nodes that make up the cluster, and add the seed
var found = new List<Node> { seed };
//get the peers
using (var result = await ExecQuery(c, "select rpc_address, data_center, rack, tokens from system.peers", logger).ConfigureAwait(false))
{
//iterate over the peers
while (await result.ReadAsync().ConfigureAwait(false))
{
//create a new node
var newNode = new Node((IPAddress)result["rpc_address"], this)
{
DataCenter = (string)result["data_center"],
Rack = (string)result["rack"],
Tokens = (ISet<string>)result["tokens"]
};
//add it if it is in scope
if (InDiscoveryScope(seed, newNode, _config.DiscoveryScope))
found.Add(newNode);
}
}
//return a new Ring of nodes
return new Ring(found, partitioner);
}
/// <summary>
/// Opens the connection
/// </summary>
private async Task OpenAsyncInternal(Logger logger)
{
//switch state to connecting if not done so
int state = Interlocked.CompareExchange(ref _connectionState, 1, 0);
if (state == 1)
return;
if (state == 2)
throw new ObjectDisposedException("Connection disposed before opening!");
try
{
//create TCP connection
_client = new TcpClient();
await _client.ConnectAsync(_address, _cluster.Config.Port).ConfigureAwait(false);
_writeStream = _client.GetStream();
_readStream = _client.GetStream();
logger.LogVerbose("TCP connection to {0} is opened", Address);
//start readloop
StartReadingAsync();
//get compression option
_allowCompression = false; //assume false unless
if (_cluster.Config.AllowCompression)
{
//check wether compression is supported by getting compression options from server
var options = new OptionsFrame();
var supported =
await SendRequestAsync(options, logger, 1, true).ConfigureAwait(false) as SupportedFrame;
if (supported == null)
throw new ProtocolException(0, "Expected Supported frame not received");
IList<string> compressionOptions;
//check if options contain compression
if (supported.SupportedOptions.TryGetValue("COMPRESSION", out compressionOptions))
{
//check wether snappy is supported
_allowCompression = compressionOptions.Contains("snappy");
}
//dispose supported frame
supported.Dispose();
}
//submit startup frame
var startup = new StartupFrame(_cluster.Config.CqlVersion);
if (_allowCompression)
{
logger.LogVerbose("Enabling Snappy Compression.");
startup.Options["COMPRESSION"] = "snappy";
}
Frame response = await SendRequestAsync(startup, logger, 1, true).ConfigureAwait(false);
//authenticate if required
var auth = response as AuthenticateFrame;
if (auth != null)
{
logger.LogVerbose("Authentication requested, attempting to provide credentials", Address);
//check if _username is actually set
if (_cluster.Config.Username == null || _cluster.Config.Password == null)
throw new UnauthorizedException("No credentials provided");
//dispose AuthenticateFrame
response.Dispose();
var cred = new CredentialsFrame(_cluster.Config.Username, _cluster.Config.Password);
response = await SendRequestAsync(cred, logger, 1, true).ConfigureAwait(false);
}
//check if ready
if (!(response is ReadyFrame))
throw new ProtocolException(0, "Expected Ready frame not received");
//dispose ready frame
response.Dispose();
using (logger.ThreadBinding())
{
if (OnConnectionChange != null)
OnConnectionChange(this, new ConnectionChangeEvent { Connected = true });
}
logger.LogInfo("{0} is opened and ready for use", this);
}
catch (Exception ex)
{
using (logger.ThreadBinding())
{
Dispose(true, ex);
throw;
}
}
}
/// <summary>
/// Submits a frame, and waits until response is received
/// </summary>
/// <param name="frame"> The frame to send. </param>
/// <param name="logger">logger to write progress to</param>
/// <param name="load"> the load indication of the request. Used for balancing queries over nodes and connections </param>
/// <param name="isConnecting">indicates if this request is send as part of connection setup protocol</param>
/// <returns> </returns>
internal async Task<Frame> SendRequestAsync(Frame frame, Logger logger, int load = 1, bool isConnecting = false)
{
try
{
//make sure we're already connected
if (!isConnecting)
await OpenAsync(logger).ConfigureAwait(false);
//make sure we are connected
if (!IsConnected)
throw new IOException("Not connected");
//count the operation
Interlocked.Increment(ref _activeRequests);
//increase the load
UpdateLoad(load, logger);
logger.LogVerbose("Waiting for connection lock on {0}...", this);
//wait until allowed to submit a frame
await _frameSubmitLock.WaitAsync().ConfigureAwait(false);
//get a task that gets completed when a response is received
var waitTask = new TaskCompletionSource<Frame>();
//get a stream id, and store wait task under that id
sbyte id;
lock (_availableQueryIds)
{
id = _availableQueryIds.Dequeue();
_openRequests.Add(id, waitTask);
}
try
{
//send frame
frame.Stream = id;
//serialize frame outside lock
Stream frameBytes = frame.GetFrameBytes(_allowCompression && !isConnecting, _cluster.Config.CompressionTreshold);
await _writeLock.WaitAsync().ConfigureAwait(false);
try
{
//final check to make sure we're connected
if (_connectionState != 1)
throw new IOException("Not connected");
logger.LogVerbose("Sending {0} Frame with Id {1}, to {2}", frame.OpCode, id, this);
await frameBytes.CopyToAsync(_writeStream).ConfigureAwait(false);
}
finally
{
_writeLock.Release();
frameBytes.Dispose();
}
//wait until response is received
Frame response = await waitTask.Task.ConfigureAwait(false);
logger.LogVerbose("{0} response for frame with Id {1} received from {2}", response.OpCode, id, Address);
//throw error if result is an error
var error = response as ErrorFrame;
if (error != null)
{
throw error.Exception;
}
//return response
return response;
}
finally
{
//return request slot to the pool
lock (_availableQueryIds)
{
_openRequests.Remove(id);
_availableQueryIds.Enqueue(id);
}
//allow another frame to be send
_frameSubmitLock.Release();
//reduce load, we are done
Interlocked.Decrement(ref _activeRequests);
UpdateLoad(-load, logger);
}
}
catch (ProtocolException pex)
{
switch (pex.Code)
{
case ErrorCode.IsBootstrapping:
case ErrorCode.Overloaded:
using (logger.ThreadBinding())
{
//IO or node status related error, dispose this connection
Dispose(true, pex);
throw;
}
default:
//some other Cql error (syntax ok?), simply rethrow
throw;
}
}
catch (Exception ex)
{
using (logger.ThreadBinding())
{
//connection collapsed, dispose this connection
Dispose(true, ex);
throw;
}
}
}
/// <summary>
/// Sends the request async internal. Cancellation supported until request is send, after which answer must be handled
/// to avoid connection corruption.
/// </summary>
/// <param name="frame"> The frame. </param>
/// <param name="logger"> The logger. </param>
/// <param name="load"> The load. </param>
/// <param name="token"> The token. </param>
/// <returns> </returns>
/// <exception cref="System.IO.IOException">Not connected</exception>
private async Task<Frame> SendRequestAsyncInternal(Frame frame, Logger logger, int load, CancellationToken token)
{
try
{
//make sure we aren't disposed
if(_connectionState == ConnectionState.Closed)
throw new ObjectDisposedException(ToString());
//count the operation
Interlocked.Increment(ref _activeRequests);
if (_connectionState == ConnectionState.Connected)
{
//increase the load
UpdateLoad(load, logger);
//wait until frame id is available to submit a frame
logger.LogVerbose("Waiting for connection lock on {0}...", this);
if (Scheduler.RunningSynchronously)
_frameSubmitLock.Wait(token);
else
await _frameSubmitLock.WaitAsync(token).AutoConfigureAwait();
}
//get a task that gets completed when a response is received
var waitTask = new TaskCompletionSource<Frame>();
//get a stream id, and store wait task under that id
short id;
lock (_availableQueryIds)
{
id = _availableQueryIds.Count > 0 ? _availableQueryIds.Dequeue() : _usedQueryIds++;
_openRequests.Add(id, waitTask);
}
try
{
//send frame
frame.Stream = id;
//set protocol version in use
frame.ProtocolVersion = Node.ProtocolVersion;
//serialize frame outside lock
PoolMemoryStream frameBytes = frame.GetFrameBytes(_allowCompression && (_connectionState!=ConnectionState.Connecting),
_config.CompressionTreshold);
//wait to get access to stream
if (Scheduler.RunningSynchronously)
_writeLock.Wait(token);
else
await _writeLock.WaitAsync(token).AutoConfigureAwait();
try
{
//make very sure we aren't disposed
if (_connectionState == ConnectionState.Closed)
throw new ObjectDisposedException(ToString());
logger.LogVerbose("Sending {0} Frame with Id {1} over {2}", frame.OpCode, id, this);
//write frame to stream, don't use cancelToken to prevent half-written frames
if (Scheduler.RunningSynchronously)
frameBytes.CopyTo(_writeStream);
else
await frameBytes.CopyToAsync(_writeStream).AutoConfigureAwait();
//unblock readloop to read result
_readLock.Release();
}
finally
{
_writeLock.Release();
frameBytes.Dispose();
}
//wait until response is received
Frame response = await waitTask.Task.AutoConfigureAwait();
logger.LogVerbose("Received {0} Frame with Id {1} on {2}", response.OpCode, id,
this);
//read frame content
await response.ReadFrameContentAsync().AutoConfigureAwait();
//throw error if result is an error
var error = response as ErrorFrame;
if (error != null)
{
//dispose error frame
error.Dispose();
//throw exception
throw error.Exception;
}
//check for keyspace change
var keyspaceChange = response as ResultFrame;
if (keyspaceChange != null && keyspaceChange.CqlResultType == CqlResultType.SetKeyspace)
{
logger.LogVerbose("{0} changed KeySpace to \"{1}\"", this, keyspaceChange.Keyspace);
CurrentKeySpace = keyspaceChange.Keyspace;
}
//dispose frame, when cancellation requested
if (token.IsCancellationRequested)
{
response.Dispose();
throw new OperationCanceledException(token);
}
//return response
return response;
}
finally
{
//return request slot to the pool
lock (_availableQueryIds)
{
_openRequests.Remove(id);
_availableQueryIds.Enqueue(id);
}
if (_connectionState == ConnectionState.Connected)
{
//allow another frame to be send
_frameSubmitLock.Release();
//reduce load, we are done
UpdateLoad(-load, logger);
}
//decrease the amount of operations
Interlocked.Decrement(ref _activeRequests);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (ProtocolException pex)
{
switch (pex.Code)
{
case ErrorCode.IsBootstrapping:
case ErrorCode.Overloaded:
using (logger.ThreadBinding())
{
//IO or node status related error, dispose this connection
Close(true);
throw;
}
default:
//some other Cql error (syntax ok?), simply rethrow
throw;
}
}
catch (ObjectDisposedException odex)
{
throw new IOException("Connection closed while processing request", odex);
}
catch (Exception)
{
using (logger.ThreadBinding())
{
//connection collapsed, dispose this connection
Close(true);
throw;
}
}
}
/// <summary>
/// Updates the load of this connection, and will trigger a corresponding event
/// </summary>
/// <param name="load">The load.</param>
/// <param name="logger">The logger.</param>
private void UpdateLoad(int load, Logger logger)
{
var newLoad = Interlocked.Add(ref _load, load);
Interlocked.Exchange(ref _lastActivity, DateTime.Now.Ticks);
EventHandler<LoadChangeEvent> handler = OnLoadChange;
if (handler != null) handler(this, new LoadChangeEvent { LoadDelta = load });
logger.LogVerbose("{0} has now a load of {1}", this, newLoad);
}
/// <summary>
/// Authenticates the connection.
/// </summary>
/// <param name="auth">The authentication request from the server.</param>
/// <param name="logger">The logger.</param>
/// <returns></returns>
/// <exception cref="AuthenticationException">
/// Unsupported Authenticator: + auth.Authenticator;null
/// or
/// Authentication failed, SASL Challenge was rejected by client
/// or
/// Authentication failed, Authenticator rejected SASL result
/// or
/// Expected a Authentication Challenge from Server!
/// or
/// No credentials provided in configuration
/// or
/// Authentication failed: Ready frame not received
/// </exception>
private async Task AuthenticateAsync(AuthenticateFrame auth, Logger logger)
{
logger.LogVerbose("Authentication requested, attempting to provide credentials");
//dispose AuthenticateFrame
auth.Dispose();
if (auth.ProtocolVersion >= 2)
{
//protocol version2: use SASL AuthResponse to authenticate
//get an AuthenticatorFactory
IAuthenticatorFactory factory =
Loader.Extensions.AuthenticationFactories.FirstOrDefault(
f => f.Name.Equals(auth.Authenticator, StringComparison.OrdinalIgnoreCase));
if (factory == null)
throw new AuthenticationException(auth.ProtocolVersion, "Unsupported Authenticator: " + auth.Authenticator);
logger.LogVerbose("Attempting authentication for scheme {0}", factory.Name);
//grab an authenticator instance
IAuthenticator authenticator = factory.CreateAuthenticator(_config);
//start authentication loop
byte[] saslChallenge = null;
while (true)
{
//check for challenge
byte[] saslResponse;
if (!authenticator.Authenticate(auth.ProtocolVersion, saslChallenge, out saslResponse))
{
throw new AuthenticationException(auth.ProtocolVersion, "Authentication failed, SASL Challenge was rejected by client");
}
//send response
var cred = new AuthResponseFrame(saslResponse);
var authResponse =
await
SendRequestAsyncInternal(cred, logger, 1, CancellationToken.None).AutoConfigureAwait();
//dispose authResponse (makes sure all is read)
authResponse.Dispose();
//check for success
var success = authResponse as AuthSuccessFrame;
if (success != null)
{
if (!authenticator.Authenticate(auth.ProtocolVersion, success.SaslResult))
{
throw new AuthenticationException(authResponse.ProtocolVersion, "Authentication failed, Authenticator rejected SASL result", authResponse.TracingId);
}
//yeah, authenticated, break from the authentication loop
break;
}
//no success yet, lets try next round
var challenge = authResponse as AuthChallengeFrame;
if (challenge == null)
{
throw new AuthenticationException(authResponse.ProtocolVersion, "Expected a Authentication Challenge from Server!", authResponse.TracingId);
}
saslChallenge = challenge.SaslChallenge;
}
}
else
{
//protocol version1: use Credentials to authenticate
//check if _username is actually set
if (_config.Username == null || _config.Password == null)
throw new AuthenticationException(auth.ProtocolVersion, "No credentials provided in configuration");
var cred = new CredentialsFrame(_config.Username, _config.Password);
var authResponse =
await
SendRequestAsyncInternal(cred, logger, 1, CancellationToken.None).AutoConfigureAwait();
//dispose authResponse (makes sure all is read)
authResponse.Dispose();
if (!(authResponse is ReadyFrame))
{
throw new AuthenticationException(authResponse.ProtocolVersion, "Authentication failed: Ready frame not received", authResponse.TracingId);
}
}
}
/// <summary>
/// Startups the connection using the required message exchange
/// </summary>
/// <param name="logger">The logger.</param>
/// <returns></returns>
/// <exception cref="ProtocolException">0;Expected Ready frame not received</exception>
private async Task StartupAsync(Logger logger)
{
//submit startup frame
var startup = new StartupFrame(_config.CqlVersion);
if (_allowCompression)
{
logger.LogVerbose("Enabling Snappy Compression.");
startup.Options["COMPRESSION"] = "snappy";
}
Frame response =
await
SendRequestAsyncInternal(startup, logger, 1, CancellationToken.None).AutoConfigureAwait();
//authenticate if required
var auth = response as AuthenticateFrame;
if (auth != null)
await AuthenticateAsync(auth, logger).AutoConfigureAwait();
//no authenticate frame, so ready frame must be received
else if (!(response is ReadyFrame))
throw new ProtocolException(response.ProtocolVersion, 0, "Expected Ready frame not received", response.TracingId);
//dispose ready frame
response.Dispose();
}
/// <summary>
/// Opens the connection. Called once per connection only
/// </summary>
private async Task OpenAsyncInternal(Logger logger)
{
//set state to connecting
int previousState = Interlocked.CompareExchange(ref _connectionState, ConnectionState.Connecting, ConnectionState.Created);
if (previousState == ConnectionState.Closed)
throw new ObjectDisposedException(ToString());
if (previousState != ConnectionState.Created)
throw new InvalidOperationException("Opening a connection that is already connected!");
try
{
while(true)
{
//connect
await ConnectAsync().AutoConfigureAwait();
//get streams
Stream tcpStream = _client.GetStream();
_writeStream = tcpStream;
_readStream = tcpStream;
logger.LogVerbose("TCP connection for {0} is opened", this);
//start readloop
Scheduler.RunOnIOThread((Action)ReadFramesAsync);
try
{
logger.LogVerbose("Attempting to connect using protocol version {0}", Node.ProtocolVersion);
await NegotiateConnectionOptionsAsync(logger).AutoConfigureAwait();
break;
}
catch(ProtocolException pex)
{
//In case of a protocol version mismatch, Cassandra will reply with an error
//using the supported protocol version. If we are using the correct version
//something else is wrong, and it is no use to retry with a different version,
//so rethrow
if(Node.ProtocolVersion == pex.ProtocolVersion)
throw;
logger.LogVerbose(
"Failed connecting using protocol version {0}, retrying with protocol version {1}...",
Node.ProtocolVersion, pex.ProtocolVersion);
//set protocol version to the one received
Node.ProtocolVersion = pex.ProtocolVersion;
//close the connection (required as protocols are not backwards compatible, so stream may be corrupt now)
using(logger.ThreadBinding())
Disconnect();
//wait until the readloop has stopped
_readLoopCompleted.Wait();
}
}
//run the startup message exchange
await StartupAsync(logger).AutoConfigureAwait();
//yeah, connected
previousState = Interlocked.CompareExchange(ref _connectionState, ConnectionState.Connected, ConnectionState.Connecting);
if(previousState!=ConnectionState.Connecting)
throw new ObjectDisposedException(ToString(), "Connection closed while opening");
//notify connection changed
using(logger.ThreadBinding())
{
if(OnConnectionChange != null)
OnConnectionChange(this, new ConnectionChangeEvent {Connected = true});
}
logger.LogInfo("{0} is opened using Cql Protocol v{1}", this, Node.ProtocolVersion);
}
catch (Exception)
{
using (logger.ThreadBinding())
{
Close(true);
throw;
}
}
}
/// <summary>
/// Executes a query.
/// </summary>
/// <param name="connection"> The connection. </param>
/// <param name="cql"> The CQL. </param>
/// <param name="logger"> The logger. </param>
/// <param name="token"> The token. </param>
/// <returns> A CqlDataReader that can be used to access the query results </returns>
private async Task<CqlDataReader> ExecQuery(Connection connection, string cql, Logger logger,
CancellationToken token)
{
//cancel if requested
token.ThrowIfCancellationRequested();
logger.LogVerbose("Excuting query {0} on {1}", cql, connection);
var query = new QueryFrame(cql, CqlConsistency.One, null);
var result =
(ResultFrame)
await
connection.SendRequestAsync(query, logger, 1, token)
.AutoConfigureAwait();
var reader = new CqlDataReader(null, result, null);
logger.LogVerbose("Query {0} returned {1} results", cql, reader.Count);
return reader;
}
/// <summary>
/// Setups the maintenance channel.
/// </summary>
private async void SetupMaintenanceConnection(Logger logger)
{
//skip if disposed
if(_disposed)
return;
try
{
if(_maintenanceConnection == null || !_maintenanceConnection.IsAvailable)
{
//setup maintenance connection
logger.LogVerbose("Creating new maintenance connection");
//get or create a connection
Connection connection;
using(logger.ThreadBinding())
{
connection = _connectionStrategy.GetOrCreateConnection(ConnectionScope.Infrastructure, null);
}
//check if we really got a connection
if(connection == null)
throw new CqlException("Can not obtain connection for maintenance channel");
//register for events
await connection.RegisterForClusterChangesAsync(logger).AutoConfigureAwait();
//setup event handlers
connection.OnConnectionChange +=
(src, ev) => Scheduler.RunOnThreadPool(() => SetupMaintenanceConnection(logger));
connection.OnClusterChange += OnClusterChange;
//store the new connection
_maintenanceConnection = connection;
logger.LogInfo("Registered for cluster changes using {0}", connection);
}
//all seems right, we're done
return;
}
catch(Exception ex)
{
logger.LogWarning("Failed to setup maintenance connection: {0}", ex);
//temporary disconnect or registration failed, reset maintenance connection
_maintenanceConnection = null;
}
//don't retry if disposed
if(_disposed)
return;
//wait a moment, try again
logger.LogVerbose("Waiting 5secs before retrying setup maintenance connection");
await Task.Delay(5000).AutoConfigureAwait();
SetupMaintenanceConnection(logger);
}