/// <summary> /// Executes a query. /// </summary> /// <param name="connection">The connection.</param> /// <param name="cql">The CQL.</param> /// <param name="logger">The logger.</param> /// <returns> /// A CqlDataReader that can be used to access the query results /// </returns> private async Task<CqlDataReader> ExecQuery(Connection connection, string cql, Logger logger) { logger.LogVerbose("Excuting query {0} on {1}", cql, connection); var query = new QueryFrame(cql, CqlConsistency.One); var result = (ResultFrame)await connection.SendRequestAsync(query, logger).ConfigureAwait(false); var reader = new CqlDataReader(result); logger.LogVerbose("Query {0} returned {1} results", cql, reader.Count); return reader; }
/// <summary> /// Setups the maintenance channel. /// </summary> private async void SetupMaintenanceConnection(Logger logger) { try { if (_maintenanceConnection == null || !_maintenanceConnection.IsConnected) { //setup maintenance connection logger.LogVerbose("Creating new maintenance connection"); //pick a random node from the list var strategy = new RandomConnectionStrategy(_nodes, _config); //get or create a connection var connection = strategy.GetOrCreateConnection(null); //allow this connection to be used by others as well _connectionSelector.ReturnConnection(connection); //setup event handlers connection.OnConnectionChange += (src, ev) => SetupMaintenanceConnection(logger); connection.OnClusterChange += OnClusterChange; //store the new connection _maintenanceConnection = connection; //register for events await connection.RegisterForClusterChangesAsync(logger).ConfigureAwait(false); logger.LogInfo("Registered for cluster changes using {0}", connection); } //all seems right, we're done return; } catch (Exception ex) { logger.LogWarning("Failed to setup maintenance connection: {0}", ex); //temporary disconnect or registration failed, reset maintenance connection _maintenanceConnection = null; } //wait a moment, try again logger.LogVerbose("Waiting 2secs before retrying setup maintenance connection"); await Task.Delay(2000).ConfigureAwait(false); SetupMaintenanceConnection(logger); }
/// <summary> /// Gets all nodes that make up the cluster /// </summary> /// <param name="seed">The reference.</param> /// <param name="logger">logger used to log progress</param> /// <returns></returns> /// <exception cref="CqlException">Could not detect datacenter or rack information from the reference specified in the config section!</exception> private async Task<Ring> DiscoverNodesAsync(Node seed, Logger logger) { Connection c; using (logger.ThreadBinding()) { //get a connection c = seed.GetOrCreateConnection(null); } //get partitioner string partitioner; using (var result = await ExecQuery(c, "select partitioner from system.local", logger).ConfigureAwait(false)) { if (!await result.ReadAsync().ConfigureAwait(false)) throw new CqlException("Could not detect the cluster partitioner"); partitioner = (string)result[0]; } logger.LogInfo("Partitioner in use: {0}", partitioner); //get the "local" data center, rack and token using (var result = await ExecQuery(c, "select data_center, rack, tokens from system.local", logger).ConfigureAwait(false)) { if (await result.ReadAsync().ConfigureAwait(false)) { seed.DataCenter = (string)result["data_center"]; seed.Rack = (string)result["rack"]; seed.Tokens = (ISet<string>)result["tokens"]; logger.LogVerbose("Seed info - Address:{0} DataCenter:{1} Rack:{2}", seed.Address, seed.DataCenter, seed.Rack); } else { //strange, no local info found?! throw new CqlException("Could not detect datacenter or rack information from the reference specified in the config section!"); } } //create list of nodes that make up the cluster, and add the seed var found = new List<Node> { seed }; //get the peers using (var result = await ExecQuery(c, "select rpc_address, data_center, rack, tokens from system.peers", logger).ConfigureAwait(false)) { //iterate over the peers while (await result.ReadAsync().ConfigureAwait(false)) { //create a new node var newNode = new Node((IPAddress)result["rpc_address"], this) { DataCenter = (string)result["data_center"], Rack = (string)result["rack"], Tokens = (ISet<string>)result["tokens"] }; //add it if it is in scope if (InDiscoveryScope(seed, newNode, _config.DiscoveryScope)) found.Add(newNode); } } //return a new Ring of nodes return new Ring(found, partitioner); }
/// <summary> /// Opens the connection /// </summary> private async Task OpenAsyncInternal(Logger logger) { //switch state to connecting if not done so int state = Interlocked.CompareExchange(ref _connectionState, 1, 0); if (state == 1) return; if (state == 2) throw new ObjectDisposedException("Connection disposed before opening!"); try { //create TCP connection _client = new TcpClient(); await _client.ConnectAsync(_address, _cluster.Config.Port).ConfigureAwait(false); _writeStream = _client.GetStream(); _readStream = _client.GetStream(); logger.LogVerbose("TCP connection to {0} is opened", Address); //start readloop StartReadingAsync(); //get compression option _allowCompression = false; //assume false unless if (_cluster.Config.AllowCompression) { //check wether compression is supported by getting compression options from server var options = new OptionsFrame(); var supported = await SendRequestAsync(options, logger, 1, true).ConfigureAwait(false) as SupportedFrame; if (supported == null) throw new ProtocolException(0, "Expected Supported frame not received"); IList<string> compressionOptions; //check if options contain compression if (supported.SupportedOptions.TryGetValue("COMPRESSION", out compressionOptions)) { //check wether snappy is supported _allowCompression = compressionOptions.Contains("snappy"); } //dispose supported frame supported.Dispose(); } //submit startup frame var startup = new StartupFrame(_cluster.Config.CqlVersion); if (_allowCompression) { logger.LogVerbose("Enabling Snappy Compression."); startup.Options["COMPRESSION"] = "snappy"; } Frame response = await SendRequestAsync(startup, logger, 1, true).ConfigureAwait(false); //authenticate if required var auth = response as AuthenticateFrame; if (auth != null) { logger.LogVerbose("Authentication requested, attempting to provide credentials", Address); //check if _username is actually set if (_cluster.Config.Username == null || _cluster.Config.Password == null) throw new UnauthorizedException("No credentials provided"); //dispose AuthenticateFrame response.Dispose(); var cred = new CredentialsFrame(_cluster.Config.Username, _cluster.Config.Password); response = await SendRequestAsync(cred, logger, 1, true).ConfigureAwait(false); } //check if ready if (!(response is ReadyFrame)) throw new ProtocolException(0, "Expected Ready frame not received"); //dispose ready frame response.Dispose(); using (logger.ThreadBinding()) { if (OnConnectionChange != null) OnConnectionChange(this, new ConnectionChangeEvent { Connected = true }); } logger.LogInfo("{0} is opened and ready for use", this); } catch (Exception ex) { using (logger.ThreadBinding()) { Dispose(true, ex); throw; } } }
/// <summary> /// Submits a frame, and waits until response is received /// </summary> /// <param name="frame"> The frame to send. </param> /// <param name="logger">logger to write progress to</param> /// <param name="load"> the load indication of the request. Used for balancing queries over nodes and connections </param> /// <param name="isConnecting">indicates if this request is send as part of connection setup protocol</param> /// <returns> </returns> internal async Task<Frame> SendRequestAsync(Frame frame, Logger logger, int load = 1, bool isConnecting = false) { try { //make sure we're already connected if (!isConnecting) await OpenAsync(logger).ConfigureAwait(false); //make sure we are connected if (!IsConnected) throw new IOException("Not connected"); //count the operation Interlocked.Increment(ref _activeRequests); //increase the load UpdateLoad(load, logger); logger.LogVerbose("Waiting for connection lock on {0}...", this); //wait until allowed to submit a frame await _frameSubmitLock.WaitAsync().ConfigureAwait(false); //get a task that gets completed when a response is received var waitTask = new TaskCompletionSource<Frame>(); //get a stream id, and store wait task under that id sbyte id; lock (_availableQueryIds) { id = _availableQueryIds.Dequeue(); _openRequests.Add(id, waitTask); } try { //send frame frame.Stream = id; //serialize frame outside lock Stream frameBytes = frame.GetFrameBytes(_allowCompression && !isConnecting, _cluster.Config.CompressionTreshold); await _writeLock.WaitAsync().ConfigureAwait(false); try { //final check to make sure we're connected if (_connectionState != 1) throw new IOException("Not connected"); logger.LogVerbose("Sending {0} Frame with Id {1}, to {2}", frame.OpCode, id, this); await frameBytes.CopyToAsync(_writeStream).ConfigureAwait(false); } finally { _writeLock.Release(); frameBytes.Dispose(); } //wait until response is received Frame response = await waitTask.Task.ConfigureAwait(false); logger.LogVerbose("{0} response for frame with Id {1} received from {2}", response.OpCode, id, Address); //throw error if result is an error var error = response as ErrorFrame; if (error != null) { throw error.Exception; } //return response return response; } finally { //return request slot to the pool lock (_availableQueryIds) { _openRequests.Remove(id); _availableQueryIds.Enqueue(id); } //allow another frame to be send _frameSubmitLock.Release(); //reduce load, we are done Interlocked.Decrement(ref _activeRequests); UpdateLoad(-load, logger); } } catch (ProtocolException pex) { switch (pex.Code) { case ErrorCode.IsBootstrapping: case ErrorCode.Overloaded: using (logger.ThreadBinding()) { //IO or node status related error, dispose this connection Dispose(true, pex); throw; } default: //some other Cql error (syntax ok?), simply rethrow throw; } } catch (Exception ex) { using (logger.ThreadBinding()) { //connection collapsed, dispose this connection Dispose(true, ex); throw; } } }
/// <summary> /// Sends the request async internal. Cancellation supported until request is send, after which answer must be handled /// to avoid connection corruption. /// </summary> /// <param name="frame"> The frame. </param> /// <param name="logger"> The logger. </param> /// <param name="load"> The load. </param> /// <param name="token"> The token. </param> /// <returns> </returns> /// <exception cref="System.IO.IOException">Not connected</exception> private async Task<Frame> SendRequestAsyncInternal(Frame frame, Logger logger, int load, CancellationToken token) { try { //make sure we aren't disposed if(_connectionState == ConnectionState.Closed) throw new ObjectDisposedException(ToString()); //count the operation Interlocked.Increment(ref _activeRequests); if (_connectionState == ConnectionState.Connected) { //increase the load UpdateLoad(load, logger); //wait until frame id is available to submit a frame logger.LogVerbose("Waiting for connection lock on {0}...", this); if (Scheduler.RunningSynchronously) _frameSubmitLock.Wait(token); else await _frameSubmitLock.WaitAsync(token).AutoConfigureAwait(); } //get a task that gets completed when a response is received var waitTask = new TaskCompletionSource<Frame>(); //get a stream id, and store wait task under that id short id; lock (_availableQueryIds) { id = _availableQueryIds.Count > 0 ? _availableQueryIds.Dequeue() : _usedQueryIds++; _openRequests.Add(id, waitTask); } try { //send frame frame.Stream = id; //set protocol version in use frame.ProtocolVersion = Node.ProtocolVersion; //serialize frame outside lock PoolMemoryStream frameBytes = frame.GetFrameBytes(_allowCompression && (_connectionState!=ConnectionState.Connecting), _config.CompressionTreshold); //wait to get access to stream if (Scheduler.RunningSynchronously) _writeLock.Wait(token); else await _writeLock.WaitAsync(token).AutoConfigureAwait(); try { //make very sure we aren't disposed if (_connectionState == ConnectionState.Closed) throw new ObjectDisposedException(ToString()); logger.LogVerbose("Sending {0} Frame with Id {1} over {2}", frame.OpCode, id, this); //write frame to stream, don't use cancelToken to prevent half-written frames if (Scheduler.RunningSynchronously) frameBytes.CopyTo(_writeStream); else await frameBytes.CopyToAsync(_writeStream).AutoConfigureAwait(); //unblock readloop to read result _readLock.Release(); } finally { _writeLock.Release(); frameBytes.Dispose(); } //wait until response is received Frame response = await waitTask.Task.AutoConfigureAwait(); logger.LogVerbose("Received {0} Frame with Id {1} on {2}", response.OpCode, id, this); //read frame content await response.ReadFrameContentAsync().AutoConfigureAwait(); //throw error if result is an error var error = response as ErrorFrame; if (error != null) { //dispose error frame error.Dispose(); //throw exception throw error.Exception; } //check for keyspace change var keyspaceChange = response as ResultFrame; if (keyspaceChange != null && keyspaceChange.CqlResultType == CqlResultType.SetKeyspace) { logger.LogVerbose("{0} changed KeySpace to \"{1}\"", this, keyspaceChange.Keyspace); CurrentKeySpace = keyspaceChange.Keyspace; } //dispose frame, when cancellation requested if (token.IsCancellationRequested) { response.Dispose(); throw new OperationCanceledException(token); } //return response return response; } finally { //return request slot to the pool lock (_availableQueryIds) { _openRequests.Remove(id); _availableQueryIds.Enqueue(id); } if (_connectionState == ConnectionState.Connected) { //allow another frame to be send _frameSubmitLock.Release(); //reduce load, we are done UpdateLoad(-load, logger); } //decrease the amount of operations Interlocked.Decrement(ref _activeRequests); } } catch (OperationCanceledException) { throw; } catch (ProtocolException pex) { switch (pex.Code) { case ErrorCode.IsBootstrapping: case ErrorCode.Overloaded: using (logger.ThreadBinding()) { //IO or node status related error, dispose this connection Close(true); throw; } default: //some other Cql error (syntax ok?), simply rethrow throw; } } catch (ObjectDisposedException odex) { throw new IOException("Connection closed while processing request", odex); } catch (Exception) { using (logger.ThreadBinding()) { //connection collapsed, dispose this connection Close(true); throw; } } }
/// <summary> /// Updates the load of this connection, and will trigger a corresponding event /// </summary> /// <param name="load">The load.</param> /// <param name="logger">The logger.</param> private void UpdateLoad(int load, Logger logger) { var newLoad = Interlocked.Add(ref _load, load); Interlocked.Exchange(ref _lastActivity, DateTime.Now.Ticks); EventHandler<LoadChangeEvent> handler = OnLoadChange; if (handler != null) handler(this, new LoadChangeEvent { LoadDelta = load }); logger.LogVerbose("{0} has now a load of {1}", this, newLoad); }
/// <summary> /// Authenticates the connection. /// </summary> /// <param name="auth">The authentication request from the server.</param> /// <param name="logger">The logger.</param> /// <returns></returns> /// <exception cref="AuthenticationException"> /// Unsupported Authenticator: + auth.Authenticator;null /// or /// Authentication failed, SASL Challenge was rejected by client /// or /// Authentication failed, Authenticator rejected SASL result /// or /// Expected a Authentication Challenge from Server! /// or /// No credentials provided in configuration /// or /// Authentication failed: Ready frame not received /// </exception> private async Task AuthenticateAsync(AuthenticateFrame auth, Logger logger) { logger.LogVerbose("Authentication requested, attempting to provide credentials"); //dispose AuthenticateFrame auth.Dispose(); if (auth.ProtocolVersion >= 2) { //protocol version2: use SASL AuthResponse to authenticate //get an AuthenticatorFactory IAuthenticatorFactory factory = Loader.Extensions.AuthenticationFactories.FirstOrDefault( f => f.Name.Equals(auth.Authenticator, StringComparison.OrdinalIgnoreCase)); if (factory == null) throw new AuthenticationException(auth.ProtocolVersion, "Unsupported Authenticator: " + auth.Authenticator); logger.LogVerbose("Attempting authentication for scheme {0}", factory.Name); //grab an authenticator instance IAuthenticator authenticator = factory.CreateAuthenticator(_config); //start authentication loop byte[] saslChallenge = null; while (true) { //check for challenge byte[] saslResponse; if (!authenticator.Authenticate(auth.ProtocolVersion, saslChallenge, out saslResponse)) { throw new AuthenticationException(auth.ProtocolVersion, "Authentication failed, SASL Challenge was rejected by client"); } //send response var cred = new AuthResponseFrame(saslResponse); var authResponse = await SendRequestAsyncInternal(cred, logger, 1, CancellationToken.None).AutoConfigureAwait(); //dispose authResponse (makes sure all is read) authResponse.Dispose(); //check for success var success = authResponse as AuthSuccessFrame; if (success != null) { if (!authenticator.Authenticate(auth.ProtocolVersion, success.SaslResult)) { throw new AuthenticationException(authResponse.ProtocolVersion, "Authentication failed, Authenticator rejected SASL result", authResponse.TracingId); } //yeah, authenticated, break from the authentication loop break; } //no success yet, lets try next round var challenge = authResponse as AuthChallengeFrame; if (challenge == null) { throw new AuthenticationException(authResponse.ProtocolVersion, "Expected a Authentication Challenge from Server!", authResponse.TracingId); } saslChallenge = challenge.SaslChallenge; } } else { //protocol version1: use Credentials to authenticate //check if _username is actually set if (_config.Username == null || _config.Password == null) throw new AuthenticationException(auth.ProtocolVersion, "No credentials provided in configuration"); var cred = new CredentialsFrame(_config.Username, _config.Password); var authResponse = await SendRequestAsyncInternal(cred, logger, 1, CancellationToken.None).AutoConfigureAwait(); //dispose authResponse (makes sure all is read) authResponse.Dispose(); if (!(authResponse is ReadyFrame)) { throw new AuthenticationException(authResponse.ProtocolVersion, "Authentication failed: Ready frame not received", authResponse.TracingId); } } }
/// <summary> /// Startups the connection using the required message exchange /// </summary> /// <param name="logger">The logger.</param> /// <returns></returns> /// <exception cref="ProtocolException">0;Expected Ready frame not received</exception> private async Task StartupAsync(Logger logger) { //submit startup frame var startup = new StartupFrame(_config.CqlVersion); if (_allowCompression) { logger.LogVerbose("Enabling Snappy Compression."); startup.Options["COMPRESSION"] = "snappy"; } Frame response = await SendRequestAsyncInternal(startup, logger, 1, CancellationToken.None).AutoConfigureAwait(); //authenticate if required var auth = response as AuthenticateFrame; if (auth != null) await AuthenticateAsync(auth, logger).AutoConfigureAwait(); //no authenticate frame, so ready frame must be received else if (!(response is ReadyFrame)) throw new ProtocolException(response.ProtocolVersion, 0, "Expected Ready frame not received", response.TracingId); //dispose ready frame response.Dispose(); }
/// <summary> /// Opens the connection. Called once per connection only /// </summary> private async Task OpenAsyncInternal(Logger logger) { //set state to connecting int previousState = Interlocked.CompareExchange(ref _connectionState, ConnectionState.Connecting, ConnectionState.Created); if (previousState == ConnectionState.Closed) throw new ObjectDisposedException(ToString()); if (previousState != ConnectionState.Created) throw new InvalidOperationException("Opening a connection that is already connected!"); try { while(true) { //connect await ConnectAsync().AutoConfigureAwait(); //get streams Stream tcpStream = _client.GetStream(); _writeStream = tcpStream; _readStream = tcpStream; logger.LogVerbose("TCP connection for {0} is opened", this); //start readloop Scheduler.RunOnIOThread((Action)ReadFramesAsync); try { logger.LogVerbose("Attempting to connect using protocol version {0}", Node.ProtocolVersion); await NegotiateConnectionOptionsAsync(logger).AutoConfigureAwait(); break; } catch(ProtocolException pex) { //In case of a protocol version mismatch, Cassandra will reply with an error //using the supported protocol version. If we are using the correct version //something else is wrong, and it is no use to retry with a different version, //so rethrow if(Node.ProtocolVersion == pex.ProtocolVersion) throw; logger.LogVerbose( "Failed connecting using protocol version {0}, retrying with protocol version {1}...", Node.ProtocolVersion, pex.ProtocolVersion); //set protocol version to the one received Node.ProtocolVersion = pex.ProtocolVersion; //close the connection (required as protocols are not backwards compatible, so stream may be corrupt now) using(logger.ThreadBinding()) Disconnect(); //wait until the readloop has stopped _readLoopCompleted.Wait(); } } //run the startup message exchange await StartupAsync(logger).AutoConfigureAwait(); //yeah, connected previousState = Interlocked.CompareExchange(ref _connectionState, ConnectionState.Connected, ConnectionState.Connecting); if(previousState!=ConnectionState.Connecting) throw new ObjectDisposedException(ToString(), "Connection closed while opening"); //notify connection changed using(logger.ThreadBinding()) { if(OnConnectionChange != null) OnConnectionChange(this, new ConnectionChangeEvent {Connected = true}); } logger.LogInfo("{0} is opened using Cql Protocol v{1}", this, Node.ProtocolVersion); } catch (Exception) { using (logger.ThreadBinding()) { Close(true); throw; } } }
/// <summary> /// Executes a query. /// </summary> /// <param name="connection"> The connection. </param> /// <param name="cql"> The CQL. </param> /// <param name="logger"> The logger. </param> /// <param name="token"> The token. </param> /// <returns> A CqlDataReader that can be used to access the query results </returns> private async Task<CqlDataReader> ExecQuery(Connection connection, string cql, Logger logger, CancellationToken token) { //cancel if requested token.ThrowIfCancellationRequested(); logger.LogVerbose("Excuting query {0} on {1}", cql, connection); var query = new QueryFrame(cql, CqlConsistency.One, null); var result = (ResultFrame) await connection.SendRequestAsync(query, logger, 1, token) .AutoConfigureAwait(); var reader = new CqlDataReader(null, result, null); logger.LogVerbose("Query {0} returned {1} results", cql, reader.Count); return reader; }
/// <summary> /// Setups the maintenance channel. /// </summary> private async void SetupMaintenanceConnection(Logger logger) { //skip if disposed if(_disposed) return; try { if(_maintenanceConnection == null || !_maintenanceConnection.IsAvailable) { //setup maintenance connection logger.LogVerbose("Creating new maintenance connection"); //get or create a connection Connection connection; using(logger.ThreadBinding()) { connection = _connectionStrategy.GetOrCreateConnection(ConnectionScope.Infrastructure, null); } //check if we really got a connection if(connection == null) throw new CqlException("Can not obtain connection for maintenance channel"); //register for events await connection.RegisterForClusterChangesAsync(logger).AutoConfigureAwait(); //setup event handlers connection.OnConnectionChange += (src, ev) => Scheduler.RunOnThreadPool(() => SetupMaintenanceConnection(logger)); connection.OnClusterChange += OnClusterChange; //store the new connection _maintenanceConnection = connection; logger.LogInfo("Registered for cluster changes using {0}", connection); } //all seems right, we're done return; } catch(Exception ex) { logger.LogWarning("Failed to setup maintenance connection: {0}", ex); //temporary disconnect or registration failed, reset maintenance connection _maintenanceConnection = null; } //don't retry if disposed if(_disposed) return; //wait a moment, try again logger.LogVerbose("Waiting 5secs before retrying setup maintenance connection"); await Task.Delay(5000).AutoConfigureAwait(); SetupMaintenanceConnection(logger); }