/// <summary> /// Opens the cluster for queries. Contains actual implementation and will be called only once per cluster /// </summary> /// <returns></returns> /// <exception cref="CqlException">Cannot construct ring from provided seeds!</exception> private async Task OpenAsyncInternal(Logger logger) { logger.LogInfo("Opening Cluster with parameters: {0}", _config.ToString()); //try to connect to the seeds in turn foreach (IPAddress seedAddress in _config.NodeAddresses) { try { var seed = new Node(seedAddress, this); _nodes = await DiscoverNodesAsync(seed, logger).ConfigureAwait(false); } catch (Exception ex) { //seed not reachable, try next logger.LogWarning("Could not discover nodes via seed {0}: {1}", seedAddress, ex); } } if (_nodes == null) { var ex = new CqlException("Cannot construct ring from provided seeds!"); logger.LogCritical("Unable to setup Cluster based on given configuration: {0}", ex); throw ex; } logger.LogInfo("Nodes detected: " + string.Join(", ", _nodes.Select(n => n.Address))); //setup cluster connection strategy switch (_config.ConnectionStrategy) { case ConnectionStrategy.Balanced: _connectionSelector = new BalancedConnectionStrategy(_nodes, _config); break; case ConnectionStrategy.Random: _connectionSelector = new RandomConnectionStrategy(_nodes, _config); break; case ConnectionStrategy.Exclusive: _connectionSelector = new ExclusiveConnectionStrategy(_nodes, _config); break; case ConnectionStrategy.PartitionAware: _connectionSelector = new PartitionAwareConnectionStrategy(_nodes, _config); if (_config.DiscoveryScope != DiscoveryScope.Cluster || _config.DiscoveryScope != DiscoveryScope.DataCenter) logger.LogWarning("PartitionAware connection strategy performs best if DiscoveryScope is set to cluster or datacenter"); break; } //setup throttle int concurrent = _config.MaxConcurrentQueries <= 0 ? _nodes.Count * _config.MaxConnectionsPerNode * 256 : _config.MaxConcurrentQueries; logger.LogInfo("Cluster is configured to allow {0} parallel queries", concurrent); _throttle = new SemaphoreSlim(concurrent, concurrent); //setup prepared query cache _prepareResultCache = new ConcurrentDictionary<string, ConcurrentDictionary<IPAddress, ResultFrame>>(); //setup maintenance connection SetupMaintenanceConnection(logger); }
/// <summary> /// Setups the maintenance channel. /// </summary> private async void SetupMaintenanceConnection(Logger logger) { try { if (_maintenanceConnection == null || !_maintenanceConnection.IsConnected) { //setup maintenance connection logger.LogVerbose("Creating new maintenance connection"); //pick a random node from the list var strategy = new RandomConnectionStrategy(_nodes, _config); //get or create a connection var connection = strategy.GetOrCreateConnection(null); //allow this connection to be used by others as well _connectionSelector.ReturnConnection(connection); //setup event handlers connection.OnConnectionChange += (src, ev) => SetupMaintenanceConnection(logger); connection.OnClusterChange += OnClusterChange; //store the new connection _maintenanceConnection = connection; //register for events await connection.RegisterForClusterChangesAsync(logger).ConfigureAwait(false); logger.LogInfo("Registered for cluster changes using {0}", connection); } //all seems right, we're done return; } catch (Exception ex) { logger.LogWarning("Failed to setup maintenance connection: {0}", ex); //temporary disconnect or registration failed, reset maintenance connection _maintenanceConnection = null; } //wait a moment, try again logger.LogVerbose("Waiting 2secs before retrying setup maintenance connection"); await Task.Delay(2000).ConfigureAwait(false); SetupMaintenanceConnection(logger); }
/// <summary> /// Gets all nodes that make up the cluster /// </summary> /// <param name="seed">The reference.</param> /// <param name="logger">logger used to log progress</param> /// <returns></returns> /// <exception cref="CqlException">Could not detect datacenter or rack information from the reference specified in the config section!</exception> private async Task<Ring> DiscoverNodesAsync(Node seed, Logger logger) { Connection c; using (logger.ThreadBinding()) { //get a connection c = seed.GetOrCreateConnection(null); } //get partitioner string partitioner; using (var result = await ExecQuery(c, "select partitioner from system.local", logger).ConfigureAwait(false)) { if (!await result.ReadAsync().ConfigureAwait(false)) throw new CqlException("Could not detect the cluster partitioner"); partitioner = (string)result[0]; } logger.LogInfo("Partitioner in use: {0}", partitioner); //get the "local" data center, rack and token using (var result = await ExecQuery(c, "select data_center, rack, tokens from system.local", logger).ConfigureAwait(false)) { if (await result.ReadAsync().ConfigureAwait(false)) { seed.DataCenter = (string)result["data_center"]; seed.Rack = (string)result["rack"]; seed.Tokens = (ISet<string>)result["tokens"]; logger.LogVerbose("Seed info - Address:{0} DataCenter:{1} Rack:{2}", seed.Address, seed.DataCenter, seed.Rack); } else { //strange, no local info found?! throw new CqlException("Could not detect datacenter or rack information from the reference specified in the config section!"); } } //create list of nodes that make up the cluster, and add the seed var found = new List<Node> { seed }; //get the peers using (var result = await ExecQuery(c, "select rpc_address, data_center, rack, tokens from system.peers", logger).ConfigureAwait(false)) { //iterate over the peers while (await result.ReadAsync().ConfigureAwait(false)) { //create a new node var newNode = new Node((IPAddress)result["rpc_address"], this) { DataCenter = (string)result["data_center"], Rack = (string)result["rack"], Tokens = (ISet<string>)result["tokens"] }; //add it if it is in scope if (InDiscoveryScope(seed, newNode, _config.DiscoveryScope)) found.Add(newNode); } } //return a new Ring of nodes return new Ring(found, partitioner); }
/// <summary> /// Opens the connection /// </summary> private async Task OpenAsyncInternal(Logger logger) { //switch state to connecting if not done so int state = Interlocked.CompareExchange(ref _connectionState, 1, 0); if (state == 1) return; if (state == 2) throw new ObjectDisposedException("Connection disposed before opening!"); try { //create TCP connection _client = new TcpClient(); await _client.ConnectAsync(_address, _cluster.Config.Port).ConfigureAwait(false); _writeStream = _client.GetStream(); _readStream = _client.GetStream(); logger.LogVerbose("TCP connection to {0} is opened", Address); //start readloop StartReadingAsync(); //get compression option _allowCompression = false; //assume false unless if (_cluster.Config.AllowCompression) { //check wether compression is supported by getting compression options from server var options = new OptionsFrame(); var supported = await SendRequestAsync(options, logger, 1, true).ConfigureAwait(false) as SupportedFrame; if (supported == null) throw new ProtocolException(0, "Expected Supported frame not received"); IList<string> compressionOptions; //check if options contain compression if (supported.SupportedOptions.TryGetValue("COMPRESSION", out compressionOptions)) { //check wether snappy is supported _allowCompression = compressionOptions.Contains("snappy"); } //dispose supported frame supported.Dispose(); } //submit startup frame var startup = new StartupFrame(_cluster.Config.CqlVersion); if (_allowCompression) { logger.LogVerbose("Enabling Snappy Compression."); startup.Options["COMPRESSION"] = "snappy"; } Frame response = await SendRequestAsync(startup, logger, 1, true).ConfigureAwait(false); //authenticate if required var auth = response as AuthenticateFrame; if (auth != null) { logger.LogVerbose("Authentication requested, attempting to provide credentials", Address); //check if _username is actually set if (_cluster.Config.Username == null || _cluster.Config.Password == null) throw new UnauthorizedException("No credentials provided"); //dispose AuthenticateFrame response.Dispose(); var cred = new CredentialsFrame(_cluster.Config.Username, _cluster.Config.Password); response = await SendRequestAsync(cred, logger, 1, true).ConfigureAwait(false); } //check if ready if (!(response is ReadyFrame)) throw new ProtocolException(0, "Expected Ready frame not received"); //dispose ready frame response.Dispose(); using (logger.ThreadBinding()) { if (OnConnectionChange != null) OnConnectionChange(this, new ConnectionChangeEvent { Connected = true }); } logger.LogInfo("{0} is opened and ready for use", this); } catch (Exception ex) { using (logger.ThreadBinding()) { Dispose(true, ex); throw; } } }
/// <summary> /// Opens the connection. Called once per connection only /// </summary> private async Task OpenAsyncInternal(Logger logger) { //set state to connecting int previousState = Interlocked.CompareExchange(ref _connectionState, ConnectionState.Connecting, ConnectionState.Created); if (previousState == ConnectionState.Closed) throw new ObjectDisposedException(ToString()); if (previousState != ConnectionState.Created) throw new InvalidOperationException("Opening a connection that is already connected!"); try { while(true) { //connect await ConnectAsync().AutoConfigureAwait(); //get streams Stream tcpStream = _client.GetStream(); _writeStream = tcpStream; _readStream = tcpStream; logger.LogVerbose("TCP connection for {0} is opened", this); //start readloop Scheduler.RunOnIOThread((Action)ReadFramesAsync); try { logger.LogVerbose("Attempting to connect using protocol version {0}", Node.ProtocolVersion); await NegotiateConnectionOptionsAsync(logger).AutoConfigureAwait(); break; } catch(ProtocolException pex) { //In case of a protocol version mismatch, Cassandra will reply with an error //using the supported protocol version. If we are using the correct version //something else is wrong, and it is no use to retry with a different version, //so rethrow if(Node.ProtocolVersion == pex.ProtocolVersion) throw; logger.LogVerbose( "Failed connecting using protocol version {0}, retrying with protocol version {1}...", Node.ProtocolVersion, pex.ProtocolVersion); //set protocol version to the one received Node.ProtocolVersion = pex.ProtocolVersion; //close the connection (required as protocols are not backwards compatible, so stream may be corrupt now) using(logger.ThreadBinding()) Disconnect(); //wait until the readloop has stopped _readLoopCompleted.Wait(); } } //run the startup message exchange await StartupAsync(logger).AutoConfigureAwait(); //yeah, connected previousState = Interlocked.CompareExchange(ref _connectionState, ConnectionState.Connected, ConnectionState.Connecting); if(previousState!=ConnectionState.Connecting) throw new ObjectDisposedException(ToString(), "Connection closed while opening"); //notify connection changed using(logger.ThreadBinding()) { if(OnConnectionChange != null) OnConnectionChange(this, new ConnectionChangeEvent {Connected = true}); } logger.LogInfo("{0} is opened using Cql Protocol v{1}", this, Node.ProtocolVersion); } catch (Exception) { using (logger.ThreadBinding()) { Close(true); throw; } } }
/// <summary> /// Gets all nodes that make up the cluster /// </summary> /// <param name="seed"> The reference. </param> /// <param name="logger"> logger used to log progress </param> /// <param name="token"> The token. </param> /// <returns> </returns> /// <exception cref="CqlException"> /// Could not detect datacenter or rack information from the reference specified in the /// config section! /// </exception> private async Task GetClusterInfoAsync(Node seed, Logger logger, CancellationToken token) { Connection c; using(logger.ThreadBinding()) { //get a connection if(seed != null && seed.IsUp) c = seed.GetOrCreateConnection(); else if(_maintenanceConnection != null && _maintenanceConnection.IsAvailable) { c = _maintenanceConnection; seed = c.Node; } else { c = _connectionStrategy.GetOrCreateConnection(ConnectionScope.Infrastructure, null); seed = c.Node; } } //get local information string partitioner; using(var result = await ExecQuery(c, "select cluster_name, cql_version, release_version, partitioner, data_center, rack, tokens from system.local", logger, token ).AutoConfigureAwait()) { if(! await result.ReadAsyncInternal(token).AutoConfigureAwait()) throw new CqlException("Could not fetch configuration data from seed"); _name = result.GetString(0); _cqlVersion = result.GetString(1); _release = result.GetString(2); partitioner = result.GetString(3); _dataCenter = seed.DataCenter = result.GetString(4); _rack = seed.Rack = result.GetString(5); seed.Tokens = result.GetSet<string>(6) ?? new HashSet<string>(); } logger.LogInfo( "Reconfigured cluster {0}: based on Cassandra Release {1}, supporting CqlVersion {2}, using partitioner '{3}'", _name, _release, _cqlVersion, partitioner); //create list of nodes that make up the cluster, and add the seed var found = new List<Node> {seed}; //get the peers using( var result = await ExecQuery(c, "select peer, rpc_address, data_center, rack, tokens, release_version from system.peers", logger, token).AutoConfigureAwait()) { //iterate over the peers while(await result.ReadAsyncInternal(token).AutoConfigureAwait()) { var newNode = GetNodeFromDataReader(result, logger); //add it if it is in scope if(InDiscoveryScope(seed, newNode, _config.DiscoveryScope)) found.Add(newNode); } } //set the new Ring of nodes _nodes.Update(found, partitioner, logger); //check if all tokens are received if(_nodes.Any(n => n.Tokens.Count == 0)) { //wait and retry the fetch later... Scheduler.RunOnThreadPool(async () => { try { logger.LogInfo("Cluster info incomplete scheduling new retrieval in 1 minute"); await Task.Delay(TimeSpan.FromMinutes(1)).AutoConfigureAwait(); await GetClusterInfoAsync(null, logger, CancellationToken.None).AutoConfigureAwait(); } catch(Exception ex) { logger.LogCritical("Critical error occured while updating cluster info: {0}", ex); } }); } }
/// <summary> /// Setups the maintenance channel. /// </summary> private async void SetupMaintenanceConnection(Logger logger) { //skip if disposed if(_disposed) return; try { if(_maintenanceConnection == null || !_maintenanceConnection.IsAvailable) { //setup maintenance connection logger.LogVerbose("Creating new maintenance connection"); //get or create a connection Connection connection; using(logger.ThreadBinding()) { connection = _connectionStrategy.GetOrCreateConnection(ConnectionScope.Infrastructure, null); } //check if we really got a connection if(connection == null) throw new CqlException("Can not obtain connection for maintenance channel"); //register for events await connection.RegisterForClusterChangesAsync(logger).AutoConfigureAwait(); //setup event handlers connection.OnConnectionChange += (src, ev) => Scheduler.RunOnThreadPool(() => SetupMaintenanceConnection(logger)); connection.OnClusterChange += OnClusterChange; //store the new connection _maintenanceConnection = connection; logger.LogInfo("Registered for cluster changes using {0}", connection); } //all seems right, we're done return; } catch(Exception ex) { logger.LogWarning("Failed to setup maintenance connection: {0}", ex); //temporary disconnect or registration failed, reset maintenance connection _maintenanceConnection = null; } //don't retry if disposed if(_disposed) return; //wait a moment, try again logger.LogVerbose("Waiting 5secs before retrying setup maintenance connection"); await Task.Delay(5000).AutoConfigureAwait(); SetupMaintenanceConnection(logger); }
/// <summary> /// Opens the cluster for queries. Contains actual implementation and will be called only once per cluster /// </summary> /// <returns> </returns> /// <exception cref="CqlException">Cannot construct ring from provided seeds!</exception> private async Task OpenAsyncInternal(Logger logger, CancellationToken token) { logger.LogInfo("Opening Cluster with parameters: {0}", _config.ToString()); //initialize the ring _nodes = new Ring(); //retry a few times to deal with bad network conditions for(int attempt = 0; attempt <= _config.MaxQueryRetries && _nodes.Count == 0; attempt++) { //try to connect to the seeds in turn foreach(IPAddress seedAddress in _config.ServerAddresses) { try { var seed = new Node(seedAddress, this); await GetClusterInfoAsync(seed, logger, token).AutoConfigureAwait(); //break from the loop as it seems we are done if(_nodes.Count > 0) break; } catch(OperationCanceledException) { logger.LogWarning("Opening connection to cluster was cancelled"); throw; } catch(ProtocolException pex) { logger.LogWarning("Could not open cluster via {0}: {1}", seedAddress, pex.Message); //node is not available, or starting up, try next, otherwise throw error if(pex.Code != ErrorCode.Overloaded && pex.Code != ErrorCode.IsBootstrapping) throw; } catch(SocketException ex) { //seed not reachable, try next logger.LogWarning("Could not open TCP connection to seed {0}: {1}", seedAddress, ex.Message); } catch(IOException ex) { //seed not reachable, try next logger.LogWarning("Could not discover nodes via seed {0}: {1}", seedAddress, ex); } } } //check if not disposed while opening if(_disposed) { foreach(var node in _nodes) node.Dispose(); throw new ObjectDisposedException("Cluster", "Cluster was disposed while opening"); } //check if we found any nodes if(_nodes.Count == 0) { var ex = new CqlException("Unable to connect to the cluster as none of the provided seeds is reachable."); logger.LogCritical("Unable to setup Cluster based on given configuration: {0}", ex); throw ex; } //setup cluster connection strategy switch(_config.ConnectionStrategy) { case CqlSharp.ConnectionStrategy.Balanced: _connectionStrategy = new BalancedConnectionStrategy(_nodes, _config); break; case CqlSharp.ConnectionStrategy.Random: _connectionStrategy = new RandomConnectionStrategy(_nodes, _config); break; case CqlSharp.ConnectionStrategy.Exclusive: _connectionStrategy = new ExclusiveConnectionStrategy(_nodes, _config); break; case CqlSharp.ConnectionStrategy.PartitionAware: _connectionStrategy = new PartitionAwareConnectionStrategy(_nodes, _config); if(_config.DiscoveryScope != DiscoveryScope.Cluster && _config.DiscoveryScope != DiscoveryScope.DataCenter) { logger.LogWarning( "PartitionAware connection strategy performs best if DiscoveryScope is set to cluster or datacenter"); } break; } //setup prepared query cache PreparedQueryCache = new ConcurrentDictionary<string, ResultFrame>(); //setup maintenance connection Scheduler.RunOnThreadPool(() => SetupMaintenanceConnection(logger)); }