/// <summary> /// Request current status from server node. /// </summary> public void Refresh(Peers peers) { if (!active) { return; } try { if (tendConnection.IsClosed()) { tendConnection = cluster.CreateConnection(host.tlsName, address, cluster.connectionTimeout, null); if (cluster.user != null) { try { AdminCommand command = new AdminCommand(ThreadLocalData.GetBuffer(), 0); command.Authenticate(tendConnection, cluster.user, cluster.password); } catch (Exception) { tendConnection.Close(); throw; } } } if (peers.usePeers) { Dictionary <string, string> infoMap = Info.Request(tendConnection, "node", "peers-generation", "partition-generation"); VerifyNodeName(infoMap); VerifyPeersGeneration(infoMap, peers); VerifyPartitionGeneration(infoMap); } else { string[] commands = cluster.useServicesAlternate ? new string[] { "node", "partition-generation", "services-alternate" } : new string[] { "node", "partition-generation", "services" }; Dictionary <string, string> infoMap = Info.Request(tendConnection, commands); VerifyNodeName(infoMap); VerifyPartitionGeneration(infoMap); AddFriends(infoMap, peers); } peers.refreshCount++; failures = 0; } catch (Exception e) { if (peers.usePeers) { peers.genChanged = true; } RefreshFailed(e); } }
/// <summary> /// Request current status from server node. /// </summary> public void Refresh(Peers peers) { if (!active) { return; } try { if (tendConnection.IsClosed()) { tendConnection = CreateConnection(host.tlsName, address, cluster.connectionTimeout, null); if (cluster.user != null) { if (!EnsureLogin()) { AdminCommand command = new AdminCommand(ThreadLocalData.GetBuffer(), 0); if (!command.Authenticate(cluster, tendConnection, sessionToken)) { // Authentication failed. Session token probably expired. // Must login again to get new session token. command.Login(cluster, tendConnection, out sessionToken, out sessionExpiration); } } } } else { if (cluster.user != null) { EnsureLogin(); } } string[] commands = cluster.rackAware ? INFO_PERIODIC_REB : INFO_PERIODIC; Dictionary <string, string> infoMap = Info.Request(tendConnection, commands); VerifyNodeName(infoMap); VerifyPeersGeneration(infoMap, peers); VerifyPartitionGeneration(infoMap); if (cluster.rackAware) { VerifyRebalanceGeneration(infoMap); } peers.refreshCount++; failures = 0; } catch (Exception e) { peers.genChanged = true; RefreshFailed(e); } }
private void VerifyPeersGeneration(Dictionary <string, string> infoMap, Peers peers) { string genString = infoMap["peers-generation"]; if (genString == null || genString.Length == 0) { throw new AerospikeException.Parse("peers-generation is empty"); } int gen = Convert.ToInt32(genString); if (peersGeneration != gen) { peers.genChanged = true; } }
private bool PrepareFriend(Host host, Peers peers) { try { NodeValidator nv = new NodeValidator(); nv.ValidateNode(cluster, host); // Check for duplicate nodes in nodes slated to be added. Node node; if (peers.nodes.TryGetValue(nv.name, out node)) { // Duplicate node name found. This usually occurs when the server // services list contains both internal and external IP addresses // for the same node. nv.primaryConn.Close(); peers.hosts.Add(host); node.aliases.Add(host); return(true); } // Check for duplicate nodes in cluster. if (cluster.nodesMap.TryGetValue(nv.name, out node)) { nv.primaryConn.Close(); peers.hosts.Add(host); node.aliases.Add(host); node.referenceCount++; cluster.aliases[host] = node; return(true); } node = cluster.CreateNode(nv); peers.hosts.Add(host); peers.nodes[nv.name] = node; return(true); } catch (Exception e) { if (Log.WarnEnabled()) { Log.Warn("Add node " + host + " failed: " + Util.GetErrorMessage(e)); } return(false); } }
private void AddFriends(Dictionary <string, string> infoMap, Peers peers) { // Parse the service addresses and add the friends to the list. String command = cluster.useServicesAlternate ? "services-alternate" : "services"; string friendString = infoMap[command]; if (friendString == null || friendString.Length == 0) { peersCount = 0; return; } string[] friendNames = friendString.Split(';'); peersCount = friendNames.Length; foreach (string friend in friendNames) { string[] friendInfo = friend.Split(':'); string hostname = friendInfo[0]; string alternativeHost; if (cluster.ipMap != null && cluster.ipMap.TryGetValue(hostname, out alternativeHost)) { hostname = alternativeHost; } int port = Convert.ToInt32(friendInfo[1]); Host host = new Host(hostname, port); Node node; // Check global aliases for existing cluster. if (!cluster.aliases.TryGetValue(host, out node)) { // Check local aliases for this tend iteration. if (!peers.hosts.Contains(host)) { PrepareFriend(host, peers); } } else { node.referenceCount++; } } }
private bool ValidatePeers(Peers peers, Node node) { try { node.Refresh(peers); if (peers.genChanged) { peers.refreshCount = 0; node.RefreshPeers(peers); } } catch (Exception) { node.Close(); throw; } if (node.peersCount == 0) { // Node is suspect because multiple seeds are used and node does not have any peers. if (fallback == null) { fallback = node; } else { node.Close(); } return(false); } // Node is valid. Drop fallback if it exists. if (fallback != null) { if (Log.InfoEnabled()) { Log.Info("Skip orphan node: " + fallback); } fallback.Close(); fallback = null; } return(true); }
private static bool FindPeerNode(Cluster cluster, Peers peers, string nodeName) { // Check global node map for existing cluster. Node node; if (cluster.nodesMap.TryGetValue(nodeName, out node)) { node.referenceCount++; return(true); } // Check local node map for this tend iteration. if (peers.nodes.TryGetValue(nodeName, out node)) { node.referenceCount++; return(true); } return(false); }
/// <summary> /// Request current status from server node. /// </summary> public void Refresh(Peers peers) { if (!active) { return; } try { if (tendConnection.IsClosed()) { tendConnection = cluster.CreateConnection(host.tlsName, address, cluster.connectionTimeout); } if (peers.usePeers) { Dictionary <string, string> infoMap = Info.Request(tendConnection, "node", "peers-generation", "partition-generation"); VerifyNodeName(infoMap); VerifyPeersGeneration(infoMap, peers); VerifyPartitionGeneration(infoMap); } else { string[] commands = cluster.useServicesAlternate ? new string[] { "node", "partition-generation", "services-alternate" } : new string[] { "node", "partition-generation", "services" }; Dictionary <string, string> infoMap = Info.Request(tendConnection, commands); VerifyNodeName(infoMap); VerifyPartitionGeneration(infoMap); AddFriends(infoMap, peers); } peers.refreshCount++; failures = 0; } catch (Exception e) { RefreshFailed(e); } }
protected internal void RefreshPeers(Peers peers) { // Do not refresh peers when node connection has already failed during this cluster tend iteration. if (failures > 0 || !active) { return; } try { if (Log.DebugEnabled()) { Log.Debug("Update peers for node " + this); } PeerParser parser = new PeerParser(cluster, tendConnection, peers.peers); peersCount = peers.peers.Count; bool peersValidated = true; foreach (Peer peer in peers.peers) { if (FindPeerNode(cluster, peers, peer.nodeName)) { // Node already exists. Do not even try to connect to hosts. continue; } bool nodeValidated = false; // Find first host that connects. foreach (Host host in peer.hosts) { try { // Attempt connection to host. NodeValidator nv = new NodeValidator(); nv.ValidateNode(cluster, host); if (!peer.nodeName.Equals(nv.name)) { // Must look for new node name in the unlikely event that node names do not agree. if (Log.WarnEnabled()) { Log.Warn("Peer node " + peer.nodeName + " is different than actual node " + nv.name + " for host " + host); } if (FindPeerNode(cluster, peers, nv.name)) { // Node already exists. Do not even try to connect to hosts. nv.primaryConn.Close(); nodeValidated = true; break; } } // Create new node. Node node = cluster.CreateNode(nv); peers.nodes[nv.name] = node; nodeValidated = true; break; } catch (Exception e) { if (Log.WarnEnabled()) { Log.Warn("Add node " + host + " failed: " + Util.GetErrorMessage(e)); } } } if (!nodeValidated) { peersValidated = false; } } // Only set new peers generation if all referenced peers are added to the cluster. if (peersValidated) { peersGeneration = parser.generation; } peers.refreshCount++; } catch (Exception e) { RefreshFailed(e); } }
/// <summary> /// Check health of all nodes in the cluster. /// </summary> private void Tend(bool failIfNotConnected) { // All node additions/deletions are performed in tend thread. // If active nodes don't exist, seed cluster. if (nodes.Length == 0) { SeedNodes(failIfNotConnected); } // Initialize tend iteration node statistics. Peers peers = new Peers(nodes.Length + 16); // Clear node reference counts. foreach (Node node in nodes) { node.referenceCount = 0; node.partitionChanged = false; node.rebalanceChanged = false; if (!node.HasPeers) { peers.usePeers = false; } } // Refresh all known nodes. foreach (Node node in nodes) { node.Refresh(peers); } // Refresh peers when necessary. if (peers.genChanged) { // Refresh peers for all nodes that responded the first time even if only one node's peers changed. peers.refreshCount = 0; foreach (Node node in nodes) { node.RefreshPeers(peers); } } // Refresh partition map when necessary. foreach (Node node in nodes) { if (node.partitionChanged) { node.RefreshPartitions(peers); } if (node.rebalanceChanged) { node.RefreshRacks(); } } if (peers.genChanged || !peers.usePeers) { // Handle nodes changes determined from refreshes. List <Node> removeList = FindNodesToRemove(peers.refreshCount); // Remove nodes in a batch. if (removeList.Count > 0) { RemoveNodes(removeList); } } // Add nodes in a batch. if (peers.nodes.Count > 0) { AddNodes(peers.nodes); } }
private bool SeedNode(Peers peers, bool failIfNotConnected) { // Must copy array reference for copy on write semantics to work. Host[] seedArray = seeds; Exception[] exceptions = null; NodeValidator nv = new NodeValidator(); for (int i = 0; i < seedArray.Length; i++) { Host seed = seedArray[i]; try { Node node = nv.SeedNode(this, seed, peers); if (node != null) { AddNode(node); return(true); } } catch (Exception e) { // Store exception and try next seed. if (failIfNotConnected) { if (exceptions == null) { exceptions = new Exception[seedArray.Length]; } exceptions[i] = e; } else { if (Log.WarnEnabled()) { Log.Warn("Seed " + seed + " failed: " + Util.GetErrorMessage(e)); } } } } // No seeds valid. Use fallback node if it exists. if (nv.fallback != null) { AddNode(nv.fallback); return(true); } if (failIfNotConnected) { StringBuilder sb = new StringBuilder(500); sb.AppendLine("Failed to connect to host(s): "); for (int i = 0; i < seedArray.Length; i++) { sb.Append(seedArray[i]); sb.Append(' '); Exception ex = exceptions[i]; if (ex != null) { sb.AppendLine(ex.Message); } } throw new AerospikeException.Connection(sb.ToString()); } return(false); }
/// <summary> /// Check health of all nodes in the cluster. /// </summary> private void Tend(bool failIfNotConnected) { // Initialize tend iteration node statistics. Peers peers = new Peers(nodes.Length + 16); // Clear node reference counts. foreach (Node node in nodes) { node.referenceCount = 0; node.partitionChanged = false; node.rebalanceChanged = false; } // All node additions/deletions are performed in tend thread. // If active nodes don't exist, seed cluster. if (nodes.Length == 0) { SeedNode(peers, failIfNotConnected); } else { // Refresh all known nodes. foreach (Node node in nodes) { node.Refresh(peers); } // Refresh peers when necessary. if (peers.genChanged) { // Refresh peers for all nodes that responded the first time even if only one node's peers changed. peers.refreshCount = 0; foreach (Node node in nodes) { node.RefreshPeers(peers); } } } // Refresh partition map when necessary. foreach (Node node in nodes) { if (node.partitionChanged) { node.RefreshPartitions(peers); } if (node.rebalanceChanged) { node.RefreshRacks(); } } if (peers.genChanged) { // Handle nodes changes determined from refreshes. List <Node> removeList = FindNodesToRemove(peers.refreshCount); // Remove nodes in a batch. if (removeList.Count > 0) { RemoveNodes(removeList); } } // Add nodes in a batch. if (peers.nodes.Count > 0) { AddNodes(peers.nodes); } tendCount++; // Balance connections every 30 tend intervals. if (tendCount % 30 == 0) { foreach (Node node in nodes) { if (node.ErrorCountWithinLimit()) { node.BalanceConnections(); } } } // Reset connection error window for all nodes every connErrorWindow tend iterations. if (maxErrorRate > 0 && tendCount % errorRateWindow == 0) { foreach (Node node in nodes) { node.ResetErrorCount(); } } }