/// <summary>
/// Request current status from server node.
/// </summary>
public void Refresh(Peers peers)
{
if (!active)
{
return;
}
try
{
if (tendConnection.IsClosed())
{
tendConnection = cluster.CreateConnection(host.tlsName, address, cluster.connectionTimeout, null);
if (cluster.user != null)
{
try
{
AdminCommand command = new AdminCommand(ThreadLocalData.GetBuffer(), 0);
command.Authenticate(tendConnection, cluster.user, cluster.password);
}
catch (Exception)
{
tendConnection.Close();
throw;
}
}
}
if (peers.usePeers)
{
Dictionary <string, string> infoMap = Info.Request(tendConnection, "node", "peers-generation", "partition-generation");
VerifyNodeName(infoMap);
VerifyPeersGeneration(infoMap, peers);
VerifyPartitionGeneration(infoMap);
}
else
{
string[] commands = cluster.useServicesAlternate ?
new string[] { "node", "partition-generation", "services-alternate" } :
new string[] { "node", "partition-generation", "services" };
Dictionary <string, string> infoMap = Info.Request(tendConnection, commands);
VerifyNodeName(infoMap);
VerifyPartitionGeneration(infoMap);
AddFriends(infoMap, peers);
}
peers.refreshCount++;
failures = 0;
}
catch (Exception e)
{
if (peers.usePeers)
{
peers.genChanged = true;
}
RefreshFailed(e);
}
}
/// <summary>
/// Request current status from server node.
/// </summary>
public void Refresh(Peers peers)
{
if (!active)
{
return;
}
try
{
if (tendConnection.IsClosed())
{
tendConnection = CreateConnection(host.tlsName, address, cluster.connectionTimeout, null);
if (cluster.user != null)
{
if (!EnsureLogin())
{
AdminCommand command = new AdminCommand(ThreadLocalData.GetBuffer(), 0);
if (!command.Authenticate(cluster, tendConnection, sessionToken))
{
// Authentication failed. Session token probably expired.
// Must login again to get new session token.
command.Login(cluster, tendConnection, out sessionToken, out sessionExpiration);
}
}
}
}
else
{
if (cluster.user != null)
{
EnsureLogin();
}
}
string[] commands = cluster.rackAware ? INFO_PERIODIC_REB : INFO_PERIODIC;
Dictionary <string, string> infoMap = Info.Request(tendConnection, commands);
VerifyNodeName(infoMap);
VerifyPeersGeneration(infoMap, peers);
VerifyPartitionGeneration(infoMap);
if (cluster.rackAware)
{
VerifyRebalanceGeneration(infoMap);
}
peers.refreshCount++;
failures = 0;
}
catch (Exception e)
{
peers.genChanged = true;
RefreshFailed(e);
}
}
private void VerifyPeersGeneration(Dictionary <string, string> infoMap, Peers peers)
{
string genString = infoMap["peers-generation"];
if (genString == null || genString.Length == 0)
{
throw new AerospikeException.Parse("peers-generation is empty");
}
int gen = Convert.ToInt32(genString);
if (peersGeneration != gen)
{
peers.genChanged = true;
}
}
private bool PrepareFriend(Host host, Peers peers)
{
try
{
NodeValidator nv = new NodeValidator();
nv.ValidateNode(cluster, host);
// Check for duplicate nodes in nodes slated to be added.
Node node;
if (peers.nodes.TryGetValue(nv.name, out node))
{
// Duplicate node name found. This usually occurs when the server
// services list contains both internal and external IP addresses
// for the same node.
nv.primaryConn.Close();
peers.hosts.Add(host);
node.aliases.Add(host);
return(true);
}
// Check for duplicate nodes in cluster.
if (cluster.nodesMap.TryGetValue(nv.name, out node))
{
nv.primaryConn.Close();
peers.hosts.Add(host);
node.aliases.Add(host);
node.referenceCount++;
cluster.aliases[host] = node;
return(true);
}
node = cluster.CreateNode(nv);
peers.hosts.Add(host);
peers.nodes[nv.name] = node;
return(true);
}
catch (Exception e)
{
if (Log.WarnEnabled())
{
Log.Warn("Add node " + host + " failed: " + Util.GetErrorMessage(e));
}
return(false);
}
}
private void AddFriends(Dictionary <string, string> infoMap, Peers peers)
{
// Parse the service addresses and add the friends to the list.
String command = cluster.useServicesAlternate ? "services-alternate" : "services";
string friendString = infoMap[command];
if (friendString == null || friendString.Length == 0)
{
peersCount = 0;
return;
}
string[] friendNames = friendString.Split(';');
peersCount = friendNames.Length;
foreach (string friend in friendNames)
{
string[] friendInfo = friend.Split(':');
string hostname = friendInfo[0];
string alternativeHost;
if (cluster.ipMap != null && cluster.ipMap.TryGetValue(hostname, out alternativeHost))
{
hostname = alternativeHost;
}
int port = Convert.ToInt32(friendInfo[1]);
Host host = new Host(hostname, port);
Node node;
// Check global aliases for existing cluster.
if (!cluster.aliases.TryGetValue(host, out node))
{
// Check local aliases for this tend iteration.
if (!peers.hosts.Contains(host))
{
PrepareFriend(host, peers);
}
}
else
{
node.referenceCount++;
}
}
}
private bool ValidatePeers(Peers peers, Node node)
{
try
{
node.Refresh(peers);
if (peers.genChanged)
{
peers.refreshCount = 0;
node.RefreshPeers(peers);
}
}
catch (Exception)
{
node.Close();
throw;
}
if (node.peersCount == 0)
{
// Node is suspect because multiple seeds are used and node does not have any peers.
if (fallback == null)
{
fallback = node;
}
else
{
node.Close();
}
return(false);
}
// Node is valid. Drop fallback if it exists.
if (fallback != null)
{
if (Log.InfoEnabled())
{
Log.Info("Skip orphan node: " + fallback);
}
fallback.Close();
fallback = null;
}
return(true);
}
private static bool FindPeerNode(Cluster cluster, Peers peers, string nodeName)
{
// Check global node map for existing cluster.
Node node;
if (cluster.nodesMap.TryGetValue(nodeName, out node))
{
node.referenceCount++;
return(true);
}
// Check local node map for this tend iteration.
if (peers.nodes.TryGetValue(nodeName, out node))
{
node.referenceCount++;
return(true);
}
return(false);
}
/// <summary>
/// Request current status from server node.
/// </summary>
public void Refresh(Peers peers)
{
if (!active)
{
return;
}
try
{
if (tendConnection.IsClosed())
{
tendConnection = cluster.CreateConnection(host.tlsName, address, cluster.connectionTimeout);
}
if (peers.usePeers)
{
Dictionary <string, string> infoMap = Info.Request(tendConnection, "node", "peers-generation", "partition-generation");
VerifyNodeName(infoMap);
VerifyPeersGeneration(infoMap, peers);
VerifyPartitionGeneration(infoMap);
}
else
{
string[] commands = cluster.useServicesAlternate ?
new string[] { "node", "partition-generation", "services-alternate" } :
new string[] { "node", "partition-generation", "services" };
Dictionary <string, string> infoMap = Info.Request(tendConnection, commands);
VerifyNodeName(infoMap);
VerifyPartitionGeneration(infoMap);
AddFriends(infoMap, peers);
}
peers.refreshCount++;
failures = 0;
}
catch (Exception e)
{
RefreshFailed(e);
}
}
protected internal void RefreshPeers(Peers peers)
{
// Do not refresh peers when node connection has already failed during this cluster tend iteration.
if (failures > 0 || !active)
{
return;
}
try
{
if (Log.DebugEnabled())
{
Log.Debug("Update peers for node " + this);
}
PeerParser parser = new PeerParser(cluster, tendConnection, peers.peers);
peersCount = peers.peers.Count;
bool peersValidated = true;
foreach (Peer peer in peers.peers)
{
if (FindPeerNode(cluster, peers, peer.nodeName))
{
// Node already exists. Do not even try to connect to hosts.
continue;
}
bool nodeValidated = false;
// Find first host that connects.
foreach (Host host in peer.hosts)
{
try
{
// Attempt connection to host.
NodeValidator nv = new NodeValidator();
nv.ValidateNode(cluster, host);
if (!peer.nodeName.Equals(nv.name))
{
// Must look for new node name in the unlikely event that node names do not agree.
if (Log.WarnEnabled())
{
Log.Warn("Peer node " + peer.nodeName + " is different than actual node " + nv.name + " for host " + host);
}
if (FindPeerNode(cluster, peers, nv.name))
{
// Node already exists. Do not even try to connect to hosts.
nv.primaryConn.Close();
nodeValidated = true;
break;
}
}
// Create new node.
Node node = cluster.CreateNode(nv);
peers.nodes[nv.name] = node;
nodeValidated = true;
break;
}
catch (Exception e)
{
if (Log.WarnEnabled())
{
Log.Warn("Add node " + host + " failed: " + Util.GetErrorMessage(e));
}
}
}
if (!nodeValidated)
{
peersValidated = false;
}
}
// Only set new peers generation if all referenced peers are added to the cluster.
if (peersValidated)
{
peersGeneration = parser.generation;
}
peers.refreshCount++;
}
catch (Exception e)
{
RefreshFailed(e);
}
}
/// <summary>
/// Check health of all nodes in the cluster.
/// </summary>
private void Tend(bool failIfNotConnected)
{
// All node additions/deletions are performed in tend thread.
// If active nodes don't exist, seed cluster.
if (nodes.Length == 0)
{
SeedNodes(failIfNotConnected);
}
// Initialize tend iteration node statistics.
Peers peers = new Peers(nodes.Length + 16);
// Clear node reference counts.
foreach (Node node in nodes)
{
node.referenceCount = 0;
node.partitionChanged = false;
node.rebalanceChanged = false;
if (!node.HasPeers)
{
peers.usePeers = false;
}
}
// Refresh all known nodes.
foreach (Node node in nodes)
{
node.Refresh(peers);
}
// Refresh peers when necessary.
if (peers.genChanged)
{
// Refresh peers for all nodes that responded the first time even if only one node's peers changed.
peers.refreshCount = 0;
foreach (Node node in nodes)
{
node.RefreshPeers(peers);
}
}
// Refresh partition map when necessary.
foreach (Node node in nodes)
{
if (node.partitionChanged)
{
node.RefreshPartitions(peers);
}
if (node.rebalanceChanged)
{
node.RefreshRacks();
}
}
if (peers.genChanged || !peers.usePeers)
{
// Handle nodes changes determined from refreshes.
List <Node> removeList = FindNodesToRemove(peers.refreshCount);
// Remove nodes in a batch.
if (removeList.Count > 0)
{
RemoveNodes(removeList);
}
}
// Add nodes in a batch.
if (peers.nodes.Count > 0)
{
AddNodes(peers.nodes);
}
}
private bool SeedNode(Peers peers, bool failIfNotConnected)
{
// Must copy array reference for copy on write semantics to work.
Host[] seedArray = seeds;
Exception[] exceptions = null;
NodeValidator nv = new NodeValidator();
for (int i = 0; i < seedArray.Length; i++)
{
Host seed = seedArray[i];
try
{
Node node = nv.SeedNode(this, seed, peers);
if (node != null)
{
AddNode(node);
return(true);
}
}
catch (Exception e)
{
// Store exception and try next seed.
if (failIfNotConnected)
{
if (exceptions == null)
{
exceptions = new Exception[seedArray.Length];
}
exceptions[i] = e;
}
else
{
if (Log.WarnEnabled())
{
Log.Warn("Seed " + seed + " failed: " + Util.GetErrorMessage(e));
}
}
}
}
// No seeds valid. Use fallback node if it exists.
if (nv.fallback != null)
{
AddNode(nv.fallback);
return(true);
}
if (failIfNotConnected)
{
StringBuilder sb = new StringBuilder(500);
sb.AppendLine("Failed to connect to host(s): ");
for (int i = 0; i < seedArray.Length; i++)
{
sb.Append(seedArray[i]);
sb.Append(' ');
Exception ex = exceptions[i];
if (ex != null)
{
sb.AppendLine(ex.Message);
}
}
throw new AerospikeException.Connection(sb.ToString());
}
return(false);
}
/// <summary>
/// Check health of all nodes in the cluster.
/// </summary>
private void Tend(bool failIfNotConnected)
{
// Initialize tend iteration node statistics.
Peers peers = new Peers(nodes.Length + 16);
// Clear node reference counts.
foreach (Node node in nodes)
{
node.referenceCount = 0;
node.partitionChanged = false;
node.rebalanceChanged = false;
}
// All node additions/deletions are performed in tend thread.
// If active nodes don't exist, seed cluster.
if (nodes.Length == 0)
{
SeedNode(peers, failIfNotConnected);
}
else
{
// Refresh all known nodes.
foreach (Node node in nodes)
{
node.Refresh(peers);
}
// Refresh peers when necessary.
if (peers.genChanged)
{
// Refresh peers for all nodes that responded the first time even if only one node's peers changed.
peers.refreshCount = 0;
foreach (Node node in nodes)
{
node.RefreshPeers(peers);
}
}
}
// Refresh partition map when necessary.
foreach (Node node in nodes)
{
if (node.partitionChanged)
{
node.RefreshPartitions(peers);
}
if (node.rebalanceChanged)
{
node.RefreshRacks();
}
}
if (peers.genChanged)
{
// Handle nodes changes determined from refreshes.
List <Node> removeList = FindNodesToRemove(peers.refreshCount);
// Remove nodes in a batch.
if (removeList.Count > 0)
{
RemoveNodes(removeList);
}
}
// Add nodes in a batch.
if (peers.nodes.Count > 0)
{
AddNodes(peers.nodes);
}
tendCount++;
// Balance connections every 30 tend intervals.
if (tendCount % 30 == 0)
{
foreach (Node node in nodes)
{
if (node.ErrorCountWithinLimit())
{
node.BalanceConnections();
}
}
}
// Reset connection error window for all nodes every connErrorWindow tend iterations.
if (maxErrorRate > 0 && tendCount % errorRateWindow == 0)
{
foreach (Node node in nodes)
{
node.ResetErrorCount();
}
}
}
