private DatabaseHealth FailedDatabaseInstanceOrNode(
ClusterTopology clusterTopology,
string node,
string db,
Dictionary <string, ClusterNodeStatusReport> current)
{
if (clusterTopology.Contains(node) == false) // this node is no longer part of the *Cluster* topology and need to be replaced.
{
return(DatabaseHealth.Bad);
}
var hasCurrent = current.TryGetValue(node, out var currentNodeStats);
// Wait until we have more info
if (hasCurrent == false)
{
return(DatabaseHealth.NotEnoughInfo);
}
// if server is down we should reassign
if (DateTime.UtcNow - currentNodeStats.LastSuccessfulUpdateDateTime > _breakdownTimeout)
{
return(DatabaseHealth.Bad);
}
if (currentNodeStats.LastGoodDatabaseStatus.TryGetValue(db, out var lastGoodTime) == false)
{
// here we have a problem, the topology says that the db needs to be in the node, but the node
// doesn't know that the db is on it, that probably indicate some problem and we'll move it
// to another node to resolve it.
return(DatabaseHealth.NotEnoughInfo);
}
if (lastGoodTime == default(DateTime) || lastGoodTime == DateTime.MinValue)
{
return(DatabaseHealth.NotEnoughInfo);
}
return(DateTime.UtcNow - lastGoodTime > _breakdownTimeout ? DatabaseHealth.Bad : DatabaseHealth.Good);
}
private static void AssertCanAddNodeWithTopologyId(ClusterTopology clusterTopology, NodeInfo nodeInfo, string nodeUrl)
{
if (clusterTopology.TopologyId != nodeInfo.TopologyId)
{
throw new TopologyMismatchException(
$"Adding a new node to cluster failed. The new node is already in another cluster. " +
$"Expected topology id: {clusterTopology.TopologyId}, but we get {nodeInfo.TopologyId}");
}
if (nodeInfo.NodeTag != RachisConsensus.InitialTag && clusterTopology.Contains(nodeInfo.NodeTag) == false)
{
// this is fine, since we probably adding back a node that we just removed
return;
}
if (nodeInfo.CurrentState != RachisState.Passive)
{
throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " +
$"because it's already in the cluster under tag :{nodeInfo.NodeTag} " +
$"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}");
}
}