private DatabaseHealth FailedDatabaseInstanceOrNode( ClusterTopology clusterTopology, string node, string db, Dictionary <string, ClusterNodeStatusReport> current) { if (clusterTopology.Contains(node) == false) // this node is no longer part of the *Cluster* topology and need to be replaced. { return(DatabaseHealth.Bad); } var hasCurrent = current.TryGetValue(node, out var currentNodeStats); // Wait until we have more info if (hasCurrent == false) { return(DatabaseHealth.NotEnoughInfo); } // if server is down we should reassign if (DateTime.UtcNow - currentNodeStats.LastSuccessfulUpdateDateTime > _breakdownTimeout) { return(DatabaseHealth.Bad); } if (currentNodeStats.LastGoodDatabaseStatus.TryGetValue(db, out var lastGoodTime) == false) { // here we have a problem, the topology says that the db needs to be in the node, but the node // doesn't know that the db is on it, that probably indicate some problem and we'll move it // to another node to resolve it. return(DatabaseHealth.NotEnoughInfo); } if (lastGoodTime == default(DateTime) || lastGoodTime == DateTime.MinValue) { return(DatabaseHealth.NotEnoughInfo); } return(DateTime.UtcNow - lastGoodTime > _breakdownTimeout ? DatabaseHealth.Bad : DatabaseHealth.Good); }
private static void AssertCanAddNodeWithTopologyId(ClusterTopology clusterTopology, NodeInfo nodeInfo, string nodeUrl) { if (clusterTopology.TopologyId != nodeInfo.TopologyId) { throw new TopologyMismatchException( $"Adding a new node to cluster failed. The new node is already in another cluster. " + $"Expected topology id: {clusterTopology.TopologyId}, but we get {nodeInfo.TopologyId}"); } if (nodeInfo.NodeTag != RachisConsensus.InitialTag && clusterTopology.Contains(nodeInfo.NodeTag) == false) { // this is fine, since we probably adding back a node that we just removed return; } if (nodeInfo.CurrentState != RachisState.Passive) { throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " + $"because it's already in the cluster under tag :{nodeInfo.NodeTag} " + $"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}"); } }