private void RemoveOtherNodesIfNeeded(string dbName, DatabaseTopology topology, ref List <DeleteDatabaseCommand> deletions)
{
if (topology.Members.Count < topology.ReplicationFactor)
{
return;
}
if (topology.Promotables.Count == 0 &&
topology.Rehabs.Count == 0)
{
return;
}
if (_logger.IsOperationsEnabled)
{
_logger.Operations("We reached the replication factor, so we remove all other rehab/promotable nodes.");
}
var nodesToDelete = topology.Promotables.Concat(topology.Rehabs);
var deletionCmd = new DeleteDatabaseCommand
{
ErrorOnDatabaseDoesNotExists = false,
DatabaseName = dbName,
FromNodes = nodesToDelete.ToArray(),
HardDelete = _hardDeleteOnReplacement,
UpdateReplicationFactor = false
};
if (deletions == null)
{
deletions = new List <DeleteDatabaseCommand>();
}
deletions.Add(deletionCmd);
}
private void RemoveOtherNodesIfNeeded(string dbName, DatabaseRecord record, ClusterTopology clusterTopology,
Dictionary <string, ClusterNodeStatusReport> current, ref List <DeleteDatabaseCommand> deletions)
{
var topology = record.Topology;
if (topology.Members.Count < topology.ReplicationFactor)
{
return;
}
if (topology.Promotables.Count == 0 &&
topology.Rehabs.Count == 0)
{
return;
}
if (_logger.IsOperationsEnabled)
{
_logger.Operations("We reached the replication factor, so we try to remove redundant nodes.");
}
var nodesToDelete = new List <string>();
var mentorChangeVector = new Dictionary <string, string>();
foreach (var node in topology.Promotables.Concat(topology.Rehabs))
{
if (TryGetMentorNode(dbName, topology, clusterTopology, node, out var mentorNode) == false ||
current.TryGetValue(mentorNode, out var metorStats) == false ||
metorStats.Report.TryGetValue(dbName, out var dbReport) == false)
{
continue;
}
if (record.DeletionInProgress?.ContainsKey(node) == true)
{
continue;
}
nodesToDelete.Add(node);
mentorChangeVector.Add(node, dbReport.DatabaseChangeVector);
}
if (nodesToDelete.Count > 0)
{
var deletionCmd = new DeleteDatabaseCommand
{
ErrorOnDatabaseDoesNotExists = false,
DatabaseName = dbName,
FromNodes = nodesToDelete.ToArray(),
HardDelete = _hardDeleteOnReplacement,
UpdateReplicationFactor = false,
MentorChangeVector = mentorChangeVector
};
if (deletions == null)
{
deletions = new List <DeleteDatabaseCommand>();
}
deletions.Add(deletionCmd);
}
}
private Task <(long Index, object Result)> Delete(DeleteDatabaseCommand cmd)
{
if (_engine.LeaderTag != _server.NodeTag)
{
throw new NotLeadingException("This node is no longer the leader, so we abort the deletion command");
}
return(_engine.PutAsync(cmd));
}
public void DeleteDatabaseCommand_Verify_1()
{
Mock <IIrbisConnection> mock = GetConnectionMock();
IIrbisConnection connection = mock.Object;
DeleteDatabaseCommand command = new DeleteDatabaseCommand(connection);
Assert.IsFalse(command.Verify(false));
}
public void DeleteDatabaseCommand_CreateQuery_1()
{
Mock <IIrbisConnection> mock = GetConnectionMock();
IIrbisConnection connection = mock.Object;
DeleteDatabaseCommand command = new DeleteDatabaseCommand(connection);
command.CreateQuery();
}
public void DeleteDatabaseCommand_Construciton_1()
{
Mock <IIrbisConnection> mock = GetConnectionMock();
IIrbisConnection connection = mock.Object;
DeleteDatabaseCommand command = new DeleteDatabaseCommand(connection);
Assert.AreSame(connection, command.Connection);
}
public void DeleteDatabaseCommand_CreateQuery_2()
{
Mock <IIrbisConnection> mock = GetConnectionMock();
IIrbisConnection connection = mock.Object;
DeleteDatabaseCommand command = new DeleteDatabaseCommand(connection)
{
Database = "IBIS2"
};
ClientQuery query = command.CreateQuery();
Assert.IsNotNull(query);
}
private void RenderDelete(CodeDom.Database database, Entity entity)
{
var deleteCommand = new DeleteDatabaseCommand
{
Schema = entity.Schema,
TableName = entity.TableName,
AutoDeleteType = true,
Type = new CodeDom.TypeReference {
Namespace = entity.Namespace, Type = entity.Name
}
};
database.Commands.Add(deleteCommand);
}
public void DeleteDatabaseCommand_ExecuteRequest_1()
{
int returnCode = 0;
Mock <IIrbisConnection> mock = GetConnectionMock();
IIrbisConnection connection = mock.Object;
DeleteDatabaseCommand command = new DeleteDatabaseCommand(connection)
{
Database = "IBIS2"
};
ResponseBuilder builder = new ResponseBuilder()
.StandardHeader(CommandCode.DeleteDatabase, 123, 456)
.NewLine()
.Append(returnCode)
.NewLine();
TestingSocket socket = (TestingSocket)connection.Socket;
socket.Response = builder.Encode();
ClientQuery query = command.CreateQuery();
ServerResponse response = command.Execute(query);
Assert.AreEqual(returnCode, response.ReturnCode);
}
private string UpdateDatabaseTopology(string dbName, DatabaseRecord record, ClusterTopology clusterTopology,
Dictionary <string, ClusterNodeStatusReport> current,
Dictionary <string, ClusterNodeStatusReport> previous,
ref List <DeleteDatabaseCommand> deletions)
{
if (record.Disabled)
{
return(null);
}
var topology = record.Topology;
var hasLivingNodes = false;
foreach (var member in topology.Members)
{
var status = None;
if (current.TryGetValue(member, out var nodeStats) == false)
{
// there isn't much we can do here, except for log it.
if (previous.TryGetValue(member, out _))
{
// if we found this node in the previous report, we will ignore it this time and wait for the next report.
continue;
}
var msg =
$"The member node {member} was not found in both current and previous reports of the cluster observer. " +
$"If this error continue to raise, check the latency between the cluster nodes.";
if (_logger.IsInfoEnabled)
{
_logger.Info(msg);
}
RaiseNodeNotFoundAlert(msg, member);
continue;
}
if (nodeStats.Status == ClusterNodeStatusReport.ReportStatus.Ok &&
nodeStats.Report.TryGetValue(dbName, out var dbStats))
{
status = dbStats.Status;
if (status == Loaded ||
status == Loading ||
status == Unloaded)
{
hasLivingNodes = true;
if (topology.PromotablesStatus.TryGetValue(member, out var _))
{
topology.DemotionReasons.Remove(member);
topology.PromotablesStatus.Remove(member);
return($"Node {member} is online");
}
continue;
}
}
// Give one minute of grace before we move the node to a rehab
if (DateTime.UtcNow.AddMilliseconds(-_moveToRehabTime) < current[member]?.LastSuccessfulUpdateDateTime)
{
continue;
}
if (TryMoveToRehab(dbName, topology, current, member))
{
return($"Node {member} is currently not responding (with status: {status}) and moved to rehab");
}
// database distribution is off and the node is down
if (topology.DynamicNodesDistribution == false && (
topology.PromotablesStatus.TryGetValue(member, out var currentStatus) == false ||
currentStatus != DatabasePromotionStatus.NotResponding))
{
topology.DemotionReasons[member] = "Not responding";
topology.PromotablesStatus[member] = DatabasePromotionStatus.NotResponding;
return($"Node {member} is currently not responding with the status '{status}'");
}
}
if (hasLivingNodes == false)
{
var recoverable = new List <string>();
foreach (var rehab in topology.Rehabs)
{
if (FailedDatabaseInstanceOrNode(clusterTopology, rehab, dbName, current) == DatabaseHealth.Good)
{
recoverable.Add(rehab);
}
}
if (recoverable.Count > 0)
{
var node = FindMostUpToDateNode(recoverable, dbName, current);
topology.Rehabs.Remove(node);
topology.Members.Add(node);
RaiseNoLivingNodesAlert($"None of '{dbName}' database nodes are responding to the supervisor, promoting {node} from rehab to avoid making the database completely unreachable.", dbName);
return($"None of '{dbName}' nodes are responding, promoting {node} from rehab");
}
if (topology.Members.Count == 0 && record.DeletionInProgress?.Count > 0)
{
return(null); // We delete the whole database.
}
RaiseNoLivingNodesAlert($"None of '{dbName}' database nodes are responding to the supervisor, the database is unreachable.", dbName);
}
var shouldUpdateTopologyStatus = false;
var updateTopologyStatusReason = new StringBuilder();
foreach (var promotable in topology.Promotables)
{
if (FailedDatabaseInstanceOrNode(clusterTopology, promotable, dbName, current) == DatabaseHealth.Bad)
{
// database distribution is off and the node is down
if (topology.DynamicNodesDistribution == false)
{
if (topology.PromotablesStatus.TryGetValue(promotable, out var currentStatus) == false ||
currentStatus != DatabasePromotionStatus.NotResponding)
{
topology.DemotionReasons[promotable] = "Not responding";
topology.PromotablesStatus[promotable] = DatabasePromotionStatus.NotResponding;
return($"Node {promotable} is currently not responding");
}
continue;
}
if (TryFindFitNode(promotable, dbName, topology, clusterTopology, current, out var node) == false)
{
if (topology.PromotablesStatus.TryGetValue(promotable, out var currentStatus) == false ||
currentStatus != DatabasePromotionStatus.NotResponding)
{
topology.DemotionReasons[promotable] = "Not responding";
topology.PromotablesStatus[promotable] = DatabasePromotionStatus.NotResponding;
return($"Node {promotable} is currently not responding");
}
continue;
}
if (_server.LicenseManager.CanDynamicallyDistributeNodes(out _) == false)
{
continue;
}
// replace the bad promotable otherwise we will continue to add more and more nodes.
topology.Promotables.Add(node);
topology.DemotionReasons[node] = $"Just replaced the promotable node {promotable}";
topology.PromotablesStatus[node] = DatabasePromotionStatus.WaitingForFirstPromotion;
var deletionCmd = new DeleteDatabaseCommand
{
ErrorOnDatabaseDoesNotExists = false,
DatabaseName = dbName,
FromNodes = new[] { promotable },
HardDelete = _hardDeleteOnReplacement,
UpdateReplicationFactor = false
};
if (deletions == null)
{
deletions = new List <DeleteDatabaseCommand>();
}
deletions.Add(deletionCmd);
return($"The promotable {promotable} is not responsive, replace it with a node {node}");
}
if (TryGetMentorNode(dbName, topology, clusterTopology, promotable, out var mentorNode) == false)
{
continue;
}
var tryPromote = TryPromote(dbName, topology, current, previous, mentorNode, promotable);
if (tryPromote.Promote)
{
topology.Promotables.Remove(promotable);
topology.Members.Add(promotable);
topology.PredefinedMentors.Remove(promotable);
RemoveOtherNodesIfNeeded(dbName, record, clusterTopology, current, ref deletions);
return($"Promoting node {promotable} to member");
}
if (tryPromote.UpdateTopologyReason != null)
{
shouldUpdateTopologyStatus = true;
updateTopologyStatusReason.AppendLine(tryPromote.UpdateTopologyReason);
}
}
var goodMembers = GetNumberOfRespondingNodes(clusterTopology, dbName, topology, current);
var pendingDelete = GetPendingDeleteNodes(record);
foreach (var rehab in topology.Rehabs)
{
var health = FailedDatabaseInstanceOrNode(clusterTopology, rehab, dbName, current);
switch (health)
{
case DatabaseHealth.Bad:
if (topology.DynamicNodesDistribution == false)
{
continue;
}
if (goodMembers < topology.ReplicationFactor &&
TryFindFitNode(rehab, dbName, topology, clusterTopology, current, out var node))
{
if (_server.LicenseManager.CanDynamicallyDistributeNodes(out _) == false)
{
continue;
}
topology.Promotables.Add(node);
topology.DemotionReasons[node] = $"Maintain the replication factor and create new replica instead of node {rehab}";
topology.PromotablesStatus[node] = DatabasePromotionStatus.WaitingForFirstPromotion;
return($"The rehab node {rehab} was too long in rehabilitation, create node {node} to replace it");
}
if (topology.PromotablesStatus.TryGetValue(rehab, out var status) == false || status != DatabasePromotionStatus.NotResponding)
{
// was already online, but now we lost the connection again
if (TryMoveToRehab(dbName, topology, current, rehab))
{
return($"Node {rehab} is currently not responding");
}
}
break;
case DatabaseHealth.Good:
if (pendingDelete.Contains(rehab) && topology.PromotablesStatus.ContainsKey(rehab) == false)
{
// already tried to promote, so we just ignore and continue
continue;
}
if (TryGetMentorNode(dbName, topology, clusterTopology, rehab, out var mentorNode) == false)
{
continue;
}
var tryPromote = TryPromote(dbName, topology, current, previous, mentorNode, rehab);
if (tryPromote.Promote)
{
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"The database {dbName} on {rehab} is reachable and up to date, so we promote it back to member.");
}
topology.Members.Add(rehab);
topology.Rehabs.Remove(rehab);
RemoveOtherNodesIfNeeded(dbName, record, clusterTopology, current, ref deletions);
return($"Node {rehab} was recovered from rehabilitation and promoted back to member");
}
if (tryPromote.UpdateTopologyReason != null)
{
shouldUpdateTopologyStatus = true;
updateTopologyStatusReason.AppendLine(tryPromote.UpdateTopologyReason);
}
break;
}
}
RemoveOtherNodesIfNeeded(dbName, record, clusterTopology, current, ref deletions);
if (shouldUpdateTopologyStatus)
{
return(updateTopologyStatusReason.ToString());
}
return(null);
}
private string UpdateDatabaseTopology(string dbName, DatabaseRecord record, ClusterTopology clusterTopology,
Dictionary <string, ClusterNodeStatusReport> current,
Dictionary <string, ClusterNodeStatusReport> previous,
ref List <DeleteDatabaseCommand> deletions)
{
var topology = record.Topology;
var hasLivingNodes = false;
foreach (var member in topology.Members)
{
if (current.TryGetValue(member, out var nodeStats) &&
nodeStats.Status == ClusterNodeStatusReport.ReportStatus.Ok &&
nodeStats.Report.TryGetValue(dbName, out var dbStats) &&
dbStats.Status == Loaded)
{
hasLivingNodes = true;
if (topology.PromotablesStatus.TryGetValue(member, out var _))
{
topology.DemotionReasons.Remove(member);
topology.PromotablesStatus.Remove(member);
return($"Node {member} is online");
}
continue;
}
if (TryMoveToRehab(dbName, topology, current, member))
{
return($"Node {member} is currently not responding and moved to rehab");
}
// node distribution is off and the node is down
if (topology.DynamicNodesDistribution == false && (
topology.PromotablesStatus.TryGetValue(member, out var currentStatus) == false ||
currentStatus != DatabasePromotionStatus.NotResponding))
{
topology.DemotionReasons[member] = "Not responding";
topology.PromotablesStatus[member] = DatabasePromotionStatus.NotResponding;
return($"Node {member} is currently not responding");
}
}
if (hasLivingNodes == false)
{
var recoverable = new List <string>();
foreach (var rehab in topology.Rehabs)
{
if (FailedDatabaseInstanceOrNode(clusterTopology, rehab, dbName, current) == DatabaseHealth.Good)
{
recoverable.Add(rehab);
}
}
if (recoverable.Count > 0)
{
var node = FindMostUpToDateNode(recoverable, dbName, current);
topology.Rehabs.Remove(node);
topology.Members.Add(node);
RaiseNoLivingNodesAlert($"It appears that all nodes of the {dbName} database are not responding to the supervisor, promoting {node} from rehab to avoid making the database completely unreachable");
return($"All nodes are not responding, promoting {node} from rehab");
}
RaiseNoLivingNodesAlert($"It appears that all nodes of the {dbName} database are not responding to the supervisor, the database is not reachable");
}
var shouldUpdateTopologyStatus = false;
var updateTopologyStatusReason = new StringBuilder();
foreach (var promotable in topology.Promotables)
{
if (FailedDatabaseInstanceOrNode(clusterTopology, promotable, dbName, current) == DatabaseHealth.Bad)
{
// node distribution is off and the node is down
if (topology.DynamicNodesDistribution == false)
{
if (topology.PromotablesStatus.TryGetValue(promotable, out var currentStatus) == false ||
currentStatus != DatabasePromotionStatus.NotResponding)
{
topology.DemotionReasons[promotable] = "Not responding";
topology.PromotablesStatus[promotable] = DatabasePromotionStatus.NotResponding;
return($"Node {promotable} is currently not responding");
}
continue;
}
if (TryFindFitNode(promotable, dbName, topology, clusterTopology, current, out var node) == false)
{
if (topology.PromotablesStatus.TryGetValue(promotable, out var currentStatus) == false ||
currentStatus != DatabasePromotionStatus.NotResponding)
{
topology.DemotionReasons[promotable] = "Not responding";
topology.PromotablesStatus[promotable] = DatabasePromotionStatus.NotResponding;
return($"Node {promotable} is currently not responding");
}
continue;
}
if (_server.LicenseManager.CanDynamicallyDistributeNodes() == false)
{
continue;
}
// replace the bad promotable otherwise we will continue to add more and more nodes.
topology.Promotables.Add(node);
topology.DemotionReasons[node] = $"Just replaced the promotable node {promotable}";
topology.PromotablesStatus[node] = DatabasePromotionStatus.WaitingForFirstPromotion;
var deletionCmd = new DeleteDatabaseCommand
{
ErrorOnDatabaseDoesNotExists = false,
DatabaseName = dbName,
FromNodes = new[] { promotable },
HardDelete = _hardDeleteOnReplacement,
UpdateReplicationFactor = false
};
if (deletions == null)
{
deletions = new List <DeleteDatabaseCommand>();
}
deletions.Add(deletionCmd);
return($"The promotable {promotable} is not responsive, replace it with a node {node}");
}
if (TryGetMentorNode(dbName, topology, clusterTopology, promotable, out var mentorNode) == false)
{
continue;
}
var tryPromote = TryPromote(dbName, topology, current, previous, mentorNode, promotable);
if (tryPromote.Promote)
{
topology.Promotables.Remove(promotable);
topology.Members.Add(promotable);
topology.PredefinedMentors.Remove(promotable);
RemoveOtherNodesIfNeeded(dbName, record, clusterTopology, current, ref deletions);
return($"Promoting node {promotable} to member");
}
if (tryPromote.UpdateTopologyReason != null)
{
shouldUpdateTopologyStatus = true;
updateTopologyStatusReason.AppendLine(tryPromote.UpdateTopologyReason);
}
}
var goodMembers = GetNumberOfRespondingNodes(clusterTopology, dbName, topology, current);
var pendingDelete = GetPendingDeleteNodes(record);
foreach (var rehab in topology.Rehabs)
{
var health = FailedDatabaseInstanceOrNode(clusterTopology, rehab, dbName, current);
switch (health)
{
case DatabaseHealth.Bad:
if (topology.DynamicNodesDistribution == false)
{
continue;
}
if (goodMembers < topology.ReplicationFactor &&
TryFindFitNode(rehab, dbName, topology, clusterTopology, current, out var node))
{
if (_server.LicenseManager.CanDynamicallyDistributeNodes() == false)
{
continue;
}
topology.Promotables.Add(node);
topology.DemotionReasons[node] = $"Maintain the replication factor and create new replica instead of node {rehab}";
topology.PromotablesStatus[node] = DatabasePromotionStatus.WaitingForFirstPromotion;
return($"The rehab node {rehab} was too long in rehabilitation, create node {node} to replace it");
}
if (topology.PromotablesStatus.TryGetValue(rehab, out var status) == false || status != DatabasePromotionStatus.NotResponding)
{
// was already online, but now we lost the connection again
if (TryMoveToRehab(dbName, topology, current, rehab))
{
return($"Node {rehab} is currently not responding");
}
}
break;
case DatabaseHealth.Good:
if (pendingDelete.Contains(rehab) && topology.PromotablesStatus.ContainsKey(rehab) == false)
{
// already tried to promote, so we just ignore and continue
continue;
}
if (TryGetMentorNode(dbName, topology, clusterTopology, rehab, out var mentorNode) == false)
{
continue;
}
var tryPromote = TryPromote(dbName, topology, current, previous, mentorNode, rehab);
if (tryPromote.Promote)
{
if (pendingDelete.Contains(rehab))
{
var msg = $"Node {rehab} was recovered from rehabilitation, but was already replaced by an other node, so it's waiting for deletion.";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(msg);
}
return(msg);
}
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"The database {dbName} on {rehab} is reachable and update, so we promote it back to member.");
}
topology.Members.Add(rehab);
topology.Rehabs.Remove(rehab);
RemoveOtherNodesIfNeeded(dbName, record, clusterTopology, current, ref deletions);
return($"Node {rehab} was recovered from rehabilitation and promoted back to member");
}
if (tryPromote.UpdateTopologyReason != null)
{
shouldUpdateTopologyStatus = true;
updateTopologyStatusReason.AppendLine(tryPromote.UpdateTopologyReason);
}
break;
}
}
RemoveOtherNodesIfNeeded(dbName, record, clusterTopology, current, ref deletions);
if (shouldUpdateTopologyStatus)
{
return(updateTopologyStatusReason.ToString());
}
return(null);
}
public void DeleteDatabase(string name)
{
var deleteDatabaseCommand = new DeleteDatabaseCommand(name);
CouchProxy.Execute <CommandDefaultResult>(deleteDatabaseCommand);
}
