internal static void EvictStoppedNodes(DatabaseAvailabilityGroup dag, IEnumerable <Server> stoppedServers, HaTaskOutputHelper output)
{
if (dag == null)
{
throw new ArgumentNullException("dag");
}
if (stoppedServers == null)
{
throw new ArgumentNullException("stoppedServers");
}
if (output == null)
{
throw new ArgumentNullException("output");
}
string error = null;
List <string> list = new List <string>(1);
string verboseLog = null;
using (AmCluster amCluster = AmCluster.OpenDagClus(dag))
{
using (IAmClusterGroup amClusterGroup = amCluster.FindCoreClusterGroup())
{
output.AppendLogMessage("EvictStoppedNodes has been called. Dumping current cluster state.", new object[0]);
try
{
using (DumpClusterTopology dumpClusterTopology = new DumpClusterTopology(amCluster, output))
{
dumpClusterTopology.Dump();
}
}
catch (ClusterException ex)
{
output.AppendLogMessage("DumpClusterTopology( {0} ) failed with exception = {1}. This is OK.", new object[]
{
dag.Name,
ex.Message
});
output.AppendLogMessage("Ignoring previous error, as it is acceptable if the cluster does not exist yet.", new object[0]);
}
IEnumerable <AmServerName> source = amCluster.EnumerateNodeNames();
AmServerName ownerNode = amClusterGroup.OwnerNode;
int num = stoppedServers.Count <Server>();
foreach (Server server in stoppedServers)
{
AmServerName amServerName = new AmServerName(server);
if (source.Contains(amServerName))
{
output.AppendLogMessage("Server '{0}' is still a node in the cluster, and will have to be evicted.", new object[]
{
amServerName.NetbiosName
});
try
{
try
{
output.WriteProgressIncrementalSimple(Strings.ProgressForceCleanupNode(server.Name), 20 / num);
output.AppendLogMessage("Running the eviction operation by issuing an RPC to the replay service on '{0}'...", new object[]
{
ownerNode.Fqdn
});
ReplayRpcClientWrapper.RunEvictNodeFromCluster(ownerNode, amServerName, out verboseLog);
}
finally
{
DagTaskHelper.LogRemoteVerboseLog(output, ownerNode.Fqdn, verboseLog);
}
}
catch (DagTaskOperationFailedException ex2)
{
output.AppendLogMessage("An exception was thrown! ex={0}", new object[]
{
ex2.Message
});
Exception ex3;
if (ex2.TryGetInnerExceptionOfType(out ex3))
{
output.AppendLogMessage("Ignore it. It was AmClusterEvictWithoutCleanupException, which is acceptable. It could be completed with cluster node /forcecleanp, but that isn't necessary.", new object[0]);
}
else if (ex2.TryGetInnerExceptionOfType(out ex3))
{
output.AppendLogMessage("That exception is fine. It means that the server has already been evicted from the cluster.", new object[0]);
}
else
{
error = ex2.Message;
output.WriteWarning(Strings.FailedToEvictNode(server.Name, dag.Name, error));
list.Add(server.Name);
}
}
catch (LocalizedException ex4)
{
error = ex4.Message;
output.WriteWarning(Strings.FailedToEvictNode(server.Name, dag.Name, error));
list.Add(server.Name);
}
}
else
{
output.AppendLogMessage("Server '{0}' is not in the cluster anymore. It must have already been evicted.", new object[]
{
amServerName.NetbiosName
});
}
}
}
}
if (list.Count != 0)
{
output.WriteErrorSimple(new FailedToEvictNodeException(string.Join(",", list.ToArray <string>()), dag.Name, error));
}
}
private void CheckClusterStateForDagServerRemoval()
{
this.m_output.AppendLogMessage("CheckClusterStateForDagServerRemoval entered. m_removeNode={0}, m_destroyCluster={1}", new object[]
{
this.m_removeNode,
this.m_destroyCluster
});
try
{
this.m_clusDag = AmCluster.OpenDagClus(this.m_dag);
}
catch (AmClusterException ex)
{
this.m_output.AppendLogMessage("Trying to open the cluster on the servers in the DAG '{0}' failed with exception {1}", new object[]
{
this.m_dagName,
ex
});
this.m_output.WriteErrorSimple(new DagTaskRemoveDagServerMustHaveQuorumException(this.m_dagName));
}
using (DumpClusterTopology dumpClusterTopology = new DumpClusterTopology(this.m_clusDag, this.m_output))
{
dumpClusterTopology.Dump();
}
this.m_output.AppendLogMessage("Trying to open the node on the cluster.", new object[0]);
IAmClusterNode amClusterNode = null;
try
{
amClusterNode = this.m_clusDag.OpenNode(this.m_mailboxAmServerName);
}
catch (ClusterException ex2)
{
this.m_output.AppendLogMessage("OpenNode threw an exception. It's probably because the server is no longer clustered. Proceeding with -configurationonly.", new object[0]);
this.m_output.AppendLogMessage("For the records, the exception was {0}.", new object[]
{
ex2
});
this.m_configurationOnly = true;
this.m_mailboxServerIsDown = true;
return;
}
using (amClusterNode)
{
AmNodeState state = amClusterNode.GetState(false);
this.m_output.AppendLogMessage("Node.GetState( {0} ) reports that it is {1}.", new object[]
{
this.m_mailboxAmServerName,
state
});
if (!AmClusterNode.IsNodeUp(state))
{
this.m_mailboxServerIsDown = true;
}
}
if (!this.m_skipDagValidation)
{
try
{
DagTaskHelper.ValidateDagClusterMembership(this.m_output, this.m_dag, this.m_clusDag, this.m_mailboxAmServerName);
}
catch (ClusterException ex3)
{
this.DagTrace("ValidateDagClusterMembership() for the mailbox server failed possibly with error {0}, ex = {1}. This is OK.", new object[]
{
LocalizedException.GenerateErrorCode(ex3.InnerException).ToString(),
ex3.Message
});
}
}
int num = this.m_clusDag.EnumerateNodeNames().Count <AmServerName>();
this.DagTrace(string.Format("There are {0} nodes in the cluster.", num), new object[0]);
if (num == 1)
{
this.m_destroyCluster = true;
}
else
{
this.m_removeNode = true;
}
bool destroyCluster = this.m_destroyCluster;
this.ReopenClusterIfNecessary();
if ((this.m_removeNode || this.m_destroyCluster) && this.m_dag.DatacenterActivationMode != DatacenterActivationModeOption.Off)
{
DagTaskHelper.CheckDagCanBeActivatedInDatacenter(this.m_output, this.m_dag, (ADObjectId)this.m_mailboxServer.Identity, (ITopologyConfigurationSession)base.DataSession);
}
this.DagTrace("CheckClusterStateForDagServerRemoval left. m_removeNode={0}, m_destroyCluster={1}.", new object[]
{
this.m_removeNode,
this.m_destroyCluster
});
}
private List <string> JoinStartedNodes(IEnumerable <Server> serversToStart)
{
if (serversToStart == null)
{
throw new ArgumentNullException("serversToStart");
}
List <string> list = new List <string>(1);
using (AmCluster amCluster = AmCluster.OpenDagClus(this.m_dag))
{
using (IAmClusterGroup amClusterGroup = amCluster.FindCoreClusterGroup())
{
try
{
using (DumpClusterTopology dumpClusterTopology = new DumpClusterTopology(amCluster, this.m_output))
{
dumpClusterTopology.Dump();
}
}
catch (ClusterException ex)
{
this.m_output.AppendLogMessage("DumpClusterTopology( {0} ) failed with exception = {1}. This is OK.", new object[]
{
this.m_dag.Name,
ex.Message
});
this.m_output.AppendLogMessage("Ignoring previous error, as it is acceptable if the cluster does not exist yet.", new object[0]);
}
AmServerName ownerNode = amClusterGroup.OwnerNode;
DatabaseTasksHelper.CheckReplayServiceRunningOnNode(ownerNode, new Task.TaskErrorLoggingDelegate(base.WriteError));
if (amCluster.CnoName != string.Empty)
{
using (IAmClusterResource amClusterResource = amClusterGroup.FindResourceByTypeName("Network Name"))
{
DagTaskHelper.LogCnoState(this.m_output, this.m_dag.Name, amClusterResource);
}
}
foreach (Server server in serversToStart)
{
bool flag = false;
AmServerName amServerName = new AmServerName(server);
try
{
if (amCluster.IsEvictedBasedOnMemberShip(amServerName))
{
this.m_output.WriteProgressSimple(Strings.ProgressJoinNode(amServerName.NetbiosName));
this.m_output.AppendLogMessage("ForceCleanup the Node {0}", new object[]
{
server.Name
});
DatabaseAvailabilityGroupAction.ForceCleanupOneNodeLocally(this.m_dag.Name, server, TimeSpan.FromSeconds(15.0), this.m_output);
this.m_output.AppendLogMessage("Join the node {0} to the cluster", new object[]
{
server.Name
});
DatabaseAvailabilityGroupAction.JoinOneNode(ownerNode, amServerName, this.m_output);
flag = true;
}
else
{
this.m_output.AppendLogMessage("{0} is not evicted", new object[]
{
amServerName
});
}
if (!AmCluster.IsRunning(amServerName))
{
try
{
this.m_output.AppendLogMessage("{0} cluster service is not running, try to start the service", new object[]
{
server.Name
});
this.m_output.WriteProgressSimple(Strings.ProgressStartClussvc(amServerName.NetbiosName));
DatabaseAvailabilityGroupAction.TryStartClussvcOnNode(amServerName, this.m_output);
}
catch (InvalidOperationException)
{
this.m_output.AppendLogMessage("Got an invalidOperationException, most likely the node {0} is force cleanedup", new object[]
{
amServerName
});
if (flag)
{
this.m_output.AppendLogMessage("STRANGE! we joined {0} but cannot start the service!", new object[]
{
amServerName
});
throw;
}
this.m_output.WriteProgressSimple(Strings.ProgressJoinNode(amServerName.NetbiosName));
DatabaseAvailabilityGroupAction.JoinForceCleanupNode(ownerNode, amServerName, this.m_output);
flag = true;
}
}
if (flag)
{
ICollection <AmServerName> startedMailboxServers = from server1 in this.m_dag.StartedMailboxServers
select new AmServerName(server1);
DatabaseAvailabilityGroupAction.FixIPAddress(new AmServerName(server.Fqdn), this.m_dag, startedMailboxServers, this.m_output);
}
}
catch (LocalizedException ex2)
{
this.m_output.WriteWarning(Strings.FailedToJoinNode(server.Name, this.m_dag.Name, ex2.Message));
list.Add(server.Name);
}
catch (InvalidOperationException ex3)
{
this.m_output.WriteWarning(Strings.FailedToJoinNode(server.Name, this.m_dag.Name, ex3.Message));
list.Add(server.Name);
}
}
}
}
return(list);
}
