internal static void EvictStoppedNodes(DatabaseAvailabilityGroup dag, IEnumerable <Server> stoppedServers, HaTaskOutputHelper output) { if (dag == null) { throw new ArgumentNullException("dag"); } if (stoppedServers == null) { throw new ArgumentNullException("stoppedServers"); } if (output == null) { throw new ArgumentNullException("output"); } string error = null; List <string> list = new List <string>(1); string verboseLog = null; using (AmCluster amCluster = AmCluster.OpenDagClus(dag)) { using (IAmClusterGroup amClusterGroup = amCluster.FindCoreClusterGroup()) { output.AppendLogMessage("EvictStoppedNodes has been called. Dumping current cluster state.", new object[0]); try { using (DumpClusterTopology dumpClusterTopology = new DumpClusterTopology(amCluster, output)) { dumpClusterTopology.Dump(); } } catch (ClusterException ex) { output.AppendLogMessage("DumpClusterTopology( {0} ) failed with exception = {1}. This is OK.", new object[] { dag.Name, ex.Message }); output.AppendLogMessage("Ignoring previous error, as it is acceptable if the cluster does not exist yet.", new object[0]); } IEnumerable <AmServerName> source = amCluster.EnumerateNodeNames(); AmServerName ownerNode = amClusterGroup.OwnerNode; int num = stoppedServers.Count <Server>(); foreach (Server server in stoppedServers) { AmServerName amServerName = new AmServerName(server); if (source.Contains(amServerName)) { output.AppendLogMessage("Server '{0}' is still a node in the cluster, and will have to be evicted.", new object[] { amServerName.NetbiosName }); try { try { output.WriteProgressIncrementalSimple(Strings.ProgressForceCleanupNode(server.Name), 20 / num); output.AppendLogMessage("Running the eviction operation by issuing an RPC to the replay service on '{0}'...", new object[] { ownerNode.Fqdn }); ReplayRpcClientWrapper.RunEvictNodeFromCluster(ownerNode, amServerName, out verboseLog); } finally { DagTaskHelper.LogRemoteVerboseLog(output, ownerNode.Fqdn, verboseLog); } } catch (DagTaskOperationFailedException ex2) { output.AppendLogMessage("An exception was thrown! ex={0}", new object[] { ex2.Message }); Exception ex3; if (ex2.TryGetInnerExceptionOfType(out ex3)) { output.AppendLogMessage("Ignore it. It was AmClusterEvictWithoutCleanupException, which is acceptable. It could be completed with cluster node /forcecleanp, but that isn't necessary.", new object[0]); } else if (ex2.TryGetInnerExceptionOfType(out ex3)) { output.AppendLogMessage("That exception is fine. It means that the server has already been evicted from the cluster.", new object[0]); } else { error = ex2.Message; output.WriteWarning(Strings.FailedToEvictNode(server.Name, dag.Name, error)); list.Add(server.Name); } } catch (LocalizedException ex4) { error = ex4.Message; output.WriteWarning(Strings.FailedToEvictNode(server.Name, dag.Name, error)); list.Add(server.Name); } } else { output.AppendLogMessage("Server '{0}' is not in the cluster anymore. It must have already been evicted.", new object[] { amServerName.NetbiosName }); } } } } if (list.Count != 0) { output.WriteErrorSimple(new FailedToEvictNodeException(string.Join(",", list.ToArray <string>()), dag.Name, error)); } }
private void CheckClusterStateForDagServerRemoval() { this.m_output.AppendLogMessage("CheckClusterStateForDagServerRemoval entered. m_removeNode={0}, m_destroyCluster={1}", new object[] { this.m_removeNode, this.m_destroyCluster }); try { this.m_clusDag = AmCluster.OpenDagClus(this.m_dag); } catch (AmClusterException ex) { this.m_output.AppendLogMessage("Trying to open the cluster on the servers in the DAG '{0}' failed with exception {1}", new object[] { this.m_dagName, ex }); this.m_output.WriteErrorSimple(new DagTaskRemoveDagServerMustHaveQuorumException(this.m_dagName)); } using (DumpClusterTopology dumpClusterTopology = new DumpClusterTopology(this.m_clusDag, this.m_output)) { dumpClusterTopology.Dump(); } this.m_output.AppendLogMessage("Trying to open the node on the cluster.", new object[0]); IAmClusterNode amClusterNode = null; try { amClusterNode = this.m_clusDag.OpenNode(this.m_mailboxAmServerName); } catch (ClusterException ex2) { this.m_output.AppendLogMessage("OpenNode threw an exception. It's probably because the server is no longer clustered. Proceeding with -configurationonly.", new object[0]); this.m_output.AppendLogMessage("For the records, the exception was {0}.", new object[] { ex2 }); this.m_configurationOnly = true; this.m_mailboxServerIsDown = true; return; } using (amClusterNode) { AmNodeState state = amClusterNode.GetState(false); this.m_output.AppendLogMessage("Node.GetState( {0} ) reports that it is {1}.", new object[] { this.m_mailboxAmServerName, state }); if (!AmClusterNode.IsNodeUp(state)) { this.m_mailboxServerIsDown = true; } } if (!this.m_skipDagValidation) { try { DagTaskHelper.ValidateDagClusterMembership(this.m_output, this.m_dag, this.m_clusDag, this.m_mailboxAmServerName); } catch (ClusterException ex3) { this.DagTrace("ValidateDagClusterMembership() for the mailbox server failed possibly with error {0}, ex = {1}. This is OK.", new object[] { LocalizedException.GenerateErrorCode(ex3.InnerException).ToString(), ex3.Message }); } } int num = this.m_clusDag.EnumerateNodeNames().Count <AmServerName>(); this.DagTrace(string.Format("There are {0} nodes in the cluster.", num), new object[0]); if (num == 1) { this.m_destroyCluster = true; } else { this.m_removeNode = true; } bool destroyCluster = this.m_destroyCluster; this.ReopenClusterIfNecessary(); if ((this.m_removeNode || this.m_destroyCluster) && this.m_dag.DatacenterActivationMode != DatacenterActivationModeOption.Off) { DagTaskHelper.CheckDagCanBeActivatedInDatacenter(this.m_output, this.m_dag, (ADObjectId)this.m_mailboxServer.Identity, (ITopologyConfigurationSession)base.DataSession); } this.DagTrace("CheckClusterStateForDagServerRemoval left. m_removeNode={0}, m_destroyCluster={1}.", new object[] { this.m_removeNode, this.m_destroyCluster }); }
private List <string> JoinStartedNodes(IEnumerable <Server> serversToStart) { if (serversToStart == null) { throw new ArgumentNullException("serversToStart"); } List <string> list = new List <string>(1); using (AmCluster amCluster = AmCluster.OpenDagClus(this.m_dag)) { using (IAmClusterGroup amClusterGroup = amCluster.FindCoreClusterGroup()) { try { using (DumpClusterTopology dumpClusterTopology = new DumpClusterTopology(amCluster, this.m_output)) { dumpClusterTopology.Dump(); } } catch (ClusterException ex) { this.m_output.AppendLogMessage("DumpClusterTopology( {0} ) failed with exception = {1}. This is OK.", new object[] { this.m_dag.Name, ex.Message }); this.m_output.AppendLogMessage("Ignoring previous error, as it is acceptable if the cluster does not exist yet.", new object[0]); } AmServerName ownerNode = amClusterGroup.OwnerNode; DatabaseTasksHelper.CheckReplayServiceRunningOnNode(ownerNode, new Task.TaskErrorLoggingDelegate(base.WriteError)); if (amCluster.CnoName != string.Empty) { using (IAmClusterResource amClusterResource = amClusterGroup.FindResourceByTypeName("Network Name")) { DagTaskHelper.LogCnoState(this.m_output, this.m_dag.Name, amClusterResource); } } foreach (Server server in serversToStart) { bool flag = false; AmServerName amServerName = new AmServerName(server); try { if (amCluster.IsEvictedBasedOnMemberShip(amServerName)) { this.m_output.WriteProgressSimple(Strings.ProgressJoinNode(amServerName.NetbiosName)); this.m_output.AppendLogMessage("ForceCleanup the Node {0}", new object[] { server.Name }); DatabaseAvailabilityGroupAction.ForceCleanupOneNodeLocally(this.m_dag.Name, server, TimeSpan.FromSeconds(15.0), this.m_output); this.m_output.AppendLogMessage("Join the node {0} to the cluster", new object[] { server.Name }); DatabaseAvailabilityGroupAction.JoinOneNode(ownerNode, amServerName, this.m_output); flag = true; } else { this.m_output.AppendLogMessage("{0} is not evicted", new object[] { amServerName }); } if (!AmCluster.IsRunning(amServerName)) { try { this.m_output.AppendLogMessage("{0} cluster service is not running, try to start the service", new object[] { server.Name }); this.m_output.WriteProgressSimple(Strings.ProgressStartClussvc(amServerName.NetbiosName)); DatabaseAvailabilityGroupAction.TryStartClussvcOnNode(amServerName, this.m_output); } catch (InvalidOperationException) { this.m_output.AppendLogMessage("Got an invalidOperationException, most likely the node {0} is force cleanedup", new object[] { amServerName }); if (flag) { this.m_output.AppendLogMessage("STRANGE! we joined {0} but cannot start the service!", new object[] { amServerName }); throw; } this.m_output.WriteProgressSimple(Strings.ProgressJoinNode(amServerName.NetbiosName)); DatabaseAvailabilityGroupAction.JoinForceCleanupNode(ownerNode, amServerName, this.m_output); flag = true; } } if (flag) { ICollection <AmServerName> startedMailboxServers = from server1 in this.m_dag.StartedMailboxServers select new AmServerName(server1); DatabaseAvailabilityGroupAction.FixIPAddress(new AmServerName(server.Fqdn), this.m_dag, startedMailboxServers, this.m_output); } } catch (LocalizedException ex2) { this.m_output.WriteWarning(Strings.FailedToJoinNode(server.Name, this.m_dag.Name, ex2.Message)); list.Add(server.Name); } catch (InvalidOperationException ex3) { this.m_output.WriteWarning(Strings.FailedToJoinNode(server.Name, this.m_dag.Name, ex3.Message)); list.Add(server.Name); } } } } return(list); }