internal static void ForceCleanupStoppedNodes(DatabaseAvailabilityGroup dag, IEnumerable <Server> shouldStopServers, TimeSpan maxTimeToWaitForOneNode, HaTaskOutputHelper output)
{
if (dag == null)
{
throw new ArgumentNullException("dag");
}
if (shouldStopServers == null)
{
throw new ArgumentNullException("shouldStopServers");
}
if (output == null)
{
throw new ArgumentNullException("output");
}
string empty = string.Empty;
List <string> list = new List <string>(1);
int num = shouldStopServers.Count <Server>();
foreach (Server server in shouldStopServers)
{
new AmServerName(server);
output.WriteProgressIncrementalSimple(Strings.ProgressEvictNode(server.Name), 20 / num);
if (!DatabaseAvailabilityGroupAction.ForceCleanupOneNodeLocally(dag.Name, server, maxTimeToWaitForOneNode, output))
{
list.Add(server.Name);
}
}
if (list.Count != 0)
{
output.WriteErrorSimple(new StopDagFailedException(string.Join(",", list.ToArray <string>()), dag.Name));
}
}
internal bool ForceADReplication()
{
Server server = this.m_adSession.FindLocalServer();
if (server == null)
{
return(false);
}
ADObjectId localServerSite = server.ServerSite;
ADObjectId[] array = (from s in DatabaseAvailabilityGroupAction.GetSitesForDag(this.m_dag)
where s != null && !s.Equals(localServerSite)
select s).Distinct <ADObjectId>().ToArray <ADObjectId>();
if (array != null && array.Length > 0)
{
this.m_output.AppendLogMessage("forcing AD replication to site {0}", new object[]
{
array[0].Name
});
DagTaskHelper.ForceReplication(this.m_adSession, this.m_dag, array, this.m_dag.Name, new Microsoft.Exchange.Configuration.Tasks.Task.TaskWarningLoggingDelegate(this.WriteWarning), new Microsoft.Exchange.Configuration.Tasks.Task.TaskVerboseLoggingDelegate(base.WriteVerbose));
this.m_output.AppendLogMessage("forcing replication succeeded", new object[0]);
return(true);
}
return(false);
}
internal static Dictionary <AmServerName, Server> GetServersInSite(ILogTraceHelper output, IEnumerable <Server> servers, AdSiteIdParameter adSite)
{
if (servers == null)
{
throw new ArgumentNullException("servers");
}
if (adSite == null)
{
throw new ArgumentNullException("adSite");
}
Dictionary <AmServerName, Server> dictionary = new Dictionary <AmServerName, Server>(16);
foreach (Server server in servers)
{
if (DatabaseAvailabilityGroupAction.SiteEquals(server.ServerSite, adSite))
{
dictionary.Add(new AmServerName(server.Fqdn), server);
}
}
output.AppendLogMessage("The following servers are in the site {0}:", new object[]
{
adSite
});
foreach (AmServerName amServerName in dictionary.Keys)
{
output.AppendLogMessage("\t{0}", new object[]
{
amServerName.NetbiosName
});
}
return(dictionary);
}
protected override void InternalValidate()
{
TaskLogger.LogEnter();
this.m_adSession = DirectorySessionFactory.Default.CreateTopologyConfigurationSession(false, ConsistencyMode.IgnoreInvalid, ADSessionSettings.FromRootOrgScopeSet(), 146, "InternalValidate", "f:\\15.00.1497\\sources\\dev\\Management\\src\\Management\\SystemConfigurationTasks\\Cluster\\DatabaseAvailabilityGroupAction.cs");
this.m_output = new HaTaskOutputHelper(this.TaskName, new Microsoft.Exchange.Configuration.Tasks.Task.TaskErrorLoggingDelegate(base.WriteError), new Microsoft.Exchange.Configuration.Tasks.Task.TaskWarningLoggingDelegate(this.WriteWarning), new Microsoft.Exchange.Configuration.Tasks.Task.TaskVerboseLoggingDelegate(base.WriteVerbose), new Microsoft.Exchange.Configuration.Tasks.Task.TaskProgressLoggingDelegate(base.WriteProgress), this.GetHashCode());
this.m_output.CreateTempLogFile();
this.NeedToUpdateAD = true;
this.NeedToUpdateCluster = true;
if (this.ConfigurationOnly && this.QuorumOnly)
{
this.m_output.WriteErrorSimple(new TaskBothConfigurationOnlyAndQuorumOnlySpecifiedException());
}
if (this.ConfigurationOnly)
{
this.NeedToUpdateCluster = false;
}
if (this.QuorumOnly)
{
this.NeedToUpdateAD = false;
}
this.m_dag = DagTaskHelper.DagIdParameterToDag(this.Identity, base.DataSession);
DagTaskHelper.VerifyDagAndServersAreWithinScopes <DatabaseAvailabilityGroup>(this, this.m_dag, true);
if (this.m_dag.DatacenterActivationMode != DatacenterActivationModeOption.DagOnly)
{
this.m_output.WriteErrorSimple(new TaskCanOnlyRunOnDacException(this.m_dag.Name));
}
DatabaseAvailabilityGroupAction.ResolveServers(this.m_output, this.m_dag, this.m_serversInDag, this.m_startedServers, this.m_stoppedServers);
if (this.MailboxServer != null)
{
this.m_mailboxServer = (Server)base.GetDataObject <Server>(this.MailboxServer, base.DataSession, null, new LocalizedString?(Strings.ErrorServerNotFound(this.MailboxServer.ToString())), new LocalizedString?(Strings.ErrorServerNotUnique(this.MailboxServer.ToString())));
if (!this.m_mailboxServer.IsMailboxServer)
{
this.m_output.WriteErrorSimple(new OperationOnlyOnMailboxServerException(this.m_mailboxServer.Name));
}
if (this.m_mailboxServer.MajorVersion != Server.CurrentExchangeMajorVersion)
{
this.m_output.WriteErrorSimple(new DagTaskErrorServerWrongVersion(this.m_mailboxServer.Name));
}
}
if (this.ActiveDirectorySite != null)
{
IConfigurationSession configurationSession = DirectorySessionFactory.Default.CreateTopologyConfigurationSession(false, ConsistencyMode.PartiallyConsistent, ADSessionSettings.FromRootOrgScopeSet(), 225, "InternalValidate", "f:\\15.00.1497\\sources\\dev\\Management\\src\\Management\\SystemConfigurationTasks\\Cluster\\DatabaseAvailabilityGroupAction.cs");
configurationSession.UseConfigNC = false;
configurationSession.UseGlobalCatalog = true;
ADSite adsite = (ADSite)base.GetDataObject <ADSite>(this.ActiveDirectorySite, configurationSession, null, new LocalizedString?(Strings.ErrorSiteNotFound(this.ActiveDirectorySite.ToString())), new LocalizedString?(Strings.ErrorSiteNotUnique(this.ActiveDirectorySite.ToString())));
}
base.InternalValidate();
TaskLogger.LogExit();
}
protected override void InternalValidate()
{
this.m_output = new HaTaskOutputHelper("restore-databaseavailabilitygroup", new Task.TaskErrorLoggingDelegate(base.WriteError), new Task.TaskWarningLoggingDelegate(this.WriteWarning), new Task.TaskVerboseLoggingDelegate(base.WriteVerbose), new Task.TaskProgressLoggingDelegate(base.WriteProgress), this.GetHashCode());
this.m_output.CreateTempLogFile();
this.LogCommandLineParameters();
this.m_output.WriteProgressIncrementalSimple(Strings.ProgressValidation, 5);
this.m_dag = DagTaskHelper.DagIdParameterToDag(this.Identity, base.DataSession);
DagTaskHelper.VerifyDagAndServersAreWithinScopes <DatabaseAvailabilityGroup>(this, this.m_dag, true);
if (this.m_dag.DatacenterActivationMode != DatacenterActivationModeOption.DagOnly)
{
this.m_output.WriteErrorSimple(new TaskCanOnlyRunOnDacException(this.m_dag.Name));
}
DatabaseAvailabilityGroupAction.ResolveServersBasedOnStoppedList(this.m_output, this.m_dag, this.m_servers, this.m_startedServers, this.m_stoppedServers);
if (this.ActiveDirectorySite != null)
{
List <ADObjectId> sitesForDag = DatabaseAvailabilityGroupAction.GetSitesForDag(this.m_dag);
ADObjectId adobjectId = null;
foreach (ADObjectId adobjectId2 in sitesForDag)
{
if (DatabaseAvailabilityGroupAction.SiteEquals(adobjectId2, this.ActiveDirectorySite))
{
adobjectId = adobjectId2;
}
}
if (adobjectId == null)
{
this.m_output.WriteErrorSimple(new InvalidAdSiteException(this.ActiveDirectorySite.ToString()));
}
}
this.m_isQuorumPresent = this.CheckClussvcRunningOnStartedServers();
if (this.UsePrimaryWitnessServer)
{
if (this.m_dag.WitnessServer == null)
{
this.m_output.WriteErrorSimple(new RestoreNeedsWitnessServerException(this.m_dag.Name));
}
}
else if (this.m_dag.AlternateWitnessServer == null && this.AlternateWitnessServer == null)
{
this.m_output.WriteErrorSimple(new RestoreNeedsAlternateWitnessServerException(this.m_dag.Name));
}
base.InternalValidate();
}
protected override void InternalProcessRecord()
{
this.m_output.WriteProgressIncrementalSimple(Strings.ProgressForceQuorum, 10);
if (!this.m_isQuorumPresent && !this.ForceQuorumIfNecessary())
{
this.m_output.WriteErrorSimple(new DagTaskQuorumNotAchievedException(this.m_dag.Name));
}
this.m_output.WriteProgressIncrementalSimple(Strings.ProgressChangeFsw, 20);
DatabaseAvailabilityGroupAction.EvictStoppedNodes(this.m_dag, this.m_stoppedServers.Values, this.m_output);
this.ProcessFileShareWitness();
IEnumerable <AmServerName> amServerNamesFromServers = RestoreDatabaseAvailabilityGroup.GetAmServerNamesFromServers(this.m_startedServers.Values);
using (AmCluster amCluster = AmCluster.OpenByNames(amServerNamesFromServers))
{
bool shouldBeFsw = DagTaskHelper.ShouldBeFileShareWitness(this.m_output, this.m_dag, amCluster, false);
DagTaskHelper.ChangeQuorumToMnsOrFswAsAppropriate(this.m_output, this, this.m_dag, amCluster, this.m_afsw, this.m_afsw, shouldBeFsw, !this.UsePrimaryWitnessServer);
this.DeleteLastLogGenerationTimeStamps(amCluster, this.m_stoppedServers.Values, this.m_output);
}
this.m_output.WriteProgressIncrementalSimple(Strings.ProgressTaskComplete, 100);
base.InternalProcessRecord();
}
internal static void TryStartClussvcOnNode(AmServerName serverName, HaTaskOutputHelper output)
{
using (ServiceController serviceController = new ServiceController("clussvc", serverName.Fqdn))
{
try
{
if (DatabaseAvailabilityGroupAction.GetServiceControllerStatus(serviceController, serverName.NetbiosName, output) == ServiceControllerStatus.StopPending)
{
output.AppendLogMessage("Service is in stop pending, wait for it to be stopped", new object[0]);
serviceController.WaitForStatus(ServiceControllerStatus.Stopped, DatabaseAvailabilityGroupAction.ServiceTimeout);
}
if (DatabaseAvailabilityGroupAction.GetServiceControllerStatus(serviceController, serverName.NetbiosName, output) == ServiceControllerStatus.Stopped)
{
serviceController.Start();
output.WriteProgressSimple(Strings.WaitingForClusterServiceToStart(serverName.NetbiosName));
serviceController.WaitForStatus(ServiceControllerStatus.Running, DatabaseAvailabilityGroupAction.ServiceTimeout);
}
}
catch (System.ServiceProcess.TimeoutException)
{
output.WriteErrorSimple(new FailedToStartClusSvcException(serverName.NetbiosName, DatabaseAvailabilityGroupAction.GetServiceControllerStatus(serviceController, serverName.NetbiosName, output).ToString()));
}
}
}
internal static void ResolveServers(ILogTraceHelper output, DatabaseAvailabilityGroup dag, Dictionary <AmServerName, Server> servers, Dictionary <AmServerName, Server> startedServers, Dictionary <AmServerName, Server> stoppedServers)
{
DatabaseAvailabilityGroupAction.ResolveServersInternal(output, dag, servers, startedServers, stoppedServers, true);
}
protected override void InternalValidate()
{
TaskLogger.LogEnter();
base.InternalValidate();
this.m_output = new HaTaskOutputHelper("set-databaseavailabilitygroup", new Task.TaskErrorLoggingDelegate(base.WriteError), new Task.TaskWarningLoggingDelegate(this.WriteWarning), new Task.TaskVerboseLoggingDelegate(base.WriteVerbose), new Task.TaskProgressLoggingDelegate(base.WriteProgress), this.GetHashCode());
this.m_dag = this.DataObject;
this.m_skipDagValidation = this.SkipDagValidation;
DagTaskHelper.VerifyDagAndServersAreWithinScopes <DatabaseAvailabilityGroup>(this, this.m_dag, true);
if (this.DatabaseAvailabilityGroupIpAddresses != null)
{
if (!this.m_dag.IsDagEmpty())
{
this.PrepareServersInDagIfRequired();
foreach (AmServerName amServerName in this.m_serverNamesInDag)
{
DagTaskHelper.ValidateIPAddressList(this.m_output, amServerName.NetbiosName, this.DatabaseAvailabilityGroupIpAddresses, this.m_dag.Name);
DagTaskHelper.ValidateIPAddressList(this.m_output, amServerName.Fqdn, this.DatabaseAvailabilityGroupIpAddresses, this.m_dag.Name);
}
}
foreach (IPAddress ipaddress in this.DatabaseAvailabilityGroupIpAddresses)
{
if (ipaddress.AddressFamily == AddressFamily.InterNetworkV6)
{
this.m_output.WriteErrorSimple(new DagTaskDagIpAddressesMustBeIpv4Exception());
}
}
}
this.m_useAlternateWitnessServer = (this.m_dag.AlternateWitnessServer != null && DagTaskHelper.IsDagFailedOverToOtherSite(this.m_output, this.m_dag));
if (!this.m_useAlternateWitnessServer)
{
this.m_fsw = new FileShareWitness((ITopologyConfigurationSession)base.DataSession, this.m_dag.Name, this.WitnessServer ?? this.m_dag.WitnessServer, this.WitnessDirectory ?? this.m_dag.WitnessDirectory);
try
{
this.m_fsw.Initialize();
}
catch (LocalizedException error)
{
this.m_output.WriteErrorSimple(error);
}
this.CheckFsw(this.m_fsw.WitnessServerFqdn);
}
if (this.AlternateWitnessServer != null || this.AlternateWitnessDirectory != null || (this.m_dag.AlternateWitnessServer != null && this.m_dag.AlternateWitnessDirectory != null && !this.m_alternateWitnessServerParameterSpecified) || this.m_useAlternateWitnessServer)
{
this.m_afsw = new FileShareWitness((ITopologyConfigurationSession)base.DataSession, this.m_dag.Name, this.AlternateWitnessServer ?? this.m_dag.AlternateWitnessServer, this.AlternateWitnessDirectory ?? this.m_dag.AlternateWitnessDirectory);
try
{
this.m_afsw.Initialize();
}
catch (LocalizedException error2)
{
this.m_output.WriteErrorSimple(error2);
}
if (this.m_fsw != null && SharedHelper.StringIEquals(this.m_afsw.WitnessServerFqdn, this.m_fsw.WitnessServerFqdn) && this.m_fsw.WitnessDirectory != this.m_afsw.WitnessDirectory)
{
this.m_output.WriteErrorSimple(new DagFswAndAlternateFswOnSameWitnessServerButPointToDifferentDirectoriesException(this.m_fsw.WitnessServer.ToString(), this.m_fsw.WitnessDirectory.ToString(), this.m_afsw.WitnessDirectory.ToString()));
}
this.CheckFsw(this.m_afsw.WitnessServerFqdn);
}
Dictionary <AmServerName, Server> startedServers = new Dictionary <AmServerName, Server>();
DatabaseAvailabilityGroupAction.ResolveServers(this.m_output, this.m_dag, this.m_allServers, startedServers, this.m_stoppedServers);
if (!this.m_dag.IsDagEmpty())
{
this.PrepareServersInDagIfRequired();
List <AmServerName> list = new List <AmServerName>(this.m_stoppedServers.Count);
IEnumerable <AmServerName> serversInCluster = null;
foreach (Server server in this.m_stoppedServers.Values)
{
list.Add(new AmServerName(server.Id));
}
using (AmCluster amCluster = AmCluster.OpenDagClus(this.m_dag))
{
serversInCluster = amCluster.EnumerateNodeNames();
}
if (!this.m_skipDagValidation)
{
DagTaskHelper.CompareDagClusterMembership(this.m_output, this.m_dag.Name, this.m_serverNamesInDag, serversInCluster, list);
}
}
if (base.Fields["DatacenterActivationMode"] != null && this.DatacenterActivationMode != DatacenterActivationModeOption.Off)
{
DagTaskHelper.CheckDagCanBeActivatedInDatacenter(this.m_output, this.m_dag, null, (ITopologyConfigurationSession)base.DataSession);
}
if (base.Fields["DatacenterActivationMode"] != null && this.DatacenterActivationMode == DatacenterActivationModeOption.Off)
{
this.DataObject.StartedMailboxServers = null;
this.DataObject.StoppedMailboxServers = null;
}
TaskLogger.LogExit();
}
protected override void InternalProcessRecord()
{
Dictionary <AmServerName, Server> dictionary = new Dictionary <AmServerName, Server>(16);
if (this.m_mailboxServer != null)
{
AmServerName serverName = new AmServerName(this.m_mailboxServer.Fqdn);
if (!this.m_serversInDag.Keys.Any((AmServerName serverInDag) => serverInDag.Equals(serverName)))
{
this.m_output.WriteErrorSimple(new DagTaskServerIsNotInDagException(serverName.Fqdn, this.m_dag.Name));
}
dictionary.Add(serverName, this.m_mailboxServer);
if (base.NeedToUpdateAD)
{
if (!this.m_stoppedServers.ContainsKey(serverName))
{
this.m_stoppedServers.Add(serverName, this.m_mailboxServer);
this.m_output.AppendLogMessage("{0} added to stopped list", new object[]
{
serverName.NetbiosName
});
}
if (this.m_startedServers.ContainsKey(serverName))
{
this.m_startedServers.Remove(serverName);
this.m_output.AppendLogMessage("{0} removed from started list", new object[]
{
serverName.NetbiosName
});
}
}
}
else
{
dictionary = DatabaseAvailabilityGroupAction.GetServersInSite(this.m_output, this.m_serversInDag.Values, base.ActiveDirectorySite);
if (base.NeedToUpdateAD)
{
foreach (AmServerName amServerName in dictionary.Keys)
{
if (!this.m_stoppedServers.ContainsKey(amServerName))
{
this.m_output.AppendLogMessage("add {0} to stopped list", new object[]
{
amServerName.NetbiosName
});
this.m_stoppedServers.Add(amServerName, this.m_serversInDag[amServerName]);
}
if (this.m_startedServers.ContainsKey(amServerName))
{
this.m_output.AppendLogMessage("Remove {0} from started list", new object[]
{
amServerName.NetbiosName
});
this.m_startedServers.Remove(amServerName);
}
}
}
}
if (base.NeedToUpdateAD)
{
this.m_dag.StoppedMailboxServers = DatabaseAvailabilityGroupAction.ServerListToFqdnList(this.m_stoppedServers.Keys);
this.m_dag.StartedMailboxServers = DatabaseAvailabilityGroupAction.ServerListToFqdnList(this.m_startedServers.Keys);
this.m_adSession.Save(this.m_dag);
this.m_output.AppendLogMessage("updated the started servers list in AD:", new object[0]);
foreach (AmServerName amServerName2 in this.m_startedServers.Keys)
{
this.m_output.AppendLogMessage("\t{0}", new object[]
{
amServerName2.NetbiosName
});
}
this.m_output.AppendLogMessage("updated the stopped servers list in AD:", new object[0]);
foreach (AmServerName amServerName3 in this.m_stoppedServers.Keys)
{
this.m_output.AppendLogMessage("\t{0}", new object[]
{
amServerName3.NetbiosName
});
}
base.ForceADReplication();
}
if (base.NeedToUpdateCluster)
{
if (DatabaseAvailabilityGroupAction.IsClusterUp(this.m_dag, this.m_output))
{
DatabaseAvailabilityGroupAction.EvictStoppedNodes(this.m_dag, dictionary.Values, this.m_output);
}
else
{
DatabaseAvailabilityGroupAction.ForceCleanupStoppedNodes(this.m_dag, dictionary.Values, TimeSpan.FromSeconds(15.0), this.m_output);
}
TimeSpan timeout = TimeSpan.FromSeconds(1.0);
foreach (AmServerName amServerName4 in dictionary.Keys)
{
this.m_output.AppendLogMessage("notify replayservice on {0} with consensus state 0", new object[]
{
amServerName4.NetbiosName
});
AmRpcClientHelper.RpcchSetAutomountConsensusStateBestEffort(amServerName4.Fqdn, 0, timeout);
}
}
base.InternalProcessRecord();
}
private bool CheckClussvcRunningOnStartedServers()
{
bool result = false;
List <AmServerName> list = new List <AmServerName>(16);
List <AmServerName> list2 = new List <AmServerName>(16);
new List <AmServerName>(16);
List <string> list3 = new List <string>(16);
foreach (Server server in this.m_startedServers.Values)
{
this.m_output.AppendLogMessage("Checking if clussvc is running on {0}...", new object[]
{
server.Name
});
using (ServiceController serviceController = new ServiceController("clussvc", server.Name))
{
ServiceControllerStatus serviceControllerStatus = DatabaseAvailabilityGroupAction.GetServiceControllerStatus(serviceController, server.Name, this.m_output);
this.m_output.AppendLogMessage("Clussvc is {0} on {1}.", new object[]
{
serviceControllerStatus,
server.Name
});
if (serviceControllerStatus == ServiceControllerStatus.Running)
{
list.Add(new AmServerName(server));
}
else
{
list2.Add(new AmServerName(server));
}
}
}
if (list2.Count == this.m_startedServers.Values.Count)
{
this.m_output.AppendLogMessage("None of the {0} servers in the started list had clussvc running, so it is safe to continue with restore-dag, but forcequorum will be necessary.", new object[]
{
list2.Count
});
}
else
{
int num = 0;
List <AmServerName> list4 = new List <AmServerName>(1);
try
{
using (AmCluster amCluster = AmCluster.OpenByNames(list))
{
IEnumerable <IAmClusterNode> enumerable = amCluster.EnumerateNodes();
foreach (IAmClusterNode amClusterNode in enumerable)
{
AmNodeState state = amClusterNode.State;
this.m_output.AppendLogMessage("The node {0} has cluster state = {1}.", new object[]
{
amClusterNode.Name,
state
});
if (state == AmNodeState.Joining)
{
this.m_output.AppendLogMessage(" Node {0} is joining! That means there is some difficulty establishing quorum.", new object[]
{
amClusterNode.Name
});
list3.Add(amClusterNode.Name.NetbiosName);
}
else if (AmClusterNode.IsNodeUp(state))
{
num++;
}
if (state == AmNodeState.Paused)
{
list4.Add(amClusterNode.Name);
}
}
foreach (IAmClusterNode amClusterNode2 in enumerable)
{
amClusterNode2.Dispose();
}
}
}
catch (ClusterException)
{
this.m_output.WriteErrorSimple(new DagTaskRestoreDagCouldNotOpenClusterException());
}
this.m_output.AppendLogMessage("There were {0} total (Stopped&Started) servers that were Up, out of {1} servers in the Started list that had clussvc running.", new object[]
{
num,
list.Count
});
if (num == list.Count)
{
this.m_output.AppendLogMessage("Because the two values are the same, that means quorum is healthy (even if not all of the servers are Up), so it is safe to continue with restore-dag.", new object[0]);
result = true;
}
else if (num > list.Count)
{
this.m_output.AppendLogMessage("Because there are more servers that are up than there are in the Started list, it is safe to continue with restore-dag.", new object[0]);
result = true;
}
else
{
this.m_output.AppendLogMessage("Because the two values are different, that means quorum is contested/in flux, so it is NOT safe to continue with restore-dag.", new object[0]);
this.m_output.WriteErrorSimple(new RestoreFailedDagUpException(string.Join(",", list3.ToArray())));
}
}
return(result);
}
private bool ForceQuorumIfNecessary()
{
bool flag = true;
bool result = false;
foreach (Server server in this.m_startedServers.Values)
{
this.m_output.AppendLogMessage("ForceQuorumIfNecessary: Checking if clussvc is running on {0}...", new object[]
{
server.Name
});
using (ServiceController serviceController = new ServiceController("clussvc", server.Name))
{
ServiceControllerStatus serviceControllerStatus = DatabaseAvailabilityGroupAction.GetServiceControllerStatus(serviceController, server.Name, this.m_output);
if (serviceControllerStatus == ServiceControllerStatus.Stopped)
{
this.m_output.AppendLogMessage("ForceQuorum: clussvc is stopped on node {0}. Starting it.", new object[]
{
server.Name
});
try
{
if (flag)
{
this.m_output.AppendLogMessage("Starting cluster service with /fq (force quorum).", new object[0]);
serviceController.Start(new string[]
{
"/fq"
});
flag = false;
}
else
{
this.m_output.AppendLogMessage("Starting cluster service in normal mode.", new object[0]);
serviceController.Start();
}
try
{
this.m_output.AppendLogMessage("Waiting up to {0} for clussvc to be in the Running state.", new object[]
{
RestoreDatabaseAvailabilityGroup.ServiceTimeOut
});
serviceController.WaitForStatus(ServiceControllerStatus.Running, RestoreDatabaseAvailabilityGroup.ServiceTimeOut);
result = true;
}
catch (System.ServiceProcess.TimeoutException)
{
this.m_output.WriteErrorSimple(new FailedToStartClusSvcException(server.Name, serviceControllerStatus.ToString()));
}
continue;
}
catch (Win32Exception ex)
{
this.m_output.AppendLogMessage("ForceQuorum: cluster service on {0}Failed to start with fq {1}", new object[]
{
server.Name,
ex.Message
});
this.m_output.WriteErrorSimple(new FailedToStartClusSvcException(server.Name, serviceControllerStatus.ToString()));
continue;
}
catch (InvalidOperationException ex2)
{
this.m_output.AppendLogMessage("ForceQuorum: cluster service on {0}Failed to start with fq {1}", new object[]
{
server.Name,
ex2.Message
});
this.m_output.WriteErrorSimple(new FailedToStartClusSvcException(server.Name, serviceControllerStatus.ToString()));
continue;
}
}
this.m_output.AppendLogMessage("ForceQuorum: cluster service on {0} is in state {1}, we will try another node (if there is one) for forcing quorum.", new object[]
{
server.Name,
serviceControllerStatus
});
flag = false;
}
}
return(result);
}
protected override void InternalProcessRecord()
{
Dictionary <AmServerName, Server> dictionary = new Dictionary <AmServerName, Server>(16);
if (this.m_mailboxServer != null)
{
AmServerName serverName = new AmServerName(this.m_mailboxServer.Fqdn);
if (!this.m_serversInDag.Keys.Any((AmServerName serverInDag) => serverInDag.Equals(serverName)))
{
this.m_output.WriteErrorSimple(new DagTaskServerIsNotInDagException(serverName.Fqdn, this.m_dag.Name));
}
dictionary.Add(serverName, this.m_mailboxServer);
if (base.NeedToUpdateAD)
{
if (this.m_stoppedServers.ContainsKey(serverName))
{
this.m_stoppedServers.Remove(serverName);
this.m_output.AppendLogMessage("{0} removed from stopped list", new object[]
{
serverName.NetbiosName
});
}
if (!this.m_startedServers.ContainsKey(serverName))
{
this.m_startedServers.Add(serverName, this.m_mailboxServer);
this.m_output.AppendLogMessage("{0} added to started list", new object[]
{
serverName.NetbiosName
});
}
}
}
else
{
dictionary = DatabaseAvailabilityGroupAction.GetServersInSite(this.m_output, this.m_serversInDag.Values, base.ActiveDirectorySite);
if (base.NeedToUpdateAD)
{
foreach (AmServerName amServerName in dictionary.Keys)
{
if (this.m_stoppedServers.ContainsKey(amServerName))
{
this.m_output.AppendLogMessage("Remove {0} from stopped list", new object[]
{
amServerName.NetbiosName
});
this.m_stoppedServers.Remove(amServerName);
}
if (!this.m_startedServers.ContainsKey(amServerName))
{
this.m_output.AppendLogMessage("Add {0} to started list", new object[]
{
amServerName.NetbiosName
});
this.m_startedServers.Add(amServerName, this.m_serversInDag[amServerName]);
}
}
}
}
if (base.NeedToUpdateAD)
{
this.m_output.WriteProgressSimple(Strings.ProgressStopUpdateAD);
this.m_dag.StoppedMailboxServers = DatabaseAvailabilityGroupAction.ServerListToFqdnList(this.m_stoppedServers.Keys);
this.m_adSession.Save(this.m_dag);
this.m_output.AppendLogMessage("Updated stopped list with :", new object[0]);
foreach (AmServerName amServerName2 in this.m_stoppedServers.Keys)
{
this.m_output.AppendLogMessage("\t{0}", new object[]
{
amServerName2.NetbiosName
});
}
this.m_output.WriteProgressSimple(Strings.ProgressStopUpdateOtherAD);
base.ForceADReplication();
}
List <string> errorServers = null;
if (base.NeedToUpdateCluster)
{
errorServers = this.JoinStartedNodes(dictionary.Values);
if (base.NeedToUpdateAD)
{
IEnumerable <AmServerName> enumerable = new List <AmServerName>(1);
enumerable = from server in this.m_startedServers.Keys
where !errorServers.Contains(server.NetbiosName, StringComparer.OrdinalIgnoreCase)
select server;
this.m_output.WriteProgressSimple(Strings.ProgressStartUpdateAD);
this.m_dag.StartedMailboxServers = DatabaseAvailabilityGroupAction.ServerListToFqdnList(enumerable);
this.m_adSession.Save(this.m_dag);
this.m_output.AppendLogMessage("Updated started list with :", new object[0]);
foreach (AmServerName amServerName3 in enumerable)
{
this.m_output.AppendLogMessage("\t{0}", new object[]
{
amServerName3.NetbiosName
});
}
this.m_output.WriteProgressSimple(Strings.ProgressStartUpdateOtherAD);
base.ForceADReplication();
if (errorServers.Count != 0)
{
this.m_output.WriteErrorSimple(new FailedToStartNodeException(string.Join(",", errorServers.ToArray <string>()), this.m_dag.Name));
}
}
this.SetAutomountConsensusOnStartedServers(dictionary);
}
base.InternalProcessRecord();
this.m_output.WriteProgressSimple(Strings.ProgressTaskComplete);
}
private List <string> JoinStartedNodes(IEnumerable <Server> serversToStart)
{
if (serversToStart == null)
{
throw new ArgumentNullException("serversToStart");
}
List <string> list = new List <string>(1);
using (AmCluster amCluster = AmCluster.OpenDagClus(this.m_dag))
{
using (IAmClusterGroup amClusterGroup = amCluster.FindCoreClusterGroup())
{
try
{
using (DumpClusterTopology dumpClusterTopology = new DumpClusterTopology(amCluster, this.m_output))
{
dumpClusterTopology.Dump();
}
}
catch (ClusterException ex)
{
this.m_output.AppendLogMessage("DumpClusterTopology( {0} ) failed with exception = {1}. This is OK.", new object[]
{
this.m_dag.Name,
ex.Message
});
this.m_output.AppendLogMessage("Ignoring previous error, as it is acceptable if the cluster does not exist yet.", new object[0]);
}
AmServerName ownerNode = amClusterGroup.OwnerNode;
DatabaseTasksHelper.CheckReplayServiceRunningOnNode(ownerNode, new Task.TaskErrorLoggingDelegate(base.WriteError));
if (amCluster.CnoName != string.Empty)
{
using (IAmClusterResource amClusterResource = amClusterGroup.FindResourceByTypeName("Network Name"))
{
DagTaskHelper.LogCnoState(this.m_output, this.m_dag.Name, amClusterResource);
}
}
foreach (Server server in serversToStart)
{
bool flag = false;
AmServerName amServerName = new AmServerName(server);
try
{
if (amCluster.IsEvictedBasedOnMemberShip(amServerName))
{
this.m_output.WriteProgressSimple(Strings.ProgressJoinNode(amServerName.NetbiosName));
this.m_output.AppendLogMessage("ForceCleanup the Node {0}", new object[]
{
server.Name
});
DatabaseAvailabilityGroupAction.ForceCleanupOneNodeLocally(this.m_dag.Name, server, TimeSpan.FromSeconds(15.0), this.m_output);
this.m_output.AppendLogMessage("Join the node {0} to the cluster", new object[]
{
server.Name
});
DatabaseAvailabilityGroupAction.JoinOneNode(ownerNode, amServerName, this.m_output);
flag = true;
}
else
{
this.m_output.AppendLogMessage("{0} is not evicted", new object[]
{
amServerName
});
}
if (!AmCluster.IsRunning(amServerName))
{
try
{
this.m_output.AppendLogMessage("{0} cluster service is not running, try to start the service", new object[]
{
server.Name
});
this.m_output.WriteProgressSimple(Strings.ProgressStartClussvc(amServerName.NetbiosName));
DatabaseAvailabilityGroupAction.TryStartClussvcOnNode(amServerName, this.m_output);
}
catch (InvalidOperationException)
{
this.m_output.AppendLogMessage("Got an invalidOperationException, most likely the node {0} is force cleanedup", new object[]
{
amServerName
});
if (flag)
{
this.m_output.AppendLogMessage("STRANGE! we joined {0} but cannot start the service!", new object[]
{
amServerName
});
throw;
}
this.m_output.WriteProgressSimple(Strings.ProgressJoinNode(amServerName.NetbiosName));
DatabaseAvailabilityGroupAction.JoinForceCleanupNode(ownerNode, amServerName, this.m_output);
flag = true;
}
}
if (flag)
{
ICollection <AmServerName> startedMailboxServers = from server1 in this.m_dag.StartedMailboxServers
select new AmServerName(server1);
DatabaseAvailabilityGroupAction.FixIPAddress(new AmServerName(server.Fqdn), this.m_dag, startedMailboxServers, this.m_output);
}
}
catch (LocalizedException ex2)
{
this.m_output.WriteWarning(Strings.FailedToJoinNode(server.Name, this.m_dag.Name, ex2.Message));
list.Add(server.Name);
}
catch (InvalidOperationException ex3)
{
this.m_output.WriteWarning(Strings.FailedToJoinNode(server.Name, this.m_dag.Name, ex3.Message));
list.Add(server.Name);
}
}
}
}
return(list);
}