protected virtual Dictionary <AmServerName, AmNodeState> GetAllClusterNodeStates(ITaskOutputHelper output)
{
Dictionary <AmServerName, AmNodeState> dictionary = new Dictionary <AmServerName, AmNodeState>(15);
try
{
using (IAmCluster amCluster = ClusterFactory.Instance.Open())
{
foreach (IAmClusterNode amClusterNode in amCluster.EnumerateNodes())
{
using (amClusterNode)
{
AmNodeState state = amClusterNode.GetState(false);
dictionary[amClusterNode.Name] = state;
}
}
}
}
catch (ClusterException ex)
{
output.AppendLogMessage("GetAllClusterNodeStates() failed with error {0}", new object[]
{
ex.Message
});
}
return(dictionary);
}
// Token: 0x0600063A RID: 1594 RVA: 0x0001EE10 File Offset: 0x0001D010
internal void InitiateFailoverIfRequired(object stateInfo)
{
AmServerName amServerName = (AmServerName)stateInfo;
lock (this.m_locker)
{
if (AmSystemManager.Instance.Config.IsPAM)
{
AmFailoverEntry amFailoverEntry = null;
if (this.EntryMap.TryGetValue(amServerName, out amFailoverEntry))
{
AmNodeState nodeState = AmSystemManager.Instance.Config.DagConfig.GetNodeState(amFailoverEntry.ServerName);
if (nodeState != AmNodeState.Up)
{
AmEvtNodeDownForLongTime amEvtNodeDownForLongTime = new AmEvtNodeDownForLongTime(amFailoverEntry.ServerName);
amEvtNodeDownForLongTime.Notify();
}
else
{
ReplayCrimsonEvents.DelayedFailoverSkippedSinceNodeIsUp.Log <AmServerName>(amServerName);
}
}
this.RemoveEntryInternal(amServerName, true);
}
}
}
// Token: 0x06000144 RID: 324 RVA: 0x00007834 File Offset: 0x00005A34
internal AmNodeState GetNodeState(AmServerName nodeName)
{
AmNodeState result = AmNodeState.Unknown;
try
{
IAmClusterNode amClusterNode2;
IAmClusterNode amClusterNode = amClusterNode2 = this.Cluster.OpenNode(nodeName);
try
{
result = amClusterNode.State;
}
finally
{
if (amClusterNode2 != null)
{
amClusterNode2.Dispose();
}
}
}
catch (ClusterException ex)
{
AmTrace.Error("Failed to open cluster node {0} (error={1})", new object[]
{
nodeName,
ex.Message
});
}
return(result);
}
protected override bool RunIndividualCheck(IAmClusterNode node)
{
base.InstanceIdentity = node.Name.NetbiosName;
AmNodeState state = node.GetState(false);
if (state == AmNodeState.Paused)
{
base.FailContinue(Strings.DagMemberPausedFailed(node.Name.NetbiosName, this.m_dag.Name));
return(false);
}
return(true);
}
protected override bool RunIndividualCheck(IAmClusterNode node)
{
base.InstanceIdentity = node.Name.NetbiosName;
AmNodeState state = node.GetState(false);
if ((state == AmNodeState.Down || state == AmNodeState.Joining || state == AmNodeState.Unknown) && !base.IsNodeStopped(node.Name))
{
base.FailContinue(Strings.DagMemberUpCheckFailed(node.Name.NetbiosName, this.m_dag.Name));
return(false);
}
return(true);
}
// Token: 0x060001DF RID: 479 RVA: 0x000088D8 File Offset: 0x00006AD8
public AmNodeState GetState(bool isThrowIfUnknown)
{
AmNodeState clusterNodeState = ClusapiMethods.GetClusterNodeState(this.Handle);
if (clusterNodeState == AmNodeState.Unknown)
{
int lastWin32Error = Marshal.GetLastWin32Error();
Exception ex = new Win32Exception(lastWin32Error);
AmTrace.Debug("GetClusterNodeState() returned error (rc={0}, message={1})", new object[]
{
lastWin32Error,
ex
});
if (isThrowIfUnknown)
{
throw AmExceptionHelper.ConstructClusterApiException(lastWin32Error, "GetClusterNodeState()", new object[0]);
}
}
return(clusterNodeState);
}
private bool IsNodePubliclyUp(AmNetworkMonitor.Node node)
{
IAmCluster cluster = this.GetCluster();
if (cluster == null)
{
AmNetworkMonitor.Tracer.TraceError <AmServerName>(0L, "If cluster object is not valid, then assume node {0} is up", node.Name);
return(true);
}
Exception ex;
AmNodeState nodeState = cluster.GetNodeState(node.Name, out ex);
if (ex != null)
{
return(false);
}
if (!AmClusterNode.IsNodeUp(nodeState))
{
return(false);
}
AmClusterNodeNetworkStatus amClusterNodeNetworkStatus = AmClusterNodeStatusAccessor.Read(cluster, node.Name, out ex);
return(amClusterNodeNetworkStatus == null || amClusterNodeNetworkStatus.HasADAccess);
}
// Token: 0x060001C8 RID: 456 RVA: 0x00008548 File Offset: 0x00006748
internal static bool IsNodeUp(AmNodeState nodeState)
{
return(nodeState == AmNodeState.Up || nodeState == AmNodeState.Paused);
}
private void CheckClusterStateForDagServerRemoval()
{
this.m_output.AppendLogMessage("CheckClusterStateForDagServerRemoval entered. m_removeNode={0}, m_destroyCluster={1}", new object[]
{
this.m_removeNode,
this.m_destroyCluster
});
try
{
this.m_clusDag = AmCluster.OpenDagClus(this.m_dag);
}
catch (AmClusterException ex)
{
this.m_output.AppendLogMessage("Trying to open the cluster on the servers in the DAG '{0}' failed with exception {1}", new object[]
{
this.m_dagName,
ex
});
this.m_output.WriteErrorSimple(new DagTaskRemoveDagServerMustHaveQuorumException(this.m_dagName));
}
using (DumpClusterTopology dumpClusterTopology = new DumpClusterTopology(this.m_clusDag, this.m_output))
{
dumpClusterTopology.Dump();
}
this.m_output.AppendLogMessage("Trying to open the node on the cluster.", new object[0]);
IAmClusterNode amClusterNode = null;
try
{
amClusterNode = this.m_clusDag.OpenNode(this.m_mailboxAmServerName);
}
catch (ClusterException ex2)
{
this.m_output.AppendLogMessage("OpenNode threw an exception. It's probably because the server is no longer clustered. Proceeding with -configurationonly.", new object[0]);
this.m_output.AppendLogMessage("For the records, the exception was {0}.", new object[]
{
ex2
});
this.m_configurationOnly = true;
this.m_mailboxServerIsDown = true;
return;
}
using (amClusterNode)
{
AmNodeState state = amClusterNode.GetState(false);
this.m_output.AppendLogMessage("Node.GetState( {0} ) reports that it is {1}.", new object[]
{
this.m_mailboxAmServerName,
state
});
if (!AmClusterNode.IsNodeUp(state))
{
this.m_mailboxServerIsDown = true;
}
}
if (!this.m_skipDagValidation)
{
try
{
DagTaskHelper.ValidateDagClusterMembership(this.m_output, this.m_dag, this.m_clusDag, this.m_mailboxAmServerName);
}
catch (ClusterException ex3)
{
this.DagTrace("ValidateDagClusterMembership() for the mailbox server failed possibly with error {0}, ex = {1}. This is OK.", new object[]
{
LocalizedException.GenerateErrorCode(ex3.InnerException).ToString(),
ex3.Message
});
}
}
int num = this.m_clusDag.EnumerateNodeNames().Count <AmServerName>();
this.DagTrace(string.Format("There are {0} nodes in the cluster.", num), new object[0]);
if (num == 1)
{
this.m_destroyCluster = true;
}
else
{
this.m_removeNode = true;
}
bool destroyCluster = this.m_destroyCluster;
this.ReopenClusterIfNecessary();
if ((this.m_removeNode || this.m_destroyCluster) && this.m_dag.DatacenterActivationMode != DatacenterActivationModeOption.Off)
{
DagTaskHelper.CheckDagCanBeActivatedInDatacenter(this.m_output, this.m_dag, (ADObjectId)this.m_mailboxServer.Identity, (ITopologyConfigurationSession)base.DataSession);
}
this.DagTrace("CheckClusterStateForDagServerRemoval left. m_removeNode={0}, m_destroyCluster={1}.", new object[]
{
this.m_removeNode,
this.m_destroyCluster
});
}
internal static LatencyChecker.ClusDbHungAction AnalyzeAndSuggestActionForClusDbHang(LatencyChecker.ClusDbHungInfo hungInfo)
{
LatencyChecker.ClusDbHungAction clusDbHungAction = new LatencyChecker.ClusDbHungAction();
clusDbHungAction.HungInfo = hungInfo;
clusDbHungAction.TakeAction = false;
clusDbHungAction.TargetNodes = hungInfo.HungNodes;
clusDbHungAction.Reason = "If you see this message, something is wrong...";
if (hungInfo.HungNodeApiException != null)
{
if (hungInfo.HungNodeApiException is HungDetectionGumIdChangedException)
{
clusDbHungAction.TakeAction = false;
clusDbHungAction.Reason = "GumId changed.";
}
else if (hungInfo.HungNodeApiException is OpenClusterTimedoutException)
{
clusDbHungAction.TakeAction = true;
OpenClusterTimedoutException ex = (OpenClusterTimedoutException)hungInfo.HungNodeApiException;
clusDbHungAction.Reason = string.Format("OpenCluster timed-out for {0}", ex.ServerName);
}
else if (hungInfo.HungNodeApiException is ClusterException)
{
clusDbHungAction.TakeAction = false;
clusDbHungAction.Reason = "ClusterException was caught.";
}
}
else
{
clusDbHungAction.TakeAction = true;
clusDbHungAction.Reason = "Hung node detected without any Exceptions caught.";
}
if (clusDbHungAction.TargetNodes == null || clusDbHungAction.TargetNodes.Length < 1)
{
clusDbHungAction.TakeAction = false;
clusDbHungAction.Reason = "No hung node detected, and Rpc timeout did not catch anything.";
if (hungInfo.RpcFailedNodes != null && hungInfo.RpcFailedNodes.Length > 0)
{
AmServerName amServerName = null;
foreach (AmServerName amServerName2 in hungInfo.RpcFailedNodes)
{
AmNodeState amNodeState = AmNodeState.Unknown;
if (hungInfo.ClusterNodesStatus.TryGetValue(amServerName2.NetbiosName, out amNodeState) && amNodeState != AmNodeState.Unknown && amNodeState != AmNodeState.Down)
{
amServerName = amServerName2;
break;
}
}
if (amServerName != null)
{
clusDbHungAction.TakeAction = true;
clusDbHungAction.TargetNodes = new AmServerName[]
{
amServerName
};
clusDbHungAction.Reason = string.Format("Hung nodes detected via Rpc timeout. Node '{0}' chosen for action. Original list={1}", amServerName.NetbiosName, LatencyChecker.ConvertAmServerNamesToString(hungInfo.RpcFailedNodes));
}
else
{
clusDbHungAction.TakeAction = false;
clusDbHungAction.TargetNodes = null;
clusDbHungAction.Reason = string.Format("No nodes in Rpc non-responsive list are UP according to cluster. Skipping reboot. Original list={0}", LatencyChecker.ConvertAmServerNamesToString(hungInfo.RpcFailedNodes));
}
}
if (!clusDbHungAction.TakeAction && !AmServerName.IsNullOrEmpty(hungInfo.CurrentLockOwnerName))
{
clusDbHungAction.TakeAction = true;
clusDbHungAction.TargetNodes = new AmServerName[]
{
hungInfo.CurrentLockOwnerName
};
clusDbHungAction.Reason = string.Format("Could not find any hung nodes, so taking restart/reboot action for the lock owner '{0}'", hungInfo.CurrentLockOwnerName.NetbiosName);
}
}
ReplayCrimsonEvents.HungNodeAnalysisResult.Log <string>(clusDbHungAction.ToString());
return(clusDbHungAction);
}
protected override void WriteResult(IConfigurable dataObject)
{
TaskLogger.LogEnter(new object[]
{
dataObject.Identity,
dataObject
});
DatabaseAvailabilityGroup databaseAvailabilityGroup = (DatabaseAvailabilityGroup)dataObject;
if (this.Status && !databaseAvailabilityGroup.IsDagEmpty())
{
List <ADObjectId> list = new List <ADObjectId>(8);
List <ADObjectId> list2 = new List <ADObjectId>(8);
AmCluster amCluster = null;
try
{
amCluster = AmCluster.OpenDagClus(databaseAvailabilityGroup);
}
catch (ClusterException exception)
{
base.WriteError(exception, ErrorCategory.InvalidArgument, null);
}
if (amCluster != null)
{
using (amCluster)
{
foreach (IAmClusterNode amClusterNode in amCluster.EnumerateNodes())
{
using (amClusterNode)
{
AmNodeState state = amClusterNode.GetState(false);
if (AmClusterNode.IsNodeUp(state))
{
ADObjectId adobjectId = DagTaskHelper.FindServerAdObjectIdInDag(databaseAvailabilityGroup, amClusterNode.Name);
if (adobjectId != null)
{
list.Add(adobjectId);
if (state == AmNodeState.Paused)
{
list2.Add(adobjectId);
}
}
else
{
this.WriteWarning(Strings.WarningClusterNodeNotFoundInDag(amClusterNode.Name.Fqdn, databaseAvailabilityGroup.Name));
}
}
}
}
databaseAvailabilityGroup.OperationalServers = list.ToArray();
databaseAvailabilityGroup.ServersInMaintenance = list2.ToArray();
DagNetworkConfiguration dagNetworkConfig = DagNetworkRpc.GetDagNetworkConfig(databaseAvailabilityGroup);
databaseAvailabilityGroup.ReplicationPort = dagNetworkConfig.ReplicationPort;
foreach (DatabaseAvailabilityGroupNetwork databaseAvailabilityGroupNetwork in dagNetworkConfig.Networks)
{
databaseAvailabilityGroup.NetworkNames.Add(databaseAvailabilityGroupNetwork.Name);
}
if (DagHelper.IsQuorumTypeFileShareWitness(amCluster))
{
IAmClusterGroup amClusterGroup = amCluster.FindCoreClusterGroup();
if (amClusterGroup == null)
{
databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.InvalidConfiguration);
this.WriteWarning(Strings.WarningClusterGroupNotFormed(databaseAvailabilityGroup.Name));
goto IL_306;
}
using (amClusterGroup)
{
IEnumerable <AmClusterResource> enumerable = null;
try
{
enumerable = amClusterGroup.EnumerateResourcesOfType("File Share Witness");
AmClusterResource amClusterResource = enumerable.ElementAtOrDefault(0);
if (amClusterResource == null)
{
databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.InvalidConfiguration);
this.WriteWarning(Strings.WarningFswNotFound(databaseAvailabilityGroup.Name));
}
else if (amClusterResource.GetState() == AmResourceState.Failed)
{
databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.InvalidConfiguration);
this.WriteWarning(Strings.WarningFswFailed(databaseAvailabilityGroup.Name));
}
else
{
string privateProperty = amClusterResource.GetPrivateProperty <string>("SharePath");
UncFileSharePath a;
if (UncFileSharePath.TryParse(privateProperty, out a))
{
if (a == databaseAvailabilityGroup.FileShareWitness)
{
databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.Primary);
}
else if (a == databaseAvailabilityGroup.AlternateFileShareWitness)
{
databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.Alternate);
}
else
{
databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.InvalidConfiguration);
this.WriteWarning(Strings.WarningFswNotPrimaryOrAlternate(databaseAvailabilityGroup.Name));
}
}
else
{
databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.InvalidConfiguration);
this.WriteWarning(Strings.WarningFswNotValidPath(databaseAvailabilityGroup.Name, privateProperty));
}
}
}
finally
{
if (enumerable != null)
{
foreach (AmClusterResource amClusterResource2 in enumerable)
{
using (amClusterResource2)
{
}
}
}
}
goto IL_306;
}
}
databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.None);
IL_306:;
}
this.UpdatePam(databaseAvailabilityGroup);
try
{
DeferredFailoverEntry[] serversInDeferredRecovery = DagTaskHelper.GetServersInDeferredRecovery(databaseAvailabilityGroup);
databaseAvailabilityGroup.ServersInDeferredRecovery = serversInDeferredRecovery;
}
catch (AmServerException ex)
{
this.WriteWarning(Strings.PAMOtherError(ex.Message));
}
}
}
base.WriteResult(databaseAvailabilityGroup);
TaskLogger.LogExit();
}
// Token: 0x060000DC RID: 220 RVA: 0x00006358 File Offset: 0x00004558
private void MonitorClusterEvents()
{
while (!this.m_fShutdown)
{
AmClusterEventInfo amClusterEventInfo;
bool flag = this.m_cen.WaitForEvent(out amClusterEventInfo, this.DefaultClusterEventNotifierTimeout);
if (this.m_fShutdown)
{
AmClusterEventManager.Tracer.TraceDebug((long)this.GetHashCode(), "MonitorClusterEvents: Detected shutdown flag is set. Exiting from cluster notification monitoring");
return;
}
if (flag)
{
if (amClusterEventInfo.IsNotifyHandleClosed)
{
AmClusterEventManager.Tracer.TraceDebug((long)this.GetHashCode(), "Cluster notification handle closed. Exiting cluster event monitoring");
return;
}
if (amClusterEventInfo.IsNodeStateChanged)
{
AmServerName nodeName = new AmServerName(amClusterEventInfo.ObjectName);
if (!string.IsNullOrEmpty(amClusterEventInfo.ObjectName))
{
Exception ex;
AmNodeState nodeState = this.Cluster.GetNodeState(nodeName, out ex);
if (ex != null)
{
AmClusterEventManager.Tracer.TraceError <Exception>(0L, "MonitorClusterEvents fails to get node state: {0}", ex);
}
AmEvtNodeStateChanged amEvtNodeStateChanged = new AmEvtNodeStateChanged(nodeName, nodeState);
amEvtNodeStateChanged.Notify();
if (ex != null)
{
throw ex;
}
}
else
{
AmTrace.Error("Node state change detected but node name is invalid", new object[0]);
}
}
if (amClusterEventInfo.IsGroupStateChanged)
{
AmSystemManager.Instance.ConfigManager.TriggerRefresh(false);
}
if (amClusterEventInfo.IsClusterStateChanged)
{
AmEvtClusterStateChanged amEvtClusterStateChanged = new AmEvtClusterStateChanged();
amEvtClusterStateChanged.Notify();
AmSystemManager.Instance.ConfigManager.TriggerRefresh(true);
return;
}
if (amClusterEventInfo.IsNodeAdded)
{
AmEvtNodeAdded amEvtNodeAdded = new AmEvtNodeAdded(new AmServerName(amClusterEventInfo.ObjectName));
amEvtNodeAdded.Notify();
}
if (amClusterEventInfo.IsNodeRemoved)
{
AmEvtNodeRemoved amEvtNodeRemoved = new AmEvtNodeRemoved(new AmServerName(amClusterEventInfo.ObjectName));
amEvtNodeRemoved.Notify();
AmServerNameCache.Instance.RemoveEntry(amClusterEventInfo.ObjectName);
}
AmSystemManager.Instance.NetworkMonitor.ProcessEvent(amClusterEventInfo);
}
}
}
// Token: 0x06000503 RID: 1283 RVA: 0x0001AA1D File Offset: 0x00018C1D
internal AmEvtNodeStateChanged(AmServerName nodeName, AmNodeState nodeState)
{
this.NodeName = nodeName;
this.State = nodeState;
}
private bool CheckClussvcRunningOnStartedServers()
{
bool result = false;
List <AmServerName> list = new List <AmServerName>(16);
List <AmServerName> list2 = new List <AmServerName>(16);
new List <AmServerName>(16);
List <string> list3 = new List <string>(16);
foreach (Server server in this.m_startedServers.Values)
{
this.m_output.AppendLogMessage("Checking if clussvc is running on {0}...", new object[]
{
server.Name
});
using (ServiceController serviceController = new ServiceController("clussvc", server.Name))
{
ServiceControllerStatus serviceControllerStatus = DatabaseAvailabilityGroupAction.GetServiceControllerStatus(serviceController, server.Name, this.m_output);
this.m_output.AppendLogMessage("Clussvc is {0} on {1}.", new object[]
{
serviceControllerStatus,
server.Name
});
if (serviceControllerStatus == ServiceControllerStatus.Running)
{
list.Add(new AmServerName(server));
}
else
{
list2.Add(new AmServerName(server));
}
}
}
if (list2.Count == this.m_startedServers.Values.Count)
{
this.m_output.AppendLogMessage("None of the {0} servers in the started list had clussvc running, so it is safe to continue with restore-dag, but forcequorum will be necessary.", new object[]
{
list2.Count
});
}
else
{
int num = 0;
List <AmServerName> list4 = new List <AmServerName>(1);
try
{
using (AmCluster amCluster = AmCluster.OpenByNames(list))
{
IEnumerable <IAmClusterNode> enumerable = amCluster.EnumerateNodes();
foreach (IAmClusterNode amClusterNode in enumerable)
{
AmNodeState state = amClusterNode.State;
this.m_output.AppendLogMessage("The node {0} has cluster state = {1}.", new object[]
{
amClusterNode.Name,
state
});
if (state == AmNodeState.Joining)
{
this.m_output.AppendLogMessage(" Node {0} is joining! That means there is some difficulty establishing quorum.", new object[]
{
amClusterNode.Name
});
list3.Add(amClusterNode.Name.NetbiosName);
}
else if (AmClusterNode.IsNodeUp(state))
{
num++;
}
if (state == AmNodeState.Paused)
{
list4.Add(amClusterNode.Name);
}
}
foreach (IAmClusterNode amClusterNode2 in enumerable)
{
amClusterNode2.Dispose();
}
}
}
catch (ClusterException)
{
this.m_output.WriteErrorSimple(new DagTaskRestoreDagCouldNotOpenClusterException());
}
this.m_output.AppendLogMessage("There were {0} total (Stopped&Started) servers that were Up, out of {1} servers in the Started list that had clussvc running.", new object[]
{
num,
list.Count
});
if (num == list.Count)
{
this.m_output.AppendLogMessage("Because the two values are the same, that means quorum is healthy (even if not all of the servers are Up), so it is safe to continue with restore-dag.", new object[0]);
result = true;
}
else if (num > list.Count)
{
this.m_output.AppendLogMessage("Because there are more servers that are up than there are in the Started list, it is safe to continue with restore-dag.", new object[0]);
result = true;
}
else
{
this.m_output.AppendLogMessage("Because the two values are different, that means quorum is contested/in flux, so it is NOT safe to continue with restore-dag.", new object[0]);
this.m_output.WriteErrorSimple(new RestoreFailedDagUpException(string.Join(",", list3.ToArray())));
}
}
return(result);
}
