protected virtual Dictionary <AmServerName, AmNodeState> GetAllClusterNodeStates(ITaskOutputHelper output) { Dictionary <AmServerName, AmNodeState> dictionary = new Dictionary <AmServerName, AmNodeState>(15); try { using (IAmCluster amCluster = ClusterFactory.Instance.Open()) { foreach (IAmClusterNode amClusterNode in amCluster.EnumerateNodes()) { using (amClusterNode) { AmNodeState state = amClusterNode.GetState(false); dictionary[amClusterNode.Name] = state; } } } } catch (ClusterException ex) { output.AppendLogMessage("GetAllClusterNodeStates() failed with error {0}", new object[] { ex.Message }); } return(dictionary); }
// Token: 0x0600063A RID: 1594 RVA: 0x0001EE10 File Offset: 0x0001D010 internal void InitiateFailoverIfRequired(object stateInfo) { AmServerName amServerName = (AmServerName)stateInfo; lock (this.m_locker) { if (AmSystemManager.Instance.Config.IsPAM) { AmFailoverEntry amFailoverEntry = null; if (this.EntryMap.TryGetValue(amServerName, out amFailoverEntry)) { AmNodeState nodeState = AmSystemManager.Instance.Config.DagConfig.GetNodeState(amFailoverEntry.ServerName); if (nodeState != AmNodeState.Up) { AmEvtNodeDownForLongTime amEvtNodeDownForLongTime = new AmEvtNodeDownForLongTime(amFailoverEntry.ServerName); amEvtNodeDownForLongTime.Notify(); } else { ReplayCrimsonEvents.DelayedFailoverSkippedSinceNodeIsUp.Log <AmServerName>(amServerName); } } this.RemoveEntryInternal(amServerName, true); } } }
// Token: 0x06000144 RID: 324 RVA: 0x00007834 File Offset: 0x00005A34 internal AmNodeState GetNodeState(AmServerName nodeName) { AmNodeState result = AmNodeState.Unknown; try { IAmClusterNode amClusterNode2; IAmClusterNode amClusterNode = amClusterNode2 = this.Cluster.OpenNode(nodeName); try { result = amClusterNode.State; } finally { if (amClusterNode2 != null) { amClusterNode2.Dispose(); } } } catch (ClusterException ex) { AmTrace.Error("Failed to open cluster node {0} (error={1})", new object[] { nodeName, ex.Message }); } return(result); }
protected override bool RunIndividualCheck(IAmClusterNode node) { base.InstanceIdentity = node.Name.NetbiosName; AmNodeState state = node.GetState(false); if (state == AmNodeState.Paused) { base.FailContinue(Strings.DagMemberPausedFailed(node.Name.NetbiosName, this.m_dag.Name)); return(false); } return(true); }
protected override bool RunIndividualCheck(IAmClusterNode node) { base.InstanceIdentity = node.Name.NetbiosName; AmNodeState state = node.GetState(false); if ((state == AmNodeState.Down || state == AmNodeState.Joining || state == AmNodeState.Unknown) && !base.IsNodeStopped(node.Name)) { base.FailContinue(Strings.DagMemberUpCheckFailed(node.Name.NetbiosName, this.m_dag.Name)); return(false); } return(true); }
// Token: 0x060001DF RID: 479 RVA: 0x000088D8 File Offset: 0x00006AD8 public AmNodeState GetState(bool isThrowIfUnknown) { AmNodeState clusterNodeState = ClusapiMethods.GetClusterNodeState(this.Handle); if (clusterNodeState == AmNodeState.Unknown) { int lastWin32Error = Marshal.GetLastWin32Error(); Exception ex = new Win32Exception(lastWin32Error); AmTrace.Debug("GetClusterNodeState() returned error (rc={0}, message={1})", new object[] { lastWin32Error, ex }); if (isThrowIfUnknown) { throw AmExceptionHelper.ConstructClusterApiException(lastWin32Error, "GetClusterNodeState()", new object[0]); } } return(clusterNodeState); }
private bool IsNodePubliclyUp(AmNetworkMonitor.Node node) { IAmCluster cluster = this.GetCluster(); if (cluster == null) { AmNetworkMonitor.Tracer.TraceError <AmServerName>(0L, "If cluster object is not valid, then assume node {0} is up", node.Name); return(true); } Exception ex; AmNodeState nodeState = cluster.GetNodeState(node.Name, out ex); if (ex != null) { return(false); } if (!AmClusterNode.IsNodeUp(nodeState)) { return(false); } AmClusterNodeNetworkStatus amClusterNodeNetworkStatus = AmClusterNodeStatusAccessor.Read(cluster, node.Name, out ex); return(amClusterNodeNetworkStatus == null || amClusterNodeNetworkStatus.HasADAccess); }
// Token: 0x060001C8 RID: 456 RVA: 0x00008548 File Offset: 0x00006748 internal static bool IsNodeUp(AmNodeState nodeState) { return(nodeState == AmNodeState.Up || nodeState == AmNodeState.Paused); }
private void CheckClusterStateForDagServerRemoval() { this.m_output.AppendLogMessage("CheckClusterStateForDagServerRemoval entered. m_removeNode={0}, m_destroyCluster={1}", new object[] { this.m_removeNode, this.m_destroyCluster }); try { this.m_clusDag = AmCluster.OpenDagClus(this.m_dag); } catch (AmClusterException ex) { this.m_output.AppendLogMessage("Trying to open the cluster on the servers in the DAG '{0}' failed with exception {1}", new object[] { this.m_dagName, ex }); this.m_output.WriteErrorSimple(new DagTaskRemoveDagServerMustHaveQuorumException(this.m_dagName)); } using (DumpClusterTopology dumpClusterTopology = new DumpClusterTopology(this.m_clusDag, this.m_output)) { dumpClusterTopology.Dump(); } this.m_output.AppendLogMessage("Trying to open the node on the cluster.", new object[0]); IAmClusterNode amClusterNode = null; try { amClusterNode = this.m_clusDag.OpenNode(this.m_mailboxAmServerName); } catch (ClusterException ex2) { this.m_output.AppendLogMessage("OpenNode threw an exception. It's probably because the server is no longer clustered. Proceeding with -configurationonly.", new object[0]); this.m_output.AppendLogMessage("For the records, the exception was {0}.", new object[] { ex2 }); this.m_configurationOnly = true; this.m_mailboxServerIsDown = true; return; } using (amClusterNode) { AmNodeState state = amClusterNode.GetState(false); this.m_output.AppendLogMessage("Node.GetState( {0} ) reports that it is {1}.", new object[] { this.m_mailboxAmServerName, state }); if (!AmClusterNode.IsNodeUp(state)) { this.m_mailboxServerIsDown = true; } } if (!this.m_skipDagValidation) { try { DagTaskHelper.ValidateDagClusterMembership(this.m_output, this.m_dag, this.m_clusDag, this.m_mailboxAmServerName); } catch (ClusterException ex3) { this.DagTrace("ValidateDagClusterMembership() for the mailbox server failed possibly with error {0}, ex = {1}. This is OK.", new object[] { LocalizedException.GenerateErrorCode(ex3.InnerException).ToString(), ex3.Message }); } } int num = this.m_clusDag.EnumerateNodeNames().Count <AmServerName>(); this.DagTrace(string.Format("There are {0} nodes in the cluster.", num), new object[0]); if (num == 1) { this.m_destroyCluster = true; } else { this.m_removeNode = true; } bool destroyCluster = this.m_destroyCluster; this.ReopenClusterIfNecessary(); if ((this.m_removeNode || this.m_destroyCluster) && this.m_dag.DatacenterActivationMode != DatacenterActivationModeOption.Off) { DagTaskHelper.CheckDagCanBeActivatedInDatacenter(this.m_output, this.m_dag, (ADObjectId)this.m_mailboxServer.Identity, (ITopologyConfigurationSession)base.DataSession); } this.DagTrace("CheckClusterStateForDagServerRemoval left. m_removeNode={0}, m_destroyCluster={1}.", new object[] { this.m_removeNode, this.m_destroyCluster }); }
internal static LatencyChecker.ClusDbHungAction AnalyzeAndSuggestActionForClusDbHang(LatencyChecker.ClusDbHungInfo hungInfo) { LatencyChecker.ClusDbHungAction clusDbHungAction = new LatencyChecker.ClusDbHungAction(); clusDbHungAction.HungInfo = hungInfo; clusDbHungAction.TakeAction = false; clusDbHungAction.TargetNodes = hungInfo.HungNodes; clusDbHungAction.Reason = "If you see this message, something is wrong..."; if (hungInfo.HungNodeApiException != null) { if (hungInfo.HungNodeApiException is HungDetectionGumIdChangedException) { clusDbHungAction.TakeAction = false; clusDbHungAction.Reason = "GumId changed."; } else if (hungInfo.HungNodeApiException is OpenClusterTimedoutException) { clusDbHungAction.TakeAction = true; OpenClusterTimedoutException ex = (OpenClusterTimedoutException)hungInfo.HungNodeApiException; clusDbHungAction.Reason = string.Format("OpenCluster timed-out for {0}", ex.ServerName); } else if (hungInfo.HungNodeApiException is ClusterException) { clusDbHungAction.TakeAction = false; clusDbHungAction.Reason = "ClusterException was caught."; } } else { clusDbHungAction.TakeAction = true; clusDbHungAction.Reason = "Hung node detected without any Exceptions caught."; } if (clusDbHungAction.TargetNodes == null || clusDbHungAction.TargetNodes.Length < 1) { clusDbHungAction.TakeAction = false; clusDbHungAction.Reason = "No hung node detected, and Rpc timeout did not catch anything."; if (hungInfo.RpcFailedNodes != null && hungInfo.RpcFailedNodes.Length > 0) { AmServerName amServerName = null; foreach (AmServerName amServerName2 in hungInfo.RpcFailedNodes) { AmNodeState amNodeState = AmNodeState.Unknown; if (hungInfo.ClusterNodesStatus.TryGetValue(amServerName2.NetbiosName, out amNodeState) && amNodeState != AmNodeState.Unknown && amNodeState != AmNodeState.Down) { amServerName = amServerName2; break; } } if (amServerName != null) { clusDbHungAction.TakeAction = true; clusDbHungAction.TargetNodes = new AmServerName[] { amServerName }; clusDbHungAction.Reason = string.Format("Hung nodes detected via Rpc timeout. Node '{0}' chosen for action. Original list={1}", amServerName.NetbiosName, LatencyChecker.ConvertAmServerNamesToString(hungInfo.RpcFailedNodes)); } else { clusDbHungAction.TakeAction = false; clusDbHungAction.TargetNodes = null; clusDbHungAction.Reason = string.Format("No nodes in Rpc non-responsive list are UP according to cluster. Skipping reboot. Original list={0}", LatencyChecker.ConvertAmServerNamesToString(hungInfo.RpcFailedNodes)); } } if (!clusDbHungAction.TakeAction && !AmServerName.IsNullOrEmpty(hungInfo.CurrentLockOwnerName)) { clusDbHungAction.TakeAction = true; clusDbHungAction.TargetNodes = new AmServerName[] { hungInfo.CurrentLockOwnerName }; clusDbHungAction.Reason = string.Format("Could not find any hung nodes, so taking restart/reboot action for the lock owner '{0}'", hungInfo.CurrentLockOwnerName.NetbiosName); } } ReplayCrimsonEvents.HungNodeAnalysisResult.Log <string>(clusDbHungAction.ToString()); return(clusDbHungAction); }
protected override void WriteResult(IConfigurable dataObject) { TaskLogger.LogEnter(new object[] { dataObject.Identity, dataObject }); DatabaseAvailabilityGroup databaseAvailabilityGroup = (DatabaseAvailabilityGroup)dataObject; if (this.Status && !databaseAvailabilityGroup.IsDagEmpty()) { List <ADObjectId> list = new List <ADObjectId>(8); List <ADObjectId> list2 = new List <ADObjectId>(8); AmCluster amCluster = null; try { amCluster = AmCluster.OpenDagClus(databaseAvailabilityGroup); } catch (ClusterException exception) { base.WriteError(exception, ErrorCategory.InvalidArgument, null); } if (amCluster != null) { using (amCluster) { foreach (IAmClusterNode amClusterNode in amCluster.EnumerateNodes()) { using (amClusterNode) { AmNodeState state = amClusterNode.GetState(false); if (AmClusterNode.IsNodeUp(state)) { ADObjectId adobjectId = DagTaskHelper.FindServerAdObjectIdInDag(databaseAvailabilityGroup, amClusterNode.Name); if (adobjectId != null) { list.Add(adobjectId); if (state == AmNodeState.Paused) { list2.Add(adobjectId); } } else { this.WriteWarning(Strings.WarningClusterNodeNotFoundInDag(amClusterNode.Name.Fqdn, databaseAvailabilityGroup.Name)); } } } } databaseAvailabilityGroup.OperationalServers = list.ToArray(); databaseAvailabilityGroup.ServersInMaintenance = list2.ToArray(); DagNetworkConfiguration dagNetworkConfig = DagNetworkRpc.GetDagNetworkConfig(databaseAvailabilityGroup); databaseAvailabilityGroup.ReplicationPort = dagNetworkConfig.ReplicationPort; foreach (DatabaseAvailabilityGroupNetwork databaseAvailabilityGroupNetwork in dagNetworkConfig.Networks) { databaseAvailabilityGroup.NetworkNames.Add(databaseAvailabilityGroupNetwork.Name); } if (DagHelper.IsQuorumTypeFileShareWitness(amCluster)) { IAmClusterGroup amClusterGroup = amCluster.FindCoreClusterGroup(); if (amClusterGroup == null) { databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.InvalidConfiguration); this.WriteWarning(Strings.WarningClusterGroupNotFormed(databaseAvailabilityGroup.Name)); goto IL_306; } using (amClusterGroup) { IEnumerable <AmClusterResource> enumerable = null; try { enumerable = amClusterGroup.EnumerateResourcesOfType("File Share Witness"); AmClusterResource amClusterResource = enumerable.ElementAtOrDefault(0); if (amClusterResource == null) { databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.InvalidConfiguration); this.WriteWarning(Strings.WarningFswNotFound(databaseAvailabilityGroup.Name)); } else if (amClusterResource.GetState() == AmResourceState.Failed) { databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.InvalidConfiguration); this.WriteWarning(Strings.WarningFswFailed(databaseAvailabilityGroup.Name)); } else { string privateProperty = amClusterResource.GetPrivateProperty <string>("SharePath"); UncFileSharePath a; if (UncFileSharePath.TryParse(privateProperty, out a)) { if (a == databaseAvailabilityGroup.FileShareWitness) { databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.Primary); } else if (a == databaseAvailabilityGroup.AlternateFileShareWitness) { databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.Alternate); } else { databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.InvalidConfiguration); this.WriteWarning(Strings.WarningFswNotPrimaryOrAlternate(databaseAvailabilityGroup.Name)); } } else { databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.InvalidConfiguration); this.WriteWarning(Strings.WarningFswNotValidPath(databaseAvailabilityGroup.Name, privateProperty)); } } } finally { if (enumerable != null) { foreach (AmClusterResource amClusterResource2 in enumerable) { using (amClusterResource2) { } } } } goto IL_306; } } databaseAvailabilityGroup.WitnessShareInUse = new WitnessShareUsage?(WitnessShareUsage.None); IL_306:; } this.UpdatePam(databaseAvailabilityGroup); try { DeferredFailoverEntry[] serversInDeferredRecovery = DagTaskHelper.GetServersInDeferredRecovery(databaseAvailabilityGroup); databaseAvailabilityGroup.ServersInDeferredRecovery = serversInDeferredRecovery; } catch (AmServerException ex) { this.WriteWarning(Strings.PAMOtherError(ex.Message)); } } } base.WriteResult(databaseAvailabilityGroup); TaskLogger.LogExit(); }
// Token: 0x060000DC RID: 220 RVA: 0x00006358 File Offset: 0x00004558 private void MonitorClusterEvents() { while (!this.m_fShutdown) { AmClusterEventInfo amClusterEventInfo; bool flag = this.m_cen.WaitForEvent(out amClusterEventInfo, this.DefaultClusterEventNotifierTimeout); if (this.m_fShutdown) { AmClusterEventManager.Tracer.TraceDebug((long)this.GetHashCode(), "MonitorClusterEvents: Detected shutdown flag is set. Exiting from cluster notification monitoring"); return; } if (flag) { if (amClusterEventInfo.IsNotifyHandleClosed) { AmClusterEventManager.Tracer.TraceDebug((long)this.GetHashCode(), "Cluster notification handle closed. Exiting cluster event monitoring"); return; } if (amClusterEventInfo.IsNodeStateChanged) { AmServerName nodeName = new AmServerName(amClusterEventInfo.ObjectName); if (!string.IsNullOrEmpty(amClusterEventInfo.ObjectName)) { Exception ex; AmNodeState nodeState = this.Cluster.GetNodeState(nodeName, out ex); if (ex != null) { AmClusterEventManager.Tracer.TraceError <Exception>(0L, "MonitorClusterEvents fails to get node state: {0}", ex); } AmEvtNodeStateChanged amEvtNodeStateChanged = new AmEvtNodeStateChanged(nodeName, nodeState); amEvtNodeStateChanged.Notify(); if (ex != null) { throw ex; } } else { AmTrace.Error("Node state change detected but node name is invalid", new object[0]); } } if (amClusterEventInfo.IsGroupStateChanged) { AmSystemManager.Instance.ConfigManager.TriggerRefresh(false); } if (amClusterEventInfo.IsClusterStateChanged) { AmEvtClusterStateChanged amEvtClusterStateChanged = new AmEvtClusterStateChanged(); amEvtClusterStateChanged.Notify(); AmSystemManager.Instance.ConfigManager.TriggerRefresh(true); return; } if (amClusterEventInfo.IsNodeAdded) { AmEvtNodeAdded amEvtNodeAdded = new AmEvtNodeAdded(new AmServerName(amClusterEventInfo.ObjectName)); amEvtNodeAdded.Notify(); } if (amClusterEventInfo.IsNodeRemoved) { AmEvtNodeRemoved amEvtNodeRemoved = new AmEvtNodeRemoved(new AmServerName(amClusterEventInfo.ObjectName)); amEvtNodeRemoved.Notify(); AmServerNameCache.Instance.RemoveEntry(amClusterEventInfo.ObjectName); } AmSystemManager.Instance.NetworkMonitor.ProcessEvent(amClusterEventInfo); } } }
// Token: 0x06000503 RID: 1283 RVA: 0x0001AA1D File Offset: 0x00018C1D internal AmEvtNodeStateChanged(AmServerName nodeName, AmNodeState nodeState) { this.NodeName = nodeName; this.State = nodeState; }
private bool CheckClussvcRunningOnStartedServers() { bool result = false; List <AmServerName> list = new List <AmServerName>(16); List <AmServerName> list2 = new List <AmServerName>(16); new List <AmServerName>(16); List <string> list3 = new List <string>(16); foreach (Server server in this.m_startedServers.Values) { this.m_output.AppendLogMessage("Checking if clussvc is running on {0}...", new object[] { server.Name }); using (ServiceController serviceController = new ServiceController("clussvc", server.Name)) { ServiceControllerStatus serviceControllerStatus = DatabaseAvailabilityGroupAction.GetServiceControllerStatus(serviceController, server.Name, this.m_output); this.m_output.AppendLogMessage("Clussvc is {0} on {1}.", new object[] { serviceControllerStatus, server.Name }); if (serviceControllerStatus == ServiceControllerStatus.Running) { list.Add(new AmServerName(server)); } else { list2.Add(new AmServerName(server)); } } } if (list2.Count == this.m_startedServers.Values.Count) { this.m_output.AppendLogMessage("None of the {0} servers in the started list had clussvc running, so it is safe to continue with restore-dag, but forcequorum will be necessary.", new object[] { list2.Count }); } else { int num = 0; List <AmServerName> list4 = new List <AmServerName>(1); try { using (AmCluster amCluster = AmCluster.OpenByNames(list)) { IEnumerable <IAmClusterNode> enumerable = amCluster.EnumerateNodes(); foreach (IAmClusterNode amClusterNode in enumerable) { AmNodeState state = amClusterNode.State; this.m_output.AppendLogMessage("The node {0} has cluster state = {1}.", new object[] { amClusterNode.Name, state }); if (state == AmNodeState.Joining) { this.m_output.AppendLogMessage(" Node {0} is joining! That means there is some difficulty establishing quorum.", new object[] { amClusterNode.Name }); list3.Add(amClusterNode.Name.NetbiosName); } else if (AmClusterNode.IsNodeUp(state)) { num++; } if (state == AmNodeState.Paused) { list4.Add(amClusterNode.Name); } } foreach (IAmClusterNode amClusterNode2 in enumerable) { amClusterNode2.Dispose(); } } } catch (ClusterException) { this.m_output.WriteErrorSimple(new DagTaskRestoreDagCouldNotOpenClusterException()); } this.m_output.AppendLogMessage("There were {0} total (Stopped&Started) servers that were Up, out of {1} servers in the Started list that had clussvc running.", new object[] { num, list.Count }); if (num == list.Count) { this.m_output.AppendLogMessage("Because the two values are the same, that means quorum is healthy (even if not all of the servers are Up), so it is safe to continue with restore-dag.", new object[0]); result = true; } else if (num > list.Count) { this.m_output.AppendLogMessage("Because there are more servers that are up than there are in the Started list, it is safe to continue with restore-dag.", new object[0]); result = true; } else { this.m_output.AppendLogMessage("Because the two values are different, that means quorum is contested/in flux, so it is NOT safe to continue with restore-dag.", new object[0]); this.m_output.WriteErrorSimple(new RestoreFailedDagUpException(string.Join(",", list3.ToArray()))); } } return(result); }