private static void TriggerNodeRestart(string currentGumId, string currentLockOwnerName, string hungNodeCsv, LatencyChecker.ClusDbHungInfo hungInfo, LatencyChecker.ClusDbHungAction hungAction) { EventNotificationItem eventNotificationItem = new EventNotificationItem("MSExchangeRepl", "Cluster", "ClusterNodeRestart", string.Format("Cluster Hung detected. GumId={0}, LockOwner={1}, HungNodes={2}, HungInfo={3}, Decision={4}", new object[] { currentGumId, currentLockOwnerName, hungNodeCsv, hungInfo.ToString(), hungAction.ToString() }), hungNodeCsv, ResultSeverityLevel.Critical); eventNotificationItem.Publish(false); }
internal static LatencyChecker.ClusDbHungInfo GatherHungNodesInformation(LatencyChecker.LatencyContext latencyContext) { LatencyChecker.ClusDbHungInfo clusDbHungInfo = new LatencyChecker.ClusDbHungInfo(); TimeSpan timeSpan = ExDateTime.Now - latencyContext.StartTime; clusDbHungInfo.ApiName = latencyContext.ApiName; clusDbHungInfo.ApiHungStartTime = latencyContext.StartTime; ReplayCrimsonEvents.ClusApiOperationAppearsToBeHung.Log <string, ExDateTime, TimeSpan, string, TimeSpan>(latencyContext.ApiName, latencyContext.StartTime, timeSpan, latencyContext.HintStr, latencyContext.MaxAllowedLatency); clusDbHungInfo.HungNodeApiException = null; try { ReplayCrimsonEvents.AttemptingToGetHungNodes.Log <string, ExDateTime, LatencyChecker.LatencyContext>(latencyContext.ApiName, latencyContext.StartTime, latencyContext); HungNodesInfo nodesHungInClusDbUpdate = HungNodesInfo.GetNodesHungInClusDbUpdate(); if (nodesHungInClusDbUpdate != null) { ReplayCrimsonEvents.HungNodeDetectionCompleted.Log <int, AmServerName, HungNodesInfo>(nodesHungInClusDbUpdate.CurrentGumId, nodesHungInClusDbUpdate.CurrentLockOwnerName, nodesHungInClusDbUpdate); clusDbHungInfo.CurrentGumId = nodesHungInClusDbUpdate.CurrentGumId; clusDbHungInfo.CurrentLockOwnerName = nodesHungInClusDbUpdate.CurrentLockOwnerName; clusDbHungInfo.HungNodes = nodesHungInClusDbUpdate.NodeMap.Values.ToArray <AmServerName>(); } } catch (HungDetectionGumIdChangedException ex) { clusDbHungInfo.HungNodeApiException = ex; ReplayCrimsonEvents.HungActionSkippedSinceGumIdChanged.Log <int, int, string, long>(ex.LocalGumId, ex.RemoteGumId, ex.LockOwnerName, ex.HungNodesMask); } catch (OpenClusterTimedoutException ex2) { clusDbHungInfo.HungNodeApiException = ex2; clusDbHungInfo.HungNodes = new AmServerName[] { new AmServerName(ex2.ServerName) }; ReplayCrimsonEvents.OpenClusterCallHung.Log <string, string, string>(ex2.ServerName, ex2.Message, ex2.Context); } catch (ClusterException ex3) { clusDbHungInfo.HungNodeApiException = ex3; ReplayCrimsonEvents.HungNodeDetectionFailed.Log <string, string>(ex3.Message, ex3.ToString()); } List <AmServerName> dagServers = LatencyChecker.DagServers; ReplayCrimsonEvents.HungNodeRpcScanStart.Log <string>(LatencyChecker.ConvertAmServerNamesToString(dagServers)); AmMultiNodeCopyStatusFetcher amMultiNodeCopyStatusFetcher = new AmMultiNodeCopyStatusFetcher(dagServers, LatencyChecker.DatabaseMap, RpcGetDatabaseCopyStatusFlags2.None, null, false, 60000); amMultiNodeCopyStatusFetcher.GetStatus(); List <AmServerName> list = new List <AmServerName>(); List <Exception> list2 = new List <Exception>(); clusDbHungInfo.ClusterNodesStatus = LatencyChecker.QueryClusterNodeStatus(TimeSpan.FromSeconds(30.0), true); foreach (AmServerName amServerName in LatencyChecker.DagServers) { Exception possibleExceptionForServer = amMultiNodeCopyStatusFetcher.GetPossibleExceptionForServer(amServerName); if (possibleExceptionForServer != null) { if (possibleExceptionForServer is ReplayServiceDownException) { list.Add(amServerName); } list2.Add(possibleExceptionForServer); } } clusDbHungInfo.RpcFailedNodes = list.ToArray(); clusDbHungInfo.RpcExceptions = list2.ToArray(); ReplayCrimsonEvents.HungNodeInformationLog.Log <string>(clusDbHungInfo.ToString()); return(clusDbHungInfo); }