internal static void OnClusApiHang(object context)
{
LatencyChecker.LatencyContext latencyContext = (LatencyChecker.LatencyContext)context;
LatencyChecker.ClusDbHungInfo clusDbHungInfo = LatencyChecker.GatherHungNodesInformation(latencyContext);
LatencyChecker.LastKnownHungInfo = clusDbHungInfo;
LatencyChecker.ClusDbHungAction action = LatencyChecker.AnalyzeAndSuggestActionForClusDbHang(clusDbHungInfo);
LatencyChecker.ActOnClusDbHang(action);
}
private static void TriggerNodeRestart(string currentGumId, string currentLockOwnerName, string hungNodeCsv, LatencyChecker.ClusDbHungInfo hungInfo, LatencyChecker.ClusDbHungAction hungAction)
{
EventNotificationItem eventNotificationItem = new EventNotificationItem("MSExchangeRepl", "Cluster", "ClusterNodeRestart", string.Format("Cluster Hung detected. GumId={0}, LockOwner={1}, HungNodes={2}, HungInfo={3}, Decision={4}", new object[]
{
currentGumId,
currentLockOwnerName,
hungNodeCsv,
hungInfo.ToString(),
hungAction.ToString()
}), hungNodeCsv, ResultSeverityLevel.Critical);
eventNotificationItem.Publish(false);
}
internal static LatencyChecker.ClusDbHungAction AnalyzeAndSuggestActionForClusDbHang(LatencyChecker.ClusDbHungInfo hungInfo)
{
LatencyChecker.ClusDbHungAction clusDbHungAction = new LatencyChecker.ClusDbHungAction();
clusDbHungAction.HungInfo = hungInfo;
clusDbHungAction.TakeAction = false;
clusDbHungAction.TargetNodes = hungInfo.HungNodes;
clusDbHungAction.Reason = "If you see this message, something is wrong...";
if (hungInfo.HungNodeApiException != null)
{
if (hungInfo.HungNodeApiException is HungDetectionGumIdChangedException)
{
clusDbHungAction.TakeAction = false;
clusDbHungAction.Reason = "GumId changed.";
}
else if (hungInfo.HungNodeApiException is OpenClusterTimedoutException)
{
clusDbHungAction.TakeAction = true;
OpenClusterTimedoutException ex = (OpenClusterTimedoutException)hungInfo.HungNodeApiException;
clusDbHungAction.Reason = string.Format("OpenCluster timed-out for {0}", ex.ServerName);
}
else if (hungInfo.HungNodeApiException is ClusterException)
{
clusDbHungAction.TakeAction = false;
clusDbHungAction.Reason = "ClusterException was caught.";
}
}
else
{
clusDbHungAction.TakeAction = true;
clusDbHungAction.Reason = "Hung node detected without any Exceptions caught.";
}
if (clusDbHungAction.TargetNodes == null || clusDbHungAction.TargetNodes.Length < 1)
{
clusDbHungAction.TakeAction = false;
clusDbHungAction.Reason = "No hung node detected, and Rpc timeout did not catch anything.";
if (hungInfo.RpcFailedNodes != null && hungInfo.RpcFailedNodes.Length > 0)
{
AmServerName amServerName = null;
foreach (AmServerName amServerName2 in hungInfo.RpcFailedNodes)
{
AmNodeState amNodeState = AmNodeState.Unknown;
if (hungInfo.ClusterNodesStatus.TryGetValue(amServerName2.NetbiosName, out amNodeState) && amNodeState != AmNodeState.Unknown && amNodeState != AmNodeState.Down)
{
amServerName = amServerName2;
break;
}
}
if (amServerName != null)
{
clusDbHungAction.TakeAction = true;
clusDbHungAction.TargetNodes = new AmServerName[]
{
amServerName
};
clusDbHungAction.Reason = string.Format("Hung nodes detected via Rpc timeout. Node '{0}' chosen for action. Original list={1}", amServerName.NetbiosName, LatencyChecker.ConvertAmServerNamesToString(hungInfo.RpcFailedNodes));
}
else
{
clusDbHungAction.TakeAction = false;
clusDbHungAction.TargetNodes = null;
clusDbHungAction.Reason = string.Format("No nodes in Rpc non-responsive list are UP according to cluster. Skipping reboot. Original list={0}", LatencyChecker.ConvertAmServerNamesToString(hungInfo.RpcFailedNodes));
}
}
if (!clusDbHungAction.TakeAction && !AmServerName.IsNullOrEmpty(hungInfo.CurrentLockOwnerName))
{
clusDbHungAction.TakeAction = true;
clusDbHungAction.TargetNodes = new AmServerName[]
{
hungInfo.CurrentLockOwnerName
};
clusDbHungAction.Reason = string.Format("Could not find any hung nodes, so taking restart/reboot action for the lock owner '{0}'", hungInfo.CurrentLockOwnerName.NetbiosName);
}
}
ReplayCrimsonEvents.HungNodeAnalysisResult.Log <string>(clusDbHungAction.ToString());
return(clusDbHungAction);
}
internal static LatencyChecker.ClusDbHungInfo GatherHungNodesInformation(LatencyChecker.LatencyContext latencyContext)
{
LatencyChecker.ClusDbHungInfo clusDbHungInfo = new LatencyChecker.ClusDbHungInfo();
TimeSpan timeSpan = ExDateTime.Now - latencyContext.StartTime;
clusDbHungInfo.ApiName = latencyContext.ApiName;
clusDbHungInfo.ApiHungStartTime = latencyContext.StartTime;
ReplayCrimsonEvents.ClusApiOperationAppearsToBeHung.Log <string, ExDateTime, TimeSpan, string, TimeSpan>(latencyContext.ApiName, latencyContext.StartTime, timeSpan, latencyContext.HintStr, latencyContext.MaxAllowedLatency);
clusDbHungInfo.HungNodeApiException = null;
try
{
ReplayCrimsonEvents.AttemptingToGetHungNodes.Log <string, ExDateTime, LatencyChecker.LatencyContext>(latencyContext.ApiName, latencyContext.StartTime, latencyContext);
HungNodesInfo nodesHungInClusDbUpdate = HungNodesInfo.GetNodesHungInClusDbUpdate();
if (nodesHungInClusDbUpdate != null)
{
ReplayCrimsonEvents.HungNodeDetectionCompleted.Log <int, AmServerName, HungNodesInfo>(nodesHungInClusDbUpdate.CurrentGumId, nodesHungInClusDbUpdate.CurrentLockOwnerName, nodesHungInClusDbUpdate);
clusDbHungInfo.CurrentGumId = nodesHungInClusDbUpdate.CurrentGumId;
clusDbHungInfo.CurrentLockOwnerName = nodesHungInClusDbUpdate.CurrentLockOwnerName;
clusDbHungInfo.HungNodes = nodesHungInClusDbUpdate.NodeMap.Values.ToArray <AmServerName>();
}
}
catch (HungDetectionGumIdChangedException ex)
{
clusDbHungInfo.HungNodeApiException = ex;
ReplayCrimsonEvents.HungActionSkippedSinceGumIdChanged.Log <int, int, string, long>(ex.LocalGumId, ex.RemoteGumId, ex.LockOwnerName, ex.HungNodesMask);
}
catch (OpenClusterTimedoutException ex2)
{
clusDbHungInfo.HungNodeApiException = ex2;
clusDbHungInfo.HungNodes = new AmServerName[]
{
new AmServerName(ex2.ServerName)
};
ReplayCrimsonEvents.OpenClusterCallHung.Log <string, string, string>(ex2.ServerName, ex2.Message, ex2.Context);
}
catch (ClusterException ex3)
{
clusDbHungInfo.HungNodeApiException = ex3;
ReplayCrimsonEvents.HungNodeDetectionFailed.Log <string, string>(ex3.Message, ex3.ToString());
}
List <AmServerName> dagServers = LatencyChecker.DagServers;
ReplayCrimsonEvents.HungNodeRpcScanStart.Log <string>(LatencyChecker.ConvertAmServerNamesToString(dagServers));
AmMultiNodeCopyStatusFetcher amMultiNodeCopyStatusFetcher = new AmMultiNodeCopyStatusFetcher(dagServers, LatencyChecker.DatabaseMap, RpcGetDatabaseCopyStatusFlags2.None, null, false, 60000);
amMultiNodeCopyStatusFetcher.GetStatus();
List <AmServerName> list = new List <AmServerName>();
List <Exception> list2 = new List <Exception>();
clusDbHungInfo.ClusterNodesStatus = LatencyChecker.QueryClusterNodeStatus(TimeSpan.FromSeconds(30.0), true);
foreach (AmServerName amServerName in LatencyChecker.DagServers)
{
Exception possibleExceptionForServer = amMultiNodeCopyStatusFetcher.GetPossibleExceptionForServer(amServerName);
if (possibleExceptionForServer != null)
{
if (possibleExceptionForServer is ReplayServiceDownException)
{
list.Add(amServerName);
}
list2.Add(possibleExceptionForServer);
}
}
clusDbHungInfo.RpcFailedNodes = list.ToArray();
clusDbHungInfo.RpcExceptions = list2.ToArray();
ReplayCrimsonEvents.HungNodeInformationLog.Log <string>(clusDbHungInfo.ToString());
return(clusDbHungInfo);
}
