internal static void OnClusApiHang(object context)
{
LatencyChecker.LatencyContext latencyContext = (LatencyChecker.LatencyContext)context;
LatencyChecker.ClusDbHungInfo clusDbHungInfo = LatencyChecker.GatherHungNodesInformation(latencyContext);
LatencyChecker.LastKnownHungInfo = clusDbHungInfo;
LatencyChecker.ClusDbHungAction action = LatencyChecker.AnalyzeAndSuggestActionForClusDbHang(clusDbHungInfo);
LatencyChecker.ActOnClusDbHang(action);
}
internal static void ReportClusApiHangLongLatency(object context)
{
LatencyChecker.LatencyContext latencyContext = (LatencyChecker.LatencyContext)context;
TimeSpan timeSpan = ExDateTime.Now - latencyContext.StartTime;
ReplayCrimsonEvents.ClusApiOperationAppearsToBeHungAlert.Log <string, ExDateTime, TimeSpan, string, TimeSpan>(latencyContext.ApiName, latencyContext.StartTime, timeSpan, latencyContext.HintStr, latencyContext.MaxAllowedLatency);
ReplayEventLogConstants.Tuple_ClusterApiHungAlert.LogEvent(null, new object[]
{
latencyContext.ApiName,
timeSpan.ToString()
});
LatencyChecker.RaiseRedEvent();
}
internal static int Measure(string apiName, string hintStr, TimeSpan maxAllowedLatency, TimeSpan maxAllowedLatencyForTimer, TimerCallback latencyCallback, Func <int> func, out TimeSpan elapsed)
{
ExDateTime now = ExDateTime.Now;
int num = 0;
bool flag = true;
Timer timer = null;
Timer timer2 = null;
try
{
if (latencyCallback != null && maxAllowedLatencyForTimer.TotalSeconds > 0.0)
{
LatencyChecker.LatencyContext latencyContext = new LatencyChecker.LatencyContext(now, apiName, hintStr, maxAllowedLatencyForTimer);
timer = new Timer(latencyCallback, latencyContext, -1, -1);
latencyContext.Timer = timer;
timer.Change(maxAllowedLatencyForTimer, TimeSpan.FromMilliseconds(-1.0));
timer2 = new Timer(new TimerCallback(LatencyChecker.ReportClusApiHangLongLatency), latencyContext, -1, -1);
TimeSpan dueTime = TimeSpan.FromSeconds((double)RegistryParameters.ClusApiHangReportLongLatencyDurationInSec);
timer2.Change(dueTime, TimeSpan.FromMilliseconds(-1.0));
}
if (RegistryParameters.IsApiLatencyTestEnabled)
{
LatencyChecker.DelayApiIfRequired(apiName);
num = RegistryParameters.GetApiSimulatedErrorCode(apiName);
if (num == 0)
{
num = func();
}
else
{
NativeMethods.SetLastError(num);
}
}
else
{
num = func();
}
flag = false;
}
finally
{
if (flag)
{
num = -1;
}
if (timer != null)
{
timer.Change(-1, -1);
timer.Dispose();
}
if (timer2 != null)
{
timer2.Change(-1, -1);
timer2.Dispose();
}
elapsed = ExDateTime.Now - now;
ExTraceGlobals.LatencyCheckerTracer.TraceDebug(0L, "Api={0}, StartTime={1}, Elapsed={2}, Hint={3}, IsUnhandled={4}, RetCode={5}, MaxLatency={6}", new object[]
{
apiName,
now,
elapsed,
hintStr,
flag,
num,
maxAllowedLatency
});
if (elapsed > maxAllowedLatency || (num != 0 && RegistryParameters.GetIsLogApiLatencyFailure()))
{
ReplayCrimsonEvents.OperationTookVeryLongTimeToComplete.Log <string, ExDateTime, TimeSpan, string, bool, int, TimeSpan>(apiName, now, elapsed, hintStr, flag, num, maxAllowedLatency);
}
}
return(num);
}
internal static LatencyChecker.ClusDbHungInfo GatherHungNodesInformation(LatencyChecker.LatencyContext latencyContext)
{
LatencyChecker.ClusDbHungInfo clusDbHungInfo = new LatencyChecker.ClusDbHungInfo();
TimeSpan timeSpan = ExDateTime.Now - latencyContext.StartTime;
clusDbHungInfo.ApiName = latencyContext.ApiName;
clusDbHungInfo.ApiHungStartTime = latencyContext.StartTime;
ReplayCrimsonEvents.ClusApiOperationAppearsToBeHung.Log <string, ExDateTime, TimeSpan, string, TimeSpan>(latencyContext.ApiName, latencyContext.StartTime, timeSpan, latencyContext.HintStr, latencyContext.MaxAllowedLatency);
clusDbHungInfo.HungNodeApiException = null;
try
{
ReplayCrimsonEvents.AttemptingToGetHungNodes.Log <string, ExDateTime, LatencyChecker.LatencyContext>(latencyContext.ApiName, latencyContext.StartTime, latencyContext);
HungNodesInfo nodesHungInClusDbUpdate = HungNodesInfo.GetNodesHungInClusDbUpdate();
if (nodesHungInClusDbUpdate != null)
{
ReplayCrimsonEvents.HungNodeDetectionCompleted.Log <int, AmServerName, HungNodesInfo>(nodesHungInClusDbUpdate.CurrentGumId, nodesHungInClusDbUpdate.CurrentLockOwnerName, nodesHungInClusDbUpdate);
clusDbHungInfo.CurrentGumId = nodesHungInClusDbUpdate.CurrentGumId;
clusDbHungInfo.CurrentLockOwnerName = nodesHungInClusDbUpdate.CurrentLockOwnerName;
clusDbHungInfo.HungNodes = nodesHungInClusDbUpdate.NodeMap.Values.ToArray <AmServerName>();
}
}
catch (HungDetectionGumIdChangedException ex)
{
clusDbHungInfo.HungNodeApiException = ex;
ReplayCrimsonEvents.HungActionSkippedSinceGumIdChanged.Log <int, int, string, long>(ex.LocalGumId, ex.RemoteGumId, ex.LockOwnerName, ex.HungNodesMask);
}
catch (OpenClusterTimedoutException ex2)
{
clusDbHungInfo.HungNodeApiException = ex2;
clusDbHungInfo.HungNodes = new AmServerName[]
{
new AmServerName(ex2.ServerName)
};
ReplayCrimsonEvents.OpenClusterCallHung.Log <string, string, string>(ex2.ServerName, ex2.Message, ex2.Context);
}
catch (ClusterException ex3)
{
clusDbHungInfo.HungNodeApiException = ex3;
ReplayCrimsonEvents.HungNodeDetectionFailed.Log <string, string>(ex3.Message, ex3.ToString());
}
List <AmServerName> dagServers = LatencyChecker.DagServers;
ReplayCrimsonEvents.HungNodeRpcScanStart.Log <string>(LatencyChecker.ConvertAmServerNamesToString(dagServers));
AmMultiNodeCopyStatusFetcher amMultiNodeCopyStatusFetcher = new AmMultiNodeCopyStatusFetcher(dagServers, LatencyChecker.DatabaseMap, RpcGetDatabaseCopyStatusFlags2.None, null, false, 60000);
amMultiNodeCopyStatusFetcher.GetStatus();
List <AmServerName> list = new List <AmServerName>();
List <Exception> list2 = new List <Exception>();
clusDbHungInfo.ClusterNodesStatus = LatencyChecker.QueryClusterNodeStatus(TimeSpan.FromSeconds(30.0), true);
foreach (AmServerName amServerName in LatencyChecker.DagServers)
{
Exception possibleExceptionForServer = amMultiNodeCopyStatusFetcher.GetPossibleExceptionForServer(amServerName);
if (possibleExceptionForServer != null)
{
if (possibleExceptionForServer is ReplayServiceDownException)
{
list.Add(amServerName);
}
list2.Add(possibleExceptionForServer);
}
}
clusDbHungInfo.RpcFailedNodes = list.ToArray();
clusDbHungInfo.RpcExceptions = list2.ToArray();
ReplayCrimsonEvents.HungNodeInformationLog.Log <string>(clusDbHungInfo.ToString());
return(clusDbHungInfo);
}
