private LockFile AcquireSessionLock(MonitoringSession session, string methodName = "")
{
string sessionFilePath = (session.EndDate != DateTime.MinValue.ToUniversalTime()) ? GetCpuMonitoringPath(MonitoringSessionDirectories.Completed) : GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
string lockFilePath = sessionFilePath + ".lock";
LockFile _lockFile = new LockFile(lockFilePath);
int loopCount = 0;
int lognum = 1;
int maximumWaitTimeInSeconds = 15 * 60;
while (!_lockFile.Lock($"AcquireSessionLock by {methodName} on {Environment.MachineName}") && loopCount <= maximumWaitTimeInSeconds)
{
++loopCount;
if (loopCount > lognum * 120)
{
++lognum;
Logger.LogCpuMonitoringVerboseEvent($"Waiting to acquire the lock on session file , loop {lognum}", session.SessionId);
}
Thread.Sleep(1000);
}
if (loopCount == maximumWaitTimeInSeconds)
{
Logger.LogCpuMonitoringVerboseEvent($"Deleting the lock file as it seems to be in an orphaned stage", session.SessionId);
_lockFile.Release();
return(null);
}
return(_lockFile);
}
private void ValidateSessionParameters(MonitoringSession monitoringSession)
{
if (monitoringSession.CpuThreshold < MIN_CPU_THRESHOLD)
{
throw new InvalidOperationException($"CpuThreshold cannot be less than {MIN_CPU_THRESHOLD} percent");
}
if (monitoringSession.MaxActions > MAX_CUSTOM_ACTIONS)
{
throw new InvalidOperationException($"MaxActions cannot be more than {MAX_CUSTOM_ACTIONS} actions");
}
if (monitoringSession.MaximumNumberOfHours > MAX_SESSION_DURATION)
{
throw new InvalidOperationException($"MaximumNumberOfHours cannot be more than {MAX_SESSION_DURATION} hours");
}
if (monitoringSession.MonitorDuration < MIN_MONITOR_DURATION_IN_SECONDS)
{
throw new InvalidOperationException($"MonitorDuration cannot be less than {MIN_MONITOR_DURATION_IN_SECONDS} seconds");
}
if (monitoringSession.ThresholdSeconds < MIN_THRESHOLD_DURATION_IN_SECONDS)
{
throw new InvalidOperationException($"ThresholdSeconds cannot be less than {MIN_THRESHOLD_DURATION_IN_SECONDS} seconds");
}
}
internal static void LogNewCpuMonitoringSession(MonitoringSession monitoringSession)
{
var details = new
{
monitoringSession.CpuThreshold,
monitoringSession.ThresholdSeconds,
monitoringSession.MonitorScmProcesses,
monitoringSession.MaxActions,
monitoringSession.MaximumNumberOfHours
};
DaasEventSource.Instance.LogNewCpuMonitoringSession(SiteName, _assemblyVersion, monitoringSession.SessionId, monitoringSession.Mode.ToString(), JsonConvert.SerializeObject(details));
LogDiagnostic("New CPU Monitoring Session submitted {0}", JsonConvert.SerializeObject(details));
}
public void SaveSession(MonitoringSession session, LockFile lockFile = null)
{
if (lockFile == null)
{
lockFile = AcquireSessionLock(session);
}
string cpuMonitorCompletedPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Completed);
cpuMonitorCompletedPath = Path.Combine(cpuMonitorCompletedPath, session.SessionId + ".json");
session.ToJsonFile(cpuMonitorCompletedPath);
if (lockFile != null)
{
lockFile.Release();
}
}
public void InitializeMonitoring(MonitoringSession session)
{
if (session != null)
{
if (session.ProcessesToMonitor != null)
{
_processesToMonitor = session.ProcessesToMonitor.Split(',').ToList();
}
_sessionId = session.SessionId;
var sessionDetails = JsonConvert.SerializeObject(session, Formatting.None);
AppendToMonitoringLog($"Monitoring started [CPU={session.CpuThreshold}%, Mode={session.Mode}, MaxActions={session.MaxActions}, Threshold={session.ThresholdSeconds}s, CheckEvery={session.MonitorDuration}s]", true);
}
ProcessList = new ConcurrentDictionary <int, MonitoredProcess>();
}
public MonitoringSession CreateSession(MonitoringSession monitoringSession)
{
string cpuMonitoringActive = GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitoringActive, "*.json", false, SearchOption.TopDirectoryOnly);
if (existingFiles.Count > 0)
{
throw new ApplicationException("Another monitoring session is already in progress");
}
else
{
ValidateSessionParameters(monitoringSession);
FileSystemHelpers.DeleteDirectoryContentsSafe(cpuMonitoringActive);
monitoringSession.StartDate = DateTime.UtcNow;
monitoringSession.EndDate = DateTime.MinValue.ToUniversalTime();
monitoringSession.SessionId = monitoringSession.StartDate.ToString(SessionConstants.SessionFileNameFormat);
cpuMonitoringActive = Path.Combine(cpuMonitoringActive, monitoringSession.SessionId + ".json");
monitoringSession.ToJsonFile(cpuMonitoringActive);
Logger.LogNewCpuMonitoringSession(monitoringSession);
}
return(monitoringSession);
}
public bool MonitorCpu(MonitoringSession session)
{
int cpuThreshold = session.CpuThreshold;
int seconds = session.ThresholdSeconds;
int monitorDuration = session.MonitorDuration;
string actionToExecute = session.ActionToExecute;
string argumentsToAction = session.ArgumentsToAction;
int maxActions = session.MaxActions == 0 ? int.MaxValue : session.MaxActions;
bool monitorScmProcesses = session.MonitorScmProcesses;
string blobSasUri = session.BlobSasUri;
if (string.IsNullOrWhiteSpace(actionToExecute))
{
actionToExecute = @"D:\devtools\sysinternals\procdump.exe";
}
if (string.IsNullOrWhiteSpace(argumentsToAction))
{
argumentsToAction = " -accepteula -ma {PROCESSID} {OUTPUTPATH}";
}
foreach (var process in Process.GetProcesses())
{
if (_processesToMonitor.Count > 0)
{
if (!_processesToMonitor.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
}
if (!monitorScmProcesses)
{
var envVar = Utilities.GetEnvironmentVariablesCore(process.Handle);
if (Utilities.GetIsScmSite(envVar))
{
continue;
}
}
if (_processesAlwaysExcluded.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
int id = process.Id;
if (!ProcessList.ContainsKey(id))
{
var cpuTime = GetProcessCpuTime(id);
MonitoredProcess p = new MonitoredProcess
{
CPUTimeStart = cpuTime,
Name = Process.GetProcessById(id).ProcessName,
CPUTimeCurrent = cpuTime,
LastMonitorTime = DateTime.UtcNow,
ThresholdExeededCount = 0
};
bool booProcessAdded = ProcessList.TryAdd(id, p);
if (booProcessAdded)
{
AppendToMonitoringLog($"Added process {process.ProcessName}({id}) to monitoring", true);
}
}
}
var processesMonitored = new List <string>();
foreach (var id in ProcessList.Keys)
{
var start = ProcessList[id].CPUTimeStart;
var oldCPUTime = ProcessList[id].CPUTimeCurrent;
var processCpuTime = GetProcessCpuTime(id);
if (processCpuTime != TimeSpan.MinValue)
{
TimeSpan newCPUTime = processCpuTime - start;
var cpuTimeSeconds = (newCPUTime - oldCPUTime).TotalSeconds;
var durationSeconds = DateTime.UtcNow.Subtract(ProcessList[id].LastMonitorTime).TotalSeconds;
// for the first time CPU Time will be
// negative as startTime is not subtracted
if (cpuTimeSeconds < 0)
{
CpuUsageLastMinute = 0;
}
else
{
CpuUsageLastMinute = cpuTimeSeconds / (Environment.ProcessorCount * durationSeconds);
}
ProcessList[id].CPUTimeCurrent = newCPUTime;
ProcessList[id].LastMonitorTime = DateTime.UtcNow;;
var cpuPercent = CpuUsageLastMinute * 100;
processesMonitored.Add($"{ProcessList[id].Name}({id}):{cpuPercent.ToString("0")} %");
var actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
bool terminateMonitoring = false;
if (actionsExecuted >= maxActions)
{
AppendToMonitoringLog("Maximum number of actions configured on all instances have executed so terminating shortly!", true);
terminateMonitoring = true;
}
if (DateTime.UtcNow.Subtract(session.StartDate).TotalHours >= session.MaximumNumberOfHours)
{
AppendToMonitoringLog("Maximum time limit for this session has reached so terminating shortly!", true);
terminateMonitoring = true;
}
if (terminateMonitoring)
{
var dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId);
int sleepCount = 0;
AppendToMonitoringLog("Waiting for all instances to collect and move the dumps", true);
while (dumpsCollected < actionsExecuted && sleepCount < 20)
{
AppendToMonitoringLog($"Total actions executed = {actionsExecuted} and dumps moved = {dumpsCollected}, waiting for the rest to finish", true);
Thread.Sleep(15000);
++sleepCount;
actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId);
}
AppendToMonitoringLog("All instances finsihed collecting data so terminating", true);
return(true);
}
if (cpuPercent >= cpuThreshold)
{
AppendToMonitoringLog($"{ProcessList[id].Name}({id}) CPU:{cpuPercent.ToString("0.00")} %");
int thresholdCount = ++ProcessList[id].ThresholdExeededCount;
int currentCpuConsumptionWithTime = thresholdCount * monitorDuration;
if (currentCpuConsumptionWithTime >= seconds)
{
actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
string fileName = Environment.MachineName + "_" + ProcessList[id].Name + "_" + id + "_" + DateTime.Now.Ticks.ToString();
string customActionFile = fileName + ".customaction";
string outputPath = MonitoringSessionController.TempFilePath;
FileSystemHelpers.CreateDirectoryIfNotExists(outputPath);
string dumpFileInTempDirectory = Path.Combine(outputPath, fileName + ".dmp");
if (session.Mode != SessionMode.Kill)
{
AppendToMonitoringLog($"Actions Executed on all instances = {actionsExecuted} of {maxActions}", true);
}
if (ShouldCollectData(session.Mode))
{
CreateCustomActionFile(session.SessionId, customActionFile);
ExecuteAction(dumpFileInTempDirectory, id, actionToExecute, argumentsToAction);
}
if (ShouldKillProcess(session.Mode))
{
KillProcessConsumingCpu(id, session.SessionId);
}
if (ShouldCollectData(session.Mode))
{
MoveToPermanentStorage(session.SessionId, dumpFileInTempDirectory, fileName + ".dmp", blobSasUri);
customActionFile = fileName + ".customactioncompleted";
CreateCustomActionFile(session.SessionId, customActionFile);
// Since we copied the file to permanent storage, delete the time file if the mode
// doesnt require analysis to be done or if the number of instances is more than 1
if (session.Mode != SessionMode.CollectKillAndAnalyze || HeartBeats.HeartBeatController.GetNumberOfLiveInstances() > 1)
{
FileSystemHelpers.DeleteFileSafe(dumpFileInTempDirectory);
}
}
actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
ProcessList[id].ThresholdExeededCount = 0;
if (actionsExecuted >= maxActions)
{
AppendToMonitoringLog("Max number of actions configured for this session have executed so terminating shortly!", true);
Thread.Sleep(5000);
return(true);
}
}
else
{
AppendToMonitoringLog($"CPU Percent {cpuPercent.ToString("0.00")} % > [{ cpuThreshold } %] for {currentCpuConsumptionWithTime} seconds for { ProcessList[id].Name} ({ id }), waiting to reach threshold of {seconds} seconds", true);
}
}
else
{
if (ProcessList[id].ThresholdExeededCount > 0)
{
AppendToMonitoringLog("Resetting CPU threshold", true);
ProcessList[id].ThresholdExeededCount = 0;
}
}
}
else
{
Trace.TraceInformation($"VERBOSE: processCpuTime == TimeSpan.MinValue ");
}
}
AppendToMonitoringLog(string.Join(", ", processesMonitored));
RemoveOldProcessesFromMonitoringList(ProcessList);
return(false);
}
