private LockFile AcquireSessionLock(MonitoringSession session, string methodName = "") { string sessionFilePath = (session.EndDate != DateTime.MinValue.ToUniversalTime()) ? GetCpuMonitoringPath(MonitoringSessionDirectories.Completed) : GetCpuMonitoringPath(MonitoringSessionDirectories.Active); string lockFilePath = sessionFilePath + ".lock"; LockFile _lockFile = new LockFile(lockFilePath); int loopCount = 0; int lognum = 1; int maximumWaitTimeInSeconds = 15 * 60; while (!_lockFile.Lock($"AcquireSessionLock by {methodName} on {Environment.MachineName}") && loopCount <= maximumWaitTimeInSeconds) { ++loopCount; if (loopCount > lognum * 120) { ++lognum; Logger.LogCpuMonitoringVerboseEvent($"Waiting to acquire the lock on session file , loop {lognum}", session.SessionId); } Thread.Sleep(1000); } if (loopCount == maximumWaitTimeInSeconds) { Logger.LogCpuMonitoringVerboseEvent($"Deleting the lock file as it seems to be in an orphaned stage", session.SessionId); _lockFile.Release(); return(null); } return(_lockFile); }
private void ValidateSessionParameters(MonitoringSession monitoringSession) { if (monitoringSession.CpuThreshold < MIN_CPU_THRESHOLD) { throw new InvalidOperationException($"CpuThreshold cannot be less than {MIN_CPU_THRESHOLD} percent"); } if (monitoringSession.MaxActions > MAX_CUSTOM_ACTIONS) { throw new InvalidOperationException($"MaxActions cannot be more than {MAX_CUSTOM_ACTIONS} actions"); } if (monitoringSession.MaximumNumberOfHours > MAX_SESSION_DURATION) { throw new InvalidOperationException($"MaximumNumberOfHours cannot be more than {MAX_SESSION_DURATION} hours"); } if (monitoringSession.MonitorDuration < MIN_MONITOR_DURATION_IN_SECONDS) { throw new InvalidOperationException($"MonitorDuration cannot be less than {MIN_MONITOR_DURATION_IN_SECONDS} seconds"); } if (monitoringSession.ThresholdSeconds < MIN_THRESHOLD_DURATION_IN_SECONDS) { throw new InvalidOperationException($"ThresholdSeconds cannot be less than {MIN_THRESHOLD_DURATION_IN_SECONDS} seconds"); } }
internal static void LogNewCpuMonitoringSession(MonitoringSession monitoringSession) { var details = new { monitoringSession.CpuThreshold, monitoringSession.ThresholdSeconds, monitoringSession.MonitorScmProcesses, monitoringSession.MaxActions, monitoringSession.MaximumNumberOfHours }; DaasEventSource.Instance.LogNewCpuMonitoringSession(SiteName, _assemblyVersion, monitoringSession.SessionId, monitoringSession.Mode.ToString(), JsonConvert.SerializeObject(details)); LogDiagnostic("New CPU Monitoring Session submitted {0}", JsonConvert.SerializeObject(details)); }
public void SaveSession(MonitoringSession session, LockFile lockFile = null) { if (lockFile == null) { lockFile = AcquireSessionLock(session); } string cpuMonitorCompletedPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Completed); cpuMonitorCompletedPath = Path.Combine(cpuMonitorCompletedPath, session.SessionId + ".json"); session.ToJsonFile(cpuMonitorCompletedPath); if (lockFile != null) { lockFile.Release(); } }
public void InitializeMonitoring(MonitoringSession session) { if (session != null) { if (session.ProcessesToMonitor != null) { _processesToMonitor = session.ProcessesToMonitor.Split(',').ToList(); } _sessionId = session.SessionId; var sessionDetails = JsonConvert.SerializeObject(session, Formatting.None); AppendToMonitoringLog($"Monitoring started [CPU={session.CpuThreshold}%, Mode={session.Mode}, MaxActions={session.MaxActions}, Threshold={session.ThresholdSeconds}s, CheckEvery={session.MonitorDuration}s]", true); } ProcessList = new ConcurrentDictionary <int, MonitoredProcess>(); }
public MonitoringSession CreateSession(MonitoringSession monitoringSession) { string cpuMonitoringActive = GetCpuMonitoringPath(MonitoringSessionDirectories.Active); var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitoringActive, "*.json", false, SearchOption.TopDirectoryOnly); if (existingFiles.Count > 0) { throw new ApplicationException("Another monitoring session is already in progress"); } else { ValidateSessionParameters(monitoringSession); FileSystemHelpers.DeleteDirectoryContentsSafe(cpuMonitoringActive); monitoringSession.StartDate = DateTime.UtcNow; monitoringSession.EndDate = DateTime.MinValue.ToUniversalTime(); monitoringSession.SessionId = monitoringSession.StartDate.ToString(SessionConstants.SessionFileNameFormat); cpuMonitoringActive = Path.Combine(cpuMonitoringActive, monitoringSession.SessionId + ".json"); monitoringSession.ToJsonFile(cpuMonitoringActive); Logger.LogNewCpuMonitoringSession(monitoringSession); } return(monitoringSession); }
public bool MonitorCpu(MonitoringSession session) { int cpuThreshold = session.CpuThreshold; int seconds = session.ThresholdSeconds; int monitorDuration = session.MonitorDuration; string actionToExecute = session.ActionToExecute; string argumentsToAction = session.ArgumentsToAction; int maxActions = session.MaxActions == 0 ? int.MaxValue : session.MaxActions; bool monitorScmProcesses = session.MonitorScmProcesses; string blobSasUri = session.BlobSasUri; if (string.IsNullOrWhiteSpace(actionToExecute)) { actionToExecute = @"D:\devtools\sysinternals\procdump.exe"; } if (string.IsNullOrWhiteSpace(argumentsToAction)) { argumentsToAction = " -accepteula -ma {PROCESSID} {OUTPUTPATH}"; } foreach (var process in Process.GetProcesses()) { if (_processesToMonitor.Count > 0) { if (!_processesToMonitor.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase))) { continue; } } if (!monitorScmProcesses) { var envVar = Utilities.GetEnvironmentVariablesCore(process.Handle); if (Utilities.GetIsScmSite(envVar)) { continue; } } if (_processesAlwaysExcluded.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase))) { continue; } int id = process.Id; if (!ProcessList.ContainsKey(id)) { var cpuTime = GetProcessCpuTime(id); MonitoredProcess p = new MonitoredProcess { CPUTimeStart = cpuTime, Name = Process.GetProcessById(id).ProcessName, CPUTimeCurrent = cpuTime, LastMonitorTime = DateTime.UtcNow, ThresholdExeededCount = 0 }; bool booProcessAdded = ProcessList.TryAdd(id, p); if (booProcessAdded) { AppendToMonitoringLog($"Added process {process.ProcessName}({id}) to monitoring", true); } } } var processesMonitored = new List <string>(); foreach (var id in ProcessList.Keys) { var start = ProcessList[id].CPUTimeStart; var oldCPUTime = ProcessList[id].CPUTimeCurrent; var processCpuTime = GetProcessCpuTime(id); if (processCpuTime != TimeSpan.MinValue) { TimeSpan newCPUTime = processCpuTime - start; var cpuTimeSeconds = (newCPUTime - oldCPUTime).TotalSeconds; var durationSeconds = DateTime.UtcNow.Subtract(ProcessList[id].LastMonitorTime).TotalSeconds; // for the first time CPU Time will be // negative as startTime is not subtracted if (cpuTimeSeconds < 0) { CpuUsageLastMinute = 0; } else { CpuUsageLastMinute = cpuTimeSeconds / (Environment.ProcessorCount * durationSeconds); } ProcessList[id].CPUTimeCurrent = newCPUTime; ProcessList[id].LastMonitorTime = DateTime.UtcNow;; var cpuPercent = CpuUsageLastMinute * 100; processesMonitored.Add($"{ProcessList[id].Name}({id}):{cpuPercent.ToString("0")} %"); var actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId); bool terminateMonitoring = false; if (actionsExecuted >= maxActions) { AppendToMonitoringLog("Maximum number of actions configured on all instances have executed so terminating shortly!", true); terminateMonitoring = true; } if (DateTime.UtcNow.Subtract(session.StartDate).TotalHours >= session.MaximumNumberOfHours) { AppendToMonitoringLog("Maximum time limit for this session has reached so terminating shortly!", true); terminateMonitoring = true; } if (terminateMonitoring) { var dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId); int sleepCount = 0; AppendToMonitoringLog("Waiting for all instances to collect and move the dumps", true); while (dumpsCollected < actionsExecuted && sleepCount < 20) { AppendToMonitoringLog($"Total actions executed = {actionsExecuted} and dumps moved = {dumpsCollected}, waiting for the rest to finish", true); Thread.Sleep(15000); ++sleepCount; actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId); dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId); } AppendToMonitoringLog("All instances finsihed collecting data so terminating", true); return(true); } if (cpuPercent >= cpuThreshold) { AppendToMonitoringLog($"{ProcessList[id].Name}({id}) CPU:{cpuPercent.ToString("0.00")} %"); int thresholdCount = ++ProcessList[id].ThresholdExeededCount; int currentCpuConsumptionWithTime = thresholdCount * monitorDuration; if (currentCpuConsumptionWithTime >= seconds) { actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId); string fileName = Environment.MachineName + "_" + ProcessList[id].Name + "_" + id + "_" + DateTime.Now.Ticks.ToString(); string customActionFile = fileName + ".customaction"; string outputPath = MonitoringSessionController.TempFilePath; FileSystemHelpers.CreateDirectoryIfNotExists(outputPath); string dumpFileInTempDirectory = Path.Combine(outputPath, fileName + ".dmp"); if (session.Mode != SessionMode.Kill) { AppendToMonitoringLog($"Actions Executed on all instances = {actionsExecuted} of {maxActions}", true); } if (ShouldCollectData(session.Mode)) { CreateCustomActionFile(session.SessionId, customActionFile); ExecuteAction(dumpFileInTempDirectory, id, actionToExecute, argumentsToAction); } if (ShouldKillProcess(session.Mode)) { KillProcessConsumingCpu(id, session.SessionId); } if (ShouldCollectData(session.Mode)) { MoveToPermanentStorage(session.SessionId, dumpFileInTempDirectory, fileName + ".dmp", blobSasUri); customActionFile = fileName + ".customactioncompleted"; CreateCustomActionFile(session.SessionId, customActionFile); // Since we copied the file to permanent storage, delete the time file if the mode // doesnt require analysis to be done or if the number of instances is more than 1 if (session.Mode != SessionMode.CollectKillAndAnalyze || HeartBeats.HeartBeatController.GetNumberOfLiveInstances() > 1) { FileSystemHelpers.DeleteFileSafe(dumpFileInTempDirectory); } } actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId); ProcessList[id].ThresholdExeededCount = 0; if (actionsExecuted >= maxActions) { AppendToMonitoringLog("Max number of actions configured for this session have executed so terminating shortly!", true); Thread.Sleep(5000); return(true); } } else { AppendToMonitoringLog($"CPU Percent {cpuPercent.ToString("0.00")} % > [{ cpuThreshold } %] for {currentCpuConsumptionWithTime} seconds for { ProcessList[id].Name} ({ id }), waiting to reach threshold of {seconds} seconds", true); } } else { if (ProcessList[id].ThresholdExeededCount > 0) { AppendToMonitoringLog("Resetting CPU threshold", true); ProcessList[id].ThresholdExeededCount = 0; } } } else { Trace.TraceInformation($"VERBOSE: processCpuTime == TimeSpan.MinValue "); } } AppendToMonitoringLog(string.Join(", ", processesMonitored)); RemoveOldProcessesFromMonitoringList(ProcessList); return(false); }