public static string GetLogsFolderForSession(string sessionId) { string logsFolderPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Logs); string folderName = Path.Combine(logsFolderPath, sessionId); FileSystemHelpers.CreateDirectoryIfNotExists(folderName); return(folderName); }
public static string GetAnalysisFolderPath(out bool errorEncountered) { errorEncountered = false; string path = Path.Combine(Settings.UserSiteStorageDirectory, MonitoringSessionController.GetCpuMonitoringPath(), MonitoringSessionDirectories.Analysis); try { FileSystemHelpers.CreateDirectoryIfNotExists(path); } catch (Exception ex) { errorEncountered = true; Logger.LogCpuMonitoringErrorEvent("Failure in GetAnalysisFolderPath", ex, string.Empty); } return(path); }
private static string CacheFileInTempDirectory(AnalysisRequest request) { string outputPath = MonitoringSessionController.TempFilePath; FileSystemHelpers.CreateDirectoryIfNotExists(outputPath); string dumpFileInTempDirectory = Path.Combine(outputPath, Path.GetFileName(request.LogFileName)); Logger.LogCpuMonitoringVerboseEvent($"Caching file {request.LogFileName} to {dumpFileInTempDirectory}", request.SessionId); if (!FileSystemHelpers.FileExists(dumpFileInTempDirectory)) { Logger.LogCpuMonitoringVerboseEvent($"File {dumpFileInTempDirectory} does not exist. Copying it locally", request.SessionId); if (!string.IsNullOrWhiteSpace(request.BlobSasUri)) { try { string filePath = Path.Combine("Monitoring", "Logs", request.SessionId, Path.GetFileName(request.LogFileName)); var blob = BlobController.GetBlobForFile(filePath, request.BlobSasUri); var ms = new MemoryStream(); blob.DownloadToStream(ms); ms.Position = 0; using (FileStream file = new FileStream(dumpFileInTempDirectory, FileMode.Create, FileAccess.Write)) { byte[] bytes = new byte[ms.Length]; ms.Read(bytes, 0, (int)ms.Length); file.Write(bytes, 0, bytes.Length); ms.Close(); } Logger.LogCpuMonitoringVerboseEvent($"Copied file from {request.LogFileName} to {dumpFileInTempDirectory} ", request.SessionId); } catch (Exception ex) { Logger.LogCpuMonitoringErrorEvent("Failed while copying file from Blob", ex, request.SessionId); } } else { FileSystemHelpers.CopyFile(request.LogFileName, dumpFileInTempDirectory); Logger.LogCpuMonitoringVerboseEvent($"Copied file from {request.LogFileName} to {dumpFileInTempDirectory} ", request.SessionId); } } return(dumpFileInTempDirectory); }
public bool MonitorCpu(MonitoringSession session) { int cpuThreshold = session.CpuThreshold; int seconds = session.ThresholdSeconds; int monitorDuration = session.MonitorDuration; string actionToExecute = session.ActionToExecute; string argumentsToAction = session.ArgumentsToAction; int maxActions = session.MaxActions == 0 ? int.MaxValue : session.MaxActions; bool monitorScmProcesses = session.MonitorScmProcesses; string blobSasUri = session.BlobSasUri; if (string.IsNullOrWhiteSpace(actionToExecute)) { actionToExecute = @"D:\devtools\sysinternals\procdump.exe"; } if (string.IsNullOrWhiteSpace(argumentsToAction)) { argumentsToAction = " -accepteula -ma {PROCESSID} {OUTPUTPATH}"; } foreach (var process in Process.GetProcesses()) { if (_processesToMonitor.Count > 0) { if (!_processesToMonitor.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase))) { continue; } } if (!monitorScmProcesses) { var envVar = Utilities.GetEnvironmentVariablesCore(process.Handle); if (Utilities.GetIsScmSite(envVar)) { continue; } } if (_processesAlwaysExcluded.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase))) { continue; } int id = process.Id; if (!ProcessList.ContainsKey(id)) { var cpuTime = GetProcessCpuTime(id); MonitoredProcess p = new MonitoredProcess { CPUTimeStart = cpuTime, Name = Process.GetProcessById(id).ProcessName, CPUTimeCurrent = cpuTime, LastMonitorTime = DateTime.UtcNow, ThresholdExeededCount = 0 }; bool booProcessAdded = ProcessList.TryAdd(id, p); if (booProcessAdded) { AppendToMonitoringLog($"Added process {process.ProcessName}({id}) to monitoring", true); } } } var processesMonitored = new List <string>(); foreach (var id in ProcessList.Keys) { var start = ProcessList[id].CPUTimeStart; var oldCPUTime = ProcessList[id].CPUTimeCurrent; var processCpuTime = GetProcessCpuTime(id); if (processCpuTime != TimeSpan.MinValue) { TimeSpan newCPUTime = processCpuTime - start; var cpuTimeSeconds = (newCPUTime - oldCPUTime).TotalSeconds; var durationSeconds = DateTime.UtcNow.Subtract(ProcessList[id].LastMonitorTime).TotalSeconds; // for the first time CPU Time will be // negative as startTime is not subtracted if (cpuTimeSeconds < 0) { CpuUsageLastMinute = 0; } else { CpuUsageLastMinute = cpuTimeSeconds / (Environment.ProcessorCount * durationSeconds); } ProcessList[id].CPUTimeCurrent = newCPUTime; ProcessList[id].LastMonitorTime = DateTime.UtcNow;; var cpuPercent = CpuUsageLastMinute * 100; processesMonitored.Add($"{ProcessList[id].Name}({id}):{cpuPercent.ToString("0")} %"); var actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId); bool terminateMonitoring = false; if (actionsExecuted >= maxActions) { AppendToMonitoringLog("Maximum number of actions configured on all instances have executed so terminating shortly!", true); terminateMonitoring = true; } if (DateTime.UtcNow.Subtract(session.StartDate).TotalHours >= session.MaximumNumberOfHours) { AppendToMonitoringLog("Maximum time limit for this session has reached so terminating shortly!", true); terminateMonitoring = true; } if (terminateMonitoring) { var dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId); int sleepCount = 0; AppendToMonitoringLog("Waiting for all instances to collect and move the dumps", true); while (dumpsCollected < actionsExecuted && sleepCount < 20) { AppendToMonitoringLog($"Total actions executed = {actionsExecuted} and dumps moved = {dumpsCollected}, waiting for the rest to finish", true); Thread.Sleep(15000); ++sleepCount; actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId); dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId); } AppendToMonitoringLog("All instances finsihed collecting data so terminating", true); return(true); } if (cpuPercent >= cpuThreshold) { AppendToMonitoringLog($"{ProcessList[id].Name}({id}) CPU:{cpuPercent.ToString("0.00")} %"); int thresholdCount = ++ProcessList[id].ThresholdExeededCount; int currentCpuConsumptionWithTime = thresholdCount * monitorDuration; if (currentCpuConsumptionWithTime >= seconds) { actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId); string fileName = Environment.MachineName + "_" + ProcessList[id].Name + "_" + id + "_" + DateTime.Now.Ticks.ToString(); string customActionFile = fileName + ".customaction"; string outputPath = MonitoringSessionController.TempFilePath; FileSystemHelpers.CreateDirectoryIfNotExists(outputPath); string dumpFileInTempDirectory = Path.Combine(outputPath, fileName + ".dmp"); if (session.Mode != SessionMode.Kill) { AppendToMonitoringLog($"Actions Executed on all instances = {actionsExecuted} of {maxActions}", true); } if (ShouldCollectData(session.Mode)) { CreateCustomActionFile(session.SessionId, customActionFile); ExecuteAction(dumpFileInTempDirectory, id, actionToExecute, argumentsToAction); } if (ShouldKillProcess(session.Mode)) { KillProcessConsumingCpu(id, session.SessionId); } if (ShouldCollectData(session.Mode)) { MoveToPermanentStorage(session.SessionId, dumpFileInTempDirectory, fileName + ".dmp", blobSasUri); customActionFile = fileName + ".customactioncompleted"; CreateCustomActionFile(session.SessionId, customActionFile); // Since we copied the file to permanent storage, delete the time file if the mode // doesnt require analysis to be done or if the number of instances is more than 1 if (session.Mode != SessionMode.CollectKillAndAnalyze || HeartBeats.HeartBeatController.GetNumberOfLiveInstances() > 1) { FileSystemHelpers.DeleteFileSafe(dumpFileInTempDirectory); } } actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId); ProcessList[id].ThresholdExeededCount = 0; if (actionsExecuted >= maxActions) { AppendToMonitoringLog("Max number of actions configured for this session have executed so terminating shortly!", true); Thread.Sleep(5000); return(true); } } else { AppendToMonitoringLog($"CPU Percent {cpuPercent.ToString("0.00")} % > [{ cpuThreshold } %] for {currentCpuConsumptionWithTime} seconds for { ProcessList[id].Name} ({ id }), waiting to reach threshold of {seconds} seconds", true); } } else { if (ProcessList[id].ThresholdExeededCount > 0) { AppendToMonitoringLog("Resetting CPU threshold", true); ProcessList[id].ThresholdExeededCount = 0; } } } else { Trace.TraceInformation($"VERBOSE: processCpuTime == TimeSpan.MinValue "); } } AppendToMonitoringLog(string.Join(", ", processesMonitored)); RemoveOldProcessesFromMonitoringList(ProcessList); return(false); }