public MonitoringSession CreateSession(MonitoringSession monitoringSession)
{
string cpuMonitoringActive = GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitoringActive, "*.json", false, SearchOption.TopDirectoryOnly);
if (existingFiles.Count > 0)
{
throw new ApplicationException("Another monitoring session is already in progress");
}
else
{
ValidateSessionParameters(monitoringSession);
FileSystemHelpers.DeleteDirectoryContentsSafe(cpuMonitoringActive);
monitoringSession.StartDate = DateTime.UtcNow;
monitoringSession.EndDate = DateTime.MinValue.ToUniversalTime();
monitoringSession.SessionId = monitoringSession.StartDate.ToString(SessionConstants.SessionFileNameFormat);
cpuMonitoringActive = Path.Combine(cpuMonitoringActive, monitoringSession.SessionId + ".json");
monitoringSession.ToJsonFile(cpuMonitoringActive);
Logger.LogNewCpuMonitoringSession(monitoringSession);
}
return(monitoringSession);
}
private bool CheckAndWaitTillFileSystemWritable(string sessionId)
{
int maxWaitCount = 6, waitCount = 0;
bool isFileSystemReadOnly = FileSystemHelpers.IsFileSystemReadOnly();
if (isFileSystemReadOnly)
{
Logger.LogCpuMonitoringVerboseEvent("Waiting till filesystem is readonly", sessionId);
while (isFileSystemReadOnly && (waitCount <= maxWaitCount))
{
isFileSystemReadOnly = FileSystemHelpers.IsFileSystemReadOnly();
Thread.Sleep(10 * 1000);
++waitCount;
}
if (waitCount >= maxWaitCount)
{
Logger.LogCpuMonitoringVerboseEvent("FileSystem is still readonly so exiting...", sessionId);
return(false);
}
Logger.LogCpuMonitoringVerboseEvent("FileSystem is no more readonly", sessionId);
}
return(true);
}
public static void ToJsonFile(this object objectToSerialize, string filePath)
{
FileSystemHelpers.WriteFile(filePath, JsonConvert.SerializeObject(objectToSerialize, Formatting.Indented));
}
public bool MonitorCpu(MonitoringSession session)
{
int cpuThreshold = session.CpuThreshold;
int seconds = session.ThresholdSeconds;
int monitorDuration = session.MonitorDuration;
string actionToExecute = session.ActionToExecute;
string argumentsToAction = session.ArgumentsToAction;
int maxActions = session.MaxActions == 0 ? int.MaxValue : session.MaxActions;
bool monitorScmProcesses = session.MonitorScmProcesses;
string blobSasUri = session.BlobSasUri;
if (string.IsNullOrWhiteSpace(actionToExecute))
{
actionToExecute = @"D:\devtools\sysinternals\procdump.exe";
}
if (string.IsNullOrWhiteSpace(argumentsToAction))
{
argumentsToAction = " -accepteula -ma {PROCESSID} {OUTPUTPATH}";
}
foreach (var process in Process.GetProcesses())
{
if (_processesToMonitor.Count > 0)
{
if (!_processesToMonitor.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
}
if (!monitorScmProcesses)
{
var envVar = Utilities.GetEnvironmentVariablesCore(process.Handle);
if (Utilities.GetIsScmSite(envVar))
{
continue;
}
}
if (_processesAlwaysExcluded.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
int id = process.Id;
if (!ProcessList.ContainsKey(id))
{
var cpuTime = GetProcessCpuTime(id);
MonitoredProcess p = new MonitoredProcess
{
CPUTimeStart = cpuTime,
Name = Process.GetProcessById(id).ProcessName,
CPUTimeCurrent = cpuTime,
LastMonitorTime = DateTime.UtcNow,
ThresholdExeededCount = 0
};
bool booProcessAdded = ProcessList.TryAdd(id, p);
if (booProcessAdded)
{
AppendToMonitoringLog($"Added process {process.ProcessName}({id}) to monitoring", true);
}
}
}
var processesMonitored = new List <string>();
foreach (var id in ProcessList.Keys)
{
var start = ProcessList[id].CPUTimeStart;
var oldCPUTime = ProcessList[id].CPUTimeCurrent;
var processCpuTime = GetProcessCpuTime(id);
if (processCpuTime != TimeSpan.MinValue)
{
TimeSpan newCPUTime = processCpuTime - start;
var cpuTimeSeconds = (newCPUTime - oldCPUTime).TotalSeconds;
var durationSeconds = DateTime.UtcNow.Subtract(ProcessList[id].LastMonitorTime).TotalSeconds;
// for the first time CPU Time will be
// negative as startTime is not subtracted
if (cpuTimeSeconds < 0)
{
CpuUsageLastMinute = 0;
}
else
{
CpuUsageLastMinute = cpuTimeSeconds / (Environment.ProcessorCount * durationSeconds);
}
ProcessList[id].CPUTimeCurrent = newCPUTime;
ProcessList[id].LastMonitorTime = DateTime.UtcNow;;
var cpuPercent = CpuUsageLastMinute * 100;
processesMonitored.Add($"{ProcessList[id].Name}({id}):{cpuPercent.ToString("0")} %");
var actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
bool terminateMonitoring = false;
if (actionsExecuted >= maxActions)
{
AppendToMonitoringLog("Maximum number of actions configured on all instances have executed so terminating shortly!", true);
terminateMonitoring = true;
}
if (DateTime.UtcNow.Subtract(session.StartDate).TotalHours >= session.MaximumNumberOfHours)
{
AppendToMonitoringLog("Maximum time limit for this session has reached so terminating shortly!", true);
terminateMonitoring = true;
}
if (terminateMonitoring)
{
var dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId);
int sleepCount = 0;
AppendToMonitoringLog("Waiting for all instances to collect and move the dumps", true);
while (dumpsCollected < actionsExecuted && sleepCount < 20)
{
AppendToMonitoringLog($"Total actions executed = {actionsExecuted} and dumps moved = {dumpsCollected}, waiting for the rest to finish", true);
Thread.Sleep(15000);
++sleepCount;
actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId);
}
AppendToMonitoringLog("All instances finsihed collecting data so terminating", true);
return(true);
}
if (cpuPercent >= cpuThreshold)
{
AppendToMonitoringLog($"{ProcessList[id].Name}({id}) CPU:{cpuPercent.ToString("0.00")} %");
int thresholdCount = ++ProcessList[id].ThresholdExeededCount;
int currentCpuConsumptionWithTime = thresholdCount * monitorDuration;
if (currentCpuConsumptionWithTime >= seconds)
{
actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
string fileName = Environment.MachineName + "_" + ProcessList[id].Name + "_" + id + "_" + DateTime.Now.Ticks.ToString();
string customActionFile = fileName + ".customaction";
string outputPath = MonitoringSessionController.TempFilePath;
FileSystemHelpers.CreateDirectoryIfNotExists(outputPath);
string dumpFileInTempDirectory = Path.Combine(outputPath, fileName + ".dmp");
if (session.Mode != SessionMode.Kill)
{
AppendToMonitoringLog($"Actions Executed on all instances = {actionsExecuted} of {maxActions}", true);
}
if (ShouldCollectData(session.Mode))
{
CreateCustomActionFile(session.SessionId, customActionFile);
ExecuteAction(dumpFileInTempDirectory, id, actionToExecute, argumentsToAction);
}
if (ShouldKillProcess(session.Mode))
{
KillProcessConsumingCpu(id, session.SessionId);
}
if (ShouldCollectData(session.Mode))
{
MoveToPermanentStorage(session.SessionId, dumpFileInTempDirectory, fileName + ".dmp", blobSasUri);
customActionFile = fileName + ".customactioncompleted";
CreateCustomActionFile(session.SessionId, customActionFile);
// Since we copied the file to permanent storage, delete the time file if the mode
// doesnt require analysis to be done or if the number of instances is more than 1
if (session.Mode != SessionMode.CollectKillAndAnalyze || HeartBeats.HeartBeatController.GetNumberOfLiveInstances() > 1)
{
FileSystemHelpers.DeleteFileSafe(dumpFileInTempDirectory);
}
}
actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
ProcessList[id].ThresholdExeededCount = 0;
if (actionsExecuted >= maxActions)
{
AppendToMonitoringLog("Max number of actions configured for this session have executed so terminating shortly!", true);
Thread.Sleep(5000);
return(true);
}
}
else
{
AppendToMonitoringLog($"CPU Percent {cpuPercent.ToString("0.00")} % > [{ cpuThreshold } %] for {currentCpuConsumptionWithTime} seconds for { ProcessList[id].Name} ({ id }), waiting to reach threshold of {seconds} seconds", true);
}
}
else
{
if (ProcessList[id].ThresholdExeededCount > 0)
{
AppendToMonitoringLog("Resetting CPU threshold", true);
ProcessList[id].ThresholdExeededCount = 0;
}
}
}
else
{
Trace.TraceInformation($"VERBOSE: processCpuTime == TimeSpan.MinValue ");
}
}
AppendToMonitoringLog(string.Join(", ", processesMonitored));
RemoveOldProcessesFromMonitoringList(ProcessList);
return(false);
}
public static void MoveFile(string sourceFileName, string destFileName)
{
FileSystemHelpers.DeleteFileSafe(destFileName);
Instance.File.Move(sourceFileName, destFileName);
}
