public static string GetLogsFolderForSession(string sessionId)
{
string logsFolderPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Logs);
string folderName = Path.Combine(logsFolderPath, sessionId);
FileSystemHelpers.CreateDirectoryIfNotExists(folderName);
return(folderName);
}
public static string GetAnalysisFolderPath(out bool errorEncountered)
{
errorEncountered = false;
string path = Path.Combine(Settings.UserSiteStorageDirectory, MonitoringSessionController.GetCpuMonitoringPath(), MonitoringSessionDirectories.Analysis);
try
{
FileSystemHelpers.CreateDirectoryIfNotExists(path);
}
catch (Exception ex)
{
errorEncountered = true;
Logger.LogCpuMonitoringErrorEvent("Failure in GetAnalysisFolderPath", ex, string.Empty);
}
return(path);
}
private static string CacheFileInTempDirectory(AnalysisRequest request)
{
string outputPath = MonitoringSessionController.TempFilePath;
FileSystemHelpers.CreateDirectoryIfNotExists(outputPath);
string dumpFileInTempDirectory = Path.Combine(outputPath, Path.GetFileName(request.LogFileName));
Logger.LogCpuMonitoringVerboseEvent($"Caching file {request.LogFileName} to {dumpFileInTempDirectory}", request.SessionId);
if (!FileSystemHelpers.FileExists(dumpFileInTempDirectory))
{
Logger.LogCpuMonitoringVerboseEvent($"File {dumpFileInTempDirectory} does not exist. Copying it locally", request.SessionId);
if (!string.IsNullOrWhiteSpace(request.BlobSasUri))
{
try
{
string filePath = Path.Combine("Monitoring", "Logs", request.SessionId, Path.GetFileName(request.LogFileName));
var blob = BlobController.GetBlobForFile(filePath, request.BlobSasUri);
var ms = new MemoryStream();
blob.DownloadToStream(ms);
ms.Position = 0;
using (FileStream file = new FileStream(dumpFileInTempDirectory, FileMode.Create, FileAccess.Write))
{
byte[] bytes = new byte[ms.Length];
ms.Read(bytes, 0, (int)ms.Length);
file.Write(bytes, 0, bytes.Length);
ms.Close();
}
Logger.LogCpuMonitoringVerboseEvent($"Copied file from {request.LogFileName} to {dumpFileInTempDirectory} ", request.SessionId);
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failed while copying file from Blob", ex, request.SessionId);
}
}
else
{
FileSystemHelpers.CopyFile(request.LogFileName, dumpFileInTempDirectory);
Logger.LogCpuMonitoringVerboseEvent($"Copied file from {request.LogFileName} to {dumpFileInTempDirectory} ", request.SessionId);
}
}
return(dumpFileInTempDirectory);
}
public bool MonitorCpu(MonitoringSession session)
{
int cpuThreshold = session.CpuThreshold;
int seconds = session.ThresholdSeconds;
int monitorDuration = session.MonitorDuration;
string actionToExecute = session.ActionToExecute;
string argumentsToAction = session.ArgumentsToAction;
int maxActions = session.MaxActions == 0 ? int.MaxValue : session.MaxActions;
bool monitorScmProcesses = session.MonitorScmProcesses;
string blobSasUri = session.BlobSasUri;
if (string.IsNullOrWhiteSpace(actionToExecute))
{
actionToExecute = @"D:\devtools\sysinternals\procdump.exe";
}
if (string.IsNullOrWhiteSpace(argumentsToAction))
{
argumentsToAction = " -accepteula -ma {PROCESSID} {OUTPUTPATH}";
}
foreach (var process in Process.GetProcesses())
{
if (_processesToMonitor.Count > 0)
{
if (!_processesToMonitor.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
}
if (!monitorScmProcesses)
{
var envVar = Utilities.GetEnvironmentVariablesCore(process.Handle);
if (Utilities.GetIsScmSite(envVar))
{
continue;
}
}
if (_processesAlwaysExcluded.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
int id = process.Id;
if (!ProcessList.ContainsKey(id))
{
var cpuTime = GetProcessCpuTime(id);
MonitoredProcess p = new MonitoredProcess
{
CPUTimeStart = cpuTime,
Name = Process.GetProcessById(id).ProcessName,
CPUTimeCurrent = cpuTime,
LastMonitorTime = DateTime.UtcNow,
ThresholdExeededCount = 0
};
bool booProcessAdded = ProcessList.TryAdd(id, p);
if (booProcessAdded)
{
AppendToMonitoringLog($"Added process {process.ProcessName}({id}) to monitoring", true);
}
}
}
var processesMonitored = new List <string>();
foreach (var id in ProcessList.Keys)
{
var start = ProcessList[id].CPUTimeStart;
var oldCPUTime = ProcessList[id].CPUTimeCurrent;
var processCpuTime = GetProcessCpuTime(id);
if (processCpuTime != TimeSpan.MinValue)
{
TimeSpan newCPUTime = processCpuTime - start;
var cpuTimeSeconds = (newCPUTime - oldCPUTime).TotalSeconds;
var durationSeconds = DateTime.UtcNow.Subtract(ProcessList[id].LastMonitorTime).TotalSeconds;
// for the first time CPU Time will be
// negative as startTime is not subtracted
if (cpuTimeSeconds < 0)
{
CpuUsageLastMinute = 0;
}
else
{
CpuUsageLastMinute = cpuTimeSeconds / (Environment.ProcessorCount * durationSeconds);
}
ProcessList[id].CPUTimeCurrent = newCPUTime;
ProcessList[id].LastMonitorTime = DateTime.UtcNow;;
var cpuPercent = CpuUsageLastMinute * 100;
processesMonitored.Add($"{ProcessList[id].Name}({id}):{cpuPercent.ToString("0")} %");
var actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
bool terminateMonitoring = false;
if (actionsExecuted >= maxActions)
{
AppendToMonitoringLog("Maximum number of actions configured on all instances have executed so terminating shortly!", true);
terminateMonitoring = true;
}
if (DateTime.UtcNow.Subtract(session.StartDate).TotalHours >= session.MaximumNumberOfHours)
{
AppendToMonitoringLog("Maximum time limit for this session has reached so terminating shortly!", true);
terminateMonitoring = true;
}
if (terminateMonitoring)
{
var dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId);
int sleepCount = 0;
AppendToMonitoringLog("Waiting for all instances to collect and move the dumps", true);
while (dumpsCollected < actionsExecuted && sleepCount < 20)
{
AppendToMonitoringLog($"Total actions executed = {actionsExecuted} and dumps moved = {dumpsCollected}, waiting for the rest to finish", true);
Thread.Sleep(15000);
++sleepCount;
actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId);
}
AppendToMonitoringLog("All instances finsihed collecting data so terminating", true);
return(true);
}
if (cpuPercent >= cpuThreshold)
{
AppendToMonitoringLog($"{ProcessList[id].Name}({id}) CPU:{cpuPercent.ToString("0.00")} %");
int thresholdCount = ++ProcessList[id].ThresholdExeededCount;
int currentCpuConsumptionWithTime = thresholdCount * monitorDuration;
if (currentCpuConsumptionWithTime >= seconds)
{
actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
string fileName = Environment.MachineName + "_" + ProcessList[id].Name + "_" + id + "_" + DateTime.Now.Ticks.ToString();
string customActionFile = fileName + ".customaction";
string outputPath = MonitoringSessionController.TempFilePath;
FileSystemHelpers.CreateDirectoryIfNotExists(outputPath);
string dumpFileInTempDirectory = Path.Combine(outputPath, fileName + ".dmp");
if (session.Mode != SessionMode.Kill)
{
AppendToMonitoringLog($"Actions Executed on all instances = {actionsExecuted} of {maxActions}", true);
}
if (ShouldCollectData(session.Mode))
{
CreateCustomActionFile(session.SessionId, customActionFile);
ExecuteAction(dumpFileInTempDirectory, id, actionToExecute, argumentsToAction);
}
if (ShouldKillProcess(session.Mode))
{
KillProcessConsumingCpu(id, session.SessionId);
}
if (ShouldCollectData(session.Mode))
{
MoveToPermanentStorage(session.SessionId, dumpFileInTempDirectory, fileName + ".dmp", blobSasUri);
customActionFile = fileName + ".customactioncompleted";
CreateCustomActionFile(session.SessionId, customActionFile);
// Since we copied the file to permanent storage, delete the time file if the mode
// doesnt require analysis to be done or if the number of instances is more than 1
if (session.Mode != SessionMode.CollectKillAndAnalyze || HeartBeats.HeartBeatController.GetNumberOfLiveInstances() > 1)
{
FileSystemHelpers.DeleteFileSafe(dumpFileInTempDirectory);
}
}
actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
ProcessList[id].ThresholdExeededCount = 0;
if (actionsExecuted >= maxActions)
{
AppendToMonitoringLog("Max number of actions configured for this session have executed so terminating shortly!", true);
Thread.Sleep(5000);
return(true);
}
}
else
{
AppendToMonitoringLog($"CPU Percent {cpuPercent.ToString("0.00")} % > [{ cpuThreshold } %] for {currentCpuConsumptionWithTime} seconds for { ProcessList[id].Name} ({ id }), waiting to reach threshold of {seconds} seconds", true);
}
}
else
{
if (ProcessList[id].ThresholdExeededCount > 0)
{
AppendToMonitoringLog("Resetting CPU threshold", true);
ProcessList[id].ThresholdExeededCount = 0;
}
}
}
else
{
Trace.TraceInformation($"VERBOSE: processCpuTime == TimeSpan.MinValue ");
}
}
AppendToMonitoringLog(string.Join(", ", processesMonitored));
RemoveOldProcessesFromMonitoringList(ProcessList);
return(false);
}