示例#1
0
        public static string GetLogsFolderForSession(string sessionId)
        {
            string logsFolderPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Logs);
            string folderName     = Path.Combine(logsFolderPath, sessionId);

            FileSystemHelpers.CreateDirectoryIfNotExists(folderName);
            return(folderName);
        }
示例#2
0
        public static string GetAnalysisFolderPath(out bool errorEncountered)
        {
            errorEncountered = false;
            string path = Path.Combine(Settings.UserSiteStorageDirectory, MonitoringSessionController.GetCpuMonitoringPath(), MonitoringSessionDirectories.Analysis);

            try
            {
                FileSystemHelpers.CreateDirectoryIfNotExists(path);
            }
            catch (Exception ex)
            {
                errorEncountered = true;
                Logger.LogCpuMonitoringErrorEvent("Failure in GetAnalysisFolderPath", ex, string.Empty);
            }
            return(path);
        }
示例#3
0
        private static string CacheFileInTempDirectory(AnalysisRequest request)
        {
            string outputPath = MonitoringSessionController.TempFilePath;

            FileSystemHelpers.CreateDirectoryIfNotExists(outputPath);

            string dumpFileInTempDirectory = Path.Combine(outputPath, Path.GetFileName(request.LogFileName));

            Logger.LogCpuMonitoringVerboseEvent($"Caching file {request.LogFileName} to {dumpFileInTempDirectory}", request.SessionId);

            if (!FileSystemHelpers.FileExists(dumpFileInTempDirectory))
            {
                Logger.LogCpuMonitoringVerboseEvent($"File {dumpFileInTempDirectory} does not exist. Copying it locally", request.SessionId);
                if (!string.IsNullOrWhiteSpace(request.BlobSasUri))
                {
                    try
                    {
                        string filePath = Path.Combine("Monitoring", "Logs", request.SessionId, Path.GetFileName(request.LogFileName));
                        var    blob     = BlobController.GetBlobForFile(filePath, request.BlobSasUri);
                        var    ms       = new MemoryStream();
                        blob.DownloadToStream(ms);
                        ms.Position = 0;
                        using (FileStream file = new FileStream(dumpFileInTempDirectory, FileMode.Create, FileAccess.Write))
                        {
                            byte[] bytes = new byte[ms.Length];
                            ms.Read(bytes, 0, (int)ms.Length);
                            file.Write(bytes, 0, bytes.Length);
                            ms.Close();
                        }
                        Logger.LogCpuMonitoringVerboseEvent($"Copied file from {request.LogFileName} to {dumpFileInTempDirectory} ", request.SessionId);
                    }
                    catch (Exception ex)
                    {
                        Logger.LogCpuMonitoringErrorEvent("Failed while copying file from Blob", ex, request.SessionId);
                    }
                }
                else
                {
                    FileSystemHelpers.CopyFile(request.LogFileName, dumpFileInTempDirectory);
                    Logger.LogCpuMonitoringVerboseEvent($"Copied file from {request.LogFileName} to {dumpFileInTempDirectory} ", request.SessionId);
                }
            }
            return(dumpFileInTempDirectory);
        }
示例#4
0
        public bool MonitorCpu(MonitoringSession session)
        {
            int    cpuThreshold        = session.CpuThreshold;
            int    seconds             = session.ThresholdSeconds;
            int    monitorDuration     = session.MonitorDuration;
            string actionToExecute     = session.ActionToExecute;
            string argumentsToAction   = session.ArgumentsToAction;
            int    maxActions          = session.MaxActions == 0 ? int.MaxValue : session.MaxActions;
            bool   monitorScmProcesses = session.MonitorScmProcesses;
            string blobSasUri          = session.BlobSasUri;

            if (string.IsNullOrWhiteSpace(actionToExecute))
            {
                actionToExecute = @"D:\devtools\sysinternals\procdump.exe";
            }
            if (string.IsNullOrWhiteSpace(argumentsToAction))
            {
                argumentsToAction = " -accepteula -ma {PROCESSID} {OUTPUTPATH}";
            }

            foreach (var process in Process.GetProcesses())
            {
                if (_processesToMonitor.Count > 0)
                {
                    if (!_processesToMonitor.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase)))
                    {
                        continue;
                    }
                }

                if (!monitorScmProcesses)
                {
                    var envVar = Utilities.GetEnvironmentVariablesCore(process.Handle);
                    if (Utilities.GetIsScmSite(envVar))
                    {
                        continue;
                    }
                }

                if (_processesAlwaysExcluded.Any(s => s.Equals(process.ProcessName, StringComparison.OrdinalIgnoreCase)))
                {
                    continue;
                }

                int id = process.Id;
                if (!ProcessList.ContainsKey(id))
                {
                    var cpuTime        = GetProcessCpuTime(id);
                    MonitoredProcess p = new MonitoredProcess
                    {
                        CPUTimeStart          = cpuTime,
                        Name                  = Process.GetProcessById(id).ProcessName,
                        CPUTimeCurrent        = cpuTime,
                        LastMonitorTime       = DateTime.UtcNow,
                        ThresholdExeededCount = 0
                    };
                    bool booProcessAdded = ProcessList.TryAdd(id, p);
                    if (booProcessAdded)
                    {
                        AppendToMonitoringLog($"Added process {process.ProcessName}({id}) to monitoring", true);
                    }
                }
            }

            var processesMonitored = new List <string>();

            foreach (var id in ProcessList.Keys)
            {
                var start          = ProcessList[id].CPUTimeStart;
                var oldCPUTime     = ProcessList[id].CPUTimeCurrent;
                var processCpuTime = GetProcessCpuTime(id);

                if (processCpuTime != TimeSpan.MinValue)
                {
                    TimeSpan newCPUTime      = processCpuTime - start;
                    var      cpuTimeSeconds  = (newCPUTime - oldCPUTime).TotalSeconds;
                    var      durationSeconds = DateTime.UtcNow.Subtract(ProcessList[id].LastMonitorTime).TotalSeconds;

                    // for the first time CPU Time will be
                    // negative as startTime is not subtracted
                    if (cpuTimeSeconds < 0)
                    {
                        CpuUsageLastMinute = 0;
                    }
                    else
                    {
                        CpuUsageLastMinute = cpuTimeSeconds / (Environment.ProcessorCount * durationSeconds);
                    }

                    ProcessList[id].CPUTimeCurrent  = newCPUTime;
                    ProcessList[id].LastMonitorTime = DateTime.UtcNow;;
                    var cpuPercent = CpuUsageLastMinute * 100;

                    processesMonitored.Add($"{ProcessList[id].Name}({id}):{cpuPercent.ToString("0")} %");


                    var actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);

                    bool terminateMonitoring = false;
                    if (actionsExecuted >= maxActions)
                    {
                        AppendToMonitoringLog("Maximum number of actions configured on all instances have executed so terminating shortly!", true);
                        terminateMonitoring = true;
                    }

                    if (DateTime.UtcNow.Subtract(session.StartDate).TotalHours >= session.MaximumNumberOfHours)
                    {
                        AppendToMonitoringLog("Maximum time limit for this session has reached so terminating shortly!", true);
                        terminateMonitoring = true;
                    }

                    if (terminateMonitoring)
                    {
                        var dumpsCollected = GetTotalCustomActionsCompleted(session.SessionId);
                        int sleepCount     = 0;
                        AppendToMonitoringLog("Waiting for all instances to collect and move the dumps", true);
                        while (dumpsCollected < actionsExecuted && sleepCount < 20)
                        {
                            AppendToMonitoringLog($"Total actions executed = {actionsExecuted} and dumps moved = {dumpsCollected}, waiting for the rest to finish", true);
                            Thread.Sleep(15000);
                            ++sleepCount;
                            actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
                            dumpsCollected  = GetTotalCustomActionsCompleted(session.SessionId);
                        }
                        AppendToMonitoringLog("All instances finsihed collecting data so terminating", true);
                        return(true);
                    }

                    if (cpuPercent >= cpuThreshold)
                    {
                        AppendToMonitoringLog($"{ProcessList[id].Name}({id}) CPU:{cpuPercent.ToString("0.00")} %");
                        int thresholdCount = ++ProcessList[id].ThresholdExeededCount;
                        int currentCpuConsumptionWithTime = thresholdCount * monitorDuration;
                        if (currentCpuConsumptionWithTime >= seconds)
                        {
                            actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);

                            string fileName         = Environment.MachineName + "_" + ProcessList[id].Name + "_" + id + "_" + DateTime.Now.Ticks.ToString();
                            string customActionFile = fileName + ".customaction";

                            string outputPath = MonitoringSessionController.TempFilePath;
                            FileSystemHelpers.CreateDirectoryIfNotExists(outputPath);
                            string dumpFileInTempDirectory = Path.Combine(outputPath, fileName + ".dmp");

                            if (session.Mode != SessionMode.Kill)
                            {
                                AppendToMonitoringLog($"Actions Executed on all instances = {actionsExecuted} of {maxActions}", true);
                            }

                            if (ShouldCollectData(session.Mode))
                            {
                                CreateCustomActionFile(session.SessionId, customActionFile);
                                ExecuteAction(dumpFileInTempDirectory, id, actionToExecute, argumentsToAction);
                            }

                            if (ShouldKillProcess(session.Mode))
                            {
                                KillProcessConsumingCpu(id, session.SessionId);
                            }

                            if (ShouldCollectData(session.Mode))
                            {
                                MoveToPermanentStorage(session.SessionId, dumpFileInTempDirectory, fileName + ".dmp", blobSasUri);
                                customActionFile = fileName + ".customactioncompleted";
                                CreateCustomActionFile(session.SessionId, customActionFile);


                                // Since we copied the file to permanent storage, delete the time file if the mode
                                // doesnt require analysis to be done or if the number of instances is more than 1
                                if (session.Mode != SessionMode.CollectKillAndAnalyze || HeartBeats.HeartBeatController.GetNumberOfLiveInstances() > 1)
                                {
                                    FileSystemHelpers.DeleteFileSafe(dumpFileInTempDirectory);
                                }
                            }

                            actionsExecuted = GetTotalCustomActionsExecuted(session.SessionId);
                            ProcessList[id].ThresholdExeededCount = 0;
                            if (actionsExecuted >= maxActions)
                            {
                                AppendToMonitoringLog("Max number of actions configured for this session have executed so terminating shortly!", true);
                                Thread.Sleep(5000);
                                return(true);
                            }
                        }
                        else
                        {
                            AppendToMonitoringLog($"CPU Percent {cpuPercent.ToString("0.00")} % > [{ cpuThreshold } %] for {currentCpuConsumptionWithTime} seconds for { ProcessList[id].Name} ({ id }), waiting to reach threshold of {seconds} seconds", true);
                        }
                    }
                    else
                    {
                        if (ProcessList[id].ThresholdExeededCount > 0)
                        {
                            AppendToMonitoringLog("Resetting CPU threshold", true);
                            ProcessList[id].ThresholdExeededCount = 0;
                        }
                    }
                }
                else
                {
                    Trace.TraceInformation($"VERBOSE: processCpuTime == TimeSpan.MinValue ");
                }
            }

            AppendToMonitoringLog(string.Join(", ", processesMonitored));
            RemoveOldProcessesFromMonitoringList(ProcessList);

            return(false);
        }