public IEnumerable <MonitoringLogsPerInstance> GetActiveSessionMonitoringLogs() { var logs = new List <MonitoringLogsPerInstance>(); string cpuMonitorPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Active); var activeInstances = HeartBeats.HeartBeatController.GetLiveInstances(); if (GetActiveSession() != null) { foreach (var logFile in FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "RD*.log", false, SearchOption.TopDirectoryOnly)) { string instanceName = Path.GetFileNameWithoutExtension(logFile); if (activeInstances.Any(x => x.Name.Equals(instanceName, StringComparison.OrdinalIgnoreCase))) { string logContent = ReadEndTokens(logFile, 10, Encoding.Default, Environment.NewLine); logs.Add(new MonitoringLogsPerInstance() { Instance = instanceName, Logs = logContent }); } } } return(logs); }
public List <MonitoringFile> GetCollectedLogsForSession(string sessionId, string blobSasUri) { var filesCollected = new List <MonitoringFile>(); if (string.IsNullOrWhiteSpace(blobSasUri)) { string folderName = CpuMonitoring.GetLogsFolderForSession(sessionId); if (FileSystemHelpers.DirectoryExists(folderName)) { var logFiles = FileSystemHelpers.GetFilesInDirectory(folderName, "*.dmp", false, SearchOption.TopDirectoryOnly); foreach (var fileName in logFiles) { string relativePath = MonitoringFile.GetRelativePath(sessionId, Path.GetFileName(fileName)); filesCollected.Add(new MonitoringFile(fileName, relativePath)); } } } else { string directoryPath = Path.Combine("Monitoring", "Logs", sessionId); List <string> files = new List <string>(); var dir = BlobController.GetBlobDirectory(directoryPath, blobSasUri); foreach ( IListBlobItem item in dir.ListBlobs(useFlatBlobListing: true)) { var relativePath = item.Uri.ToString().Replace(item.Container.Uri.ToString() + "/", ""); string fileName = item.Uri.Segments.Last(); filesCollected.Add(new MonitoringFile(fileName, relativePath)); } } return(filesCollected); }
private int GetTotalCustomActionsCompleted(string sessionId) { var sessionDirectory = GetLogsFolderForSession(sessionId); var dumpCount = FileSystemHelpers.GetFilesInDirectory(sessionDirectory, "*.customactioncompleted", false, SearchOption.TopDirectoryOnly).Count; return(dumpCount); }
private void AppendToMonitoringLog(string message, bool logInKusto = false) { Interlocked.Increment(ref _loggerCount); string cpuMonitorPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Active); string logFilePath = Path.Combine(cpuMonitorPath, Environment.MachineName + ".log"); string logMessage = $"[{DateTime.UtcNow.ToShortDateString()} {DateTime.UtcNow.ToString("hh:mm:ss")}] {message}{Environment.NewLine}"; if (_loggerCount > MAX_LINES_IN_LOGFILE) { var sessionDirectory = GetLogsFolderForSession(_sessionId); var existingFileCount = FileSystemHelpers.GetFilesInDirectory(sessionDirectory, $"{Environment.MachineName}*.log", false, SearchOption.TopDirectoryOnly).Count; var newFileName = $"{Environment.MachineName}_{existingFileCount}.log"; newFileName = Path.Combine(sessionDirectory, newFileName); FileSystemHelpers.MoveFile(logFilePath, newFileName); Interlocked.Exchange(ref _loggerCount, 0); } try { FileSystemHelpers.AppendAllTextToFile(logFilePath, logMessage); } catch (Exception) { } if (logInKusto) { Logger.LogCpuMonitoringEvent(message, _sessionId); } }
public static void DequeueAnalysisRequest() { string analysisFolderPath = GetAnalysisFolderPath(out bool errorEncountered); if (!errorEncountered) { try { string requestFile = FileSystemHelpers.GetFilesInDirectory(analysisFolderPath, "*.request", false, SearchOption.TopDirectoryOnly).FirstOrDefault(); if (requestFile != null) { var analysisRequest = FileSystemHelpers.FromJsonFile <AnalysisRequest>(requestFile); var isRequestFileFromSameInstance = analysisRequest.LogFileName.StartsWith(Environment.MachineName); var exceededAnalysisTimelimit = analysisRequest.StartTime != null && DateTime.UtcNow.Subtract(analysisRequest.StartTime).TotalMinutes > MAX_ANALYSISTIME_ON_SAME_INSTANCE; Logger.LogCpuMonitoringVerboseEvent($"Checking AnalysisRequest LogFileName={analysisRequest.LogFileName}, MachineName={Environment.MachineName}, exceededAnalysisTimelimit={exceededAnalysisTimelimit}, isRequestFileFromSameInstance={isRequestFileFromSameInstance}", analysisRequest.SessionId); if (exceededAnalysisTimelimit || isRequestFileFromSameInstance) { var inprogressFile = Path.ChangeExtension(requestFile, ".inprogress"); FileSystemHelpers.MoveFile(requestFile, inprogressFile); AnalyzeKeepingExpirationAlive(inprogressFile); } } } catch (Exception ex) { Logger.LogCpuMonitoringErrorEvent("Failure in DequeueAnalysisRequest", ex, string.Empty); } } }
public static void QueueAnalysisRequest(string sessionId, string logFileName, string blobSasUri) { try { Logger.LogCpuMonitoringVerboseEvent($"Going to Queue Analysis request for {sessionId} for {logFileName}", sessionId); int retryCount = 0; retryLabel: string analysisFolderPath = GetAnalysisFolderPath(out bool errorEncountered); if (errorEncountered && retryCount < 5) { Thread.Sleep(60 * 1000); ++retryCount; Logger.LogCpuMonitoringVerboseEvent($"Encountered error while getting analysis folder path. RetryCount is {retryCount}", sessionId); goto retryLabel; } if (retryCount == 5) { throw new Exception("Storage not ready or in-accessible"); } if (!errorEncountered) { var request = new AnalysisRequest() { StartTime = DateTime.UtcNow, SessionId = sessionId, LogFileName = logFileName, ExpirationTime = DateTime.MaxValue, BlobSasUri = blobSasUri, RetryCount = 0 }; var fileName = $"{ sessionId }_{ Path.GetFileNameWithoutExtension(logFileName)}"; var requestFile = $"{fileName}.request"; requestFile = Path.Combine(analysisFolderPath, requestFile); var existingRequests = FileSystemHelpers.GetFilesInDirectory(analysisFolderPath, $"{fileName}*", false, SearchOption.TopDirectoryOnly); if (existingRequests.Count > 0) { // // There is already and existing analysis request for the same file which is // either in progress or currently submitted so no need to submit a new one // Logger.LogCpuMonitoringVerboseEvent($"An existing request for the same file exists - [{string.Join(",", existingRequests)}]", sessionId); } else { Logger.LogCpuMonitoringVerboseEvent($"Queued {requestFile} for Analysis", sessionId); request.ToJsonFile(requestFile); } } } catch (Exception ex) { Logger.LogCpuMonitoringErrorEvent("Failed while queuing analysis request", ex, sessionId); } }
public bool StopMonitoringSession() { Logger.LogCpuMonitoringVerboseEvent($"Inside the StopMonitoringSession method of MonitoringSessionController", string.Empty); string cpuMonitoringActivePath = GetCpuMonitoringPath(MonitoringSessionDirectories.Active); var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitoringActivePath, "*.json", false, SearchOption.TopDirectoryOnly); if (existingFiles.Count > 0) { var monitoringSession = FileSystemHelpers.FromJsonFile <MonitoringSession>(existingFiles.FirstOrDefault()); Logger.LogCpuMonitoringVerboseEvent($"Stopping an active session {existingFiles.FirstOrDefault()}", monitoringSession.SessionId); var canwriteToFileSystem = CheckAndWaitTillFileSystemWritable(monitoringSession.SessionId); if (!canwriteToFileSystem) { return(false); } try { monitoringSession.EndDate = DateTime.UtcNow; string cpuMonitorCompletedPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Completed); cpuMonitorCompletedPath = Path.Combine(cpuMonitorCompletedPath, monitoringSession.SessionId + ".json"); if (!FileSystemHelpers.FileExists(cpuMonitorCompletedPath)) { monitoringSession.FilesCollected = GetCollectedLogsForSession(monitoringSession.SessionId, monitoringSession.BlobSasUri); Logger.LogCpuMonitoringVerboseEvent($"Found {monitoringSession.FilesCollected.Count} files collected by CPU monitoring", monitoringSession.SessionId); SaveSession(monitoringSession); MoveMonitoringLogsToSession(monitoringSession.SessionId); } else { // some other instance probably ended up writing the file // lets hope that finishes and files get moved properly } // // Now delete the Active Session File // try { FileSystemHelpers.DeleteFileSafe(existingFiles.FirstOrDefault(), false); } catch (Exception ex) { Logger.LogCpuMonitoringErrorEvent("Failed while deleting the Active session file", ex, monitoringSession.SessionId); return(false); } } catch (Exception ex) { Logger.LogCpuMonitoringErrorEvent("Failed while marking a session as Complete", ex, monitoringSession.SessionId); return(false); } } return(true); }
public void TerminateActiveMonitoringSession() { string cpuMonitorPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Active); var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "*.json", false, SearchOption.TopDirectoryOnly); if (existingFiles.Count > 0) { FileSystemHelpers.DeleteDirectoryContentsSafe(cpuMonitorPath, true); } }
public static void CleanRemainingLogsIfAny() { string cpuMonitorPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Active); if (FileSystemHelpers.DirectoryExists(cpuMonitorPath)) { foreach (var log in FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "*.log", false, SearchOption.TopDirectoryOnly)) { FileSystemHelpers.DeleteFileSafe(log); } } }
public List <MonitoringSession> GetAllCompletedSessions() { var sessions = new List <MonitoringSession>(); string cpuMonitorPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Completed); var existingSessions = FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "*.json", false, SearchOption.TopDirectoryOnly); foreach (var session in existingSessions) { var monitoringSession = FileSystemHelpers.FromJsonFile <MonitoringSession>(session); sessions.Add(monitoringSession); } return(sessions); }
private void MoveMonitoringLogsToSession(string sessionId) { string logsFolderPath = CpuMonitoring.GetLogsFolderForSession(sessionId); string monitoringFolderActive = GetCpuMonitoringPath(MonitoringSessionDirectories.Active); var filesCollected = FileSystemHelpers.GetFilesInDirectory(monitoringFolderActive, "*.log", false, SearchOption.TopDirectoryOnly); foreach (string monitoringLog in filesCollected) { string fileName = Path.GetFileName(monitoringLog); fileName = Path.Combine(logsFolderPath, fileName); Logger.LogCpuMonitoringVerboseEvent($"Moving {monitoringLog} to {fileName}", sessionId); RetryHelper.RetryOnException("Moving monitoring log to logs folder...", () => { FileSystemHelpers.MoveFile(monitoringLog, fileName); }, TimeSpan.FromSeconds(5), 5); } }
public MonitoringSession GetActiveSession() { var sessions = new List <MonitoringSession>(); string cpuMonitorPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Active); var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "*.json", false, SearchOption.TopDirectoryOnly); if (existingFiles.Count > 0) { Logger.LogDiagnostic($"Found an active monitoring session {existingFiles.FirstOrDefault()}"); var session = FileSystemHelpers.FromJsonFile <MonitoringSession>(existingFiles.FirstOrDefault()); return(session); } else { Logger.LogDiagnostic($"Found no active monitoring session"); return(null); } }
public static void ReSubmitExpiredRequests() { string analysisFolderPath = GetAnalysisFolderPath(out bool errorEncountered); if (!errorEncountered) { var inProgressRequests = FileSystemHelpers.GetFilesInDirectory(analysisFolderPath, "*.inprogress", false, SearchOption.TopDirectoryOnly); foreach (var inprogressFile in inProgressRequests) { var analysisRequest = FileSystemHelpers.FromJsonFile <AnalysisRequest>(inprogressFile); if (analysisRequest.ExpirationTime < DateTime.UtcNow) { Logger.LogCpuMonitoringVerboseEvent($"Found an expired analysis request {inprogressFile} that expired {DateTime.UtcNow.Subtract(analysisRequest.ExpirationTime).TotalSeconds} seconds ago", string.Empty); if (analysisRequest.RetryCount < MAX_ANALYSIS_RETRY_COUNT) { try { ++analysisRequest.RetryCount; analysisRequest.ExpirationTime = DateTime.MaxValue; var requestFile = Path.ChangeExtension(inprogressFile, ".request"); FileSystemHelpers.DeleteFileSafe(inprogressFile); analysisRequest.ToJsonFile(requestFile); } catch (Exception ex) { Logger.LogCpuMonitoringErrorEvent("Failed while deleting an expired analysis request", ex, string.Empty); } } else { FileSystemHelpers.DeleteFileSafe(inprogressFile); Logger.LogCpuMonitoringVerboseEvent($"Deleting {inprogressFile} because the analysis retry count was reached", string.Empty); } } } } }
public MonitoringSession CreateSession(MonitoringSession monitoringSession) { string cpuMonitoringActive = GetCpuMonitoringPath(MonitoringSessionDirectories.Active); var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitoringActive, "*.json", false, SearchOption.TopDirectoryOnly); if (existingFiles.Count > 0) { throw new ApplicationException("Another monitoring session is already in progress"); } else { ValidateSessionParameters(monitoringSession); FileSystemHelpers.DeleteDirectoryContentsSafe(cpuMonitoringActive); monitoringSession.StartDate = DateTime.UtcNow; monitoringSession.EndDate = DateTime.MinValue.ToUniversalTime(); monitoringSession.SessionId = monitoringSession.StartDate.ToString(SessionConstants.SessionFileNameFormat); cpuMonitoringActive = Path.Combine(cpuMonitoringActive, monitoringSession.SessionId + ".json"); monitoringSession.ToJsonFile(cpuMonitoringActive); Logger.LogNewCpuMonitoringSession(monitoringSession); } return(monitoringSession); }
private static string AnalyzeDumpFile(AnalysisRequest analysisRequest, string inprogressFile, CancellationToken token) { var blobSasUri = analysisRequest.BlobSasUri; string cpuMonitorPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Logs); var diagnosticToolsPath = Infrastructure.Settings.GetDiagnosticToolsPath(); string outputPath = Path.Combine(cpuMonitorPath, analysisRequest.SessionId); string inputFile = CacheFileInTempDirectory(analysisRequest); string args = $@"-File ""{diagnosticToolsPath}\DumpAnalyzer.ps1"" ""{inputFile}"" ""{outputPath}"""; var command = @"D:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"; Logger.LogCpuMonitoringVerboseEvent($"Powershell started with args [{args}]", analysisRequest.SessionId); double secondsWaited = 0; var processStartTime = DateTime.UtcNow; var toolProcess = Infrastructure.RunProcess(command, args, analysisRequest.SessionId); while (!toolProcess.HasExited) { // // Keep updating the Expiration time while we are waiting for the DumpAnalyzer to finish // analysisRequest.ExpirationTime = DateTime.UtcNow.AddMinutes(ANALYSIS_HEARTBEAT_EXPIRATION_IN_MINUTES); analysisRequest.ToJsonFile(inprogressFile); Thread.Sleep(10 * 1000); secondsWaited = secondsWaited + 10; if (secondsWaited > 120) { secondsWaited = 0; Logger.LogCpuMonitoringVerboseEvent($"Waiting for Analysis process {command} {args} to finish. Process running for {DateTime.UtcNow.Subtract(processStartTime).TotalSeconds} seconds", analysisRequest.SessionId); } if (token != CancellationToken.None && token.IsCancellationRequested) { Logger.LogCpuMonitoringVerboseEvent($"Kill tool process [{command} {args}] because cancellation is requested", analysisRequest.SessionId); toolProcess.SafeKillProcess(); foreach (var dumpAnalyzer in Process.GetProcesses().Where(x => x.ProcessName.Equals("DumpAnalyzer", StringComparison.OrdinalIgnoreCase))) { Logger.LogCpuMonitoringVerboseEvent($"Going to kill [DumpAnalyzer ({dumpAnalyzer.Id})] because cancellation is requested", analysisRequest.SessionId); dumpAnalyzer.SafeKillProcess(); } token.ThrowIfCancellationRequested(); } } // Delete the file in the temp directory once analysis is done FileSystemHelpers.DeleteFileSafe(inputFile); if (toolProcess.ExitCode != 0) { throw new Exception($"Analysis process exited with error code {toolProcess.ExitCode}"); } var reportNamePattern = Path.GetFileNameWithoutExtension(analysisRequest.LogFileName) + "*.mht"; var reportFilePath = FileSystemHelpers.GetFilesInDirectory(outputPath, reportNamePattern).FirstOrDefault(); Logger.LogCpuMonitoringVerboseEvent($"DumpAnalyzer completed and reportPath is [{reportFilePath}]", analysisRequest.SessionId); return(reportFilePath); }