public static void QueueAnalysisRequest(string sessionId, string logFileName, string blobSasUri) { try { Logger.LogCpuMonitoringVerboseEvent($"Going to Queue Analysis request for {sessionId} for {logFileName}", sessionId); int retryCount = 0; retryLabel: string analysisFolderPath = GetAnalysisFolderPath(out bool errorEncountered); if (errorEncountered && retryCount < 5) { Thread.Sleep(60 * 1000); ++retryCount; Logger.LogCpuMonitoringVerboseEvent($"Encountered error while getting analysis folder path. RetryCount is {retryCount}", sessionId); goto retryLabel; } if (retryCount == 5) { throw new Exception("Storage not ready or in-accessible"); } if (!errorEncountered) { var request = new AnalysisRequest() { StartTime = DateTime.UtcNow, SessionId = sessionId, LogFileName = logFileName, ExpirationTime = DateTime.MaxValue, BlobSasUri = blobSasUri, RetryCount = 0 }; var fileName = $"{ sessionId }_{ Path.GetFileNameWithoutExtension(logFileName)}"; var requestFile = $"{fileName}.request"; requestFile = Path.Combine(analysisFolderPath, requestFile); var existingRequests = FileSystemHelpers.GetFilesInDirectory(analysisFolderPath, $"{fileName}*", false, SearchOption.TopDirectoryOnly); if (existingRequests.Count > 0) { // // There is already and existing analysis request for the same file which is // either in progress or currently submitted so no need to submit a new one // Logger.LogCpuMonitoringVerboseEvent($"An existing request for the same file exists - [{string.Join(",", existingRequests)}]", sessionId); } else { Logger.LogCpuMonitoringVerboseEvent($"Queued {requestFile} for Analysis", sessionId); request.ToJsonFile(requestFile); } } } catch (Exception ex) { Logger.LogCpuMonitoringErrorEvent("Failed while queuing analysis request", ex, sessionId); } }
private static void HandleErrorsDuringAnalysis(Exception ex, AnalysisRequest analysisRequest, string inprogressFile, bool cancelFurtherAnalysis = false) { List <string> errors = new List <string> { ex.Message }; bool shouldUpdateAnalysisStatus = (analysisRequest.RetryCount >= MAX_ANALYSIS_RETRY_COUNT) || (cancelFurtherAnalysis); UpdateSession(analysisRequest.SessionId, analysisRequest.LogFileName, "", errors, shouldUpdateAnalysisStatus); // Delete the in-progress file as we have retried too many times or operation was cancelled if (shouldUpdateAnalysisStatus) { FileSystemHelpers.DeleteFileSafe(inprogressFile); } Logger.LogCpuMonitoringErrorEvent($"HandleErrorsDuringAnalysis, retry count is {analysisRequest.RetryCount}, cancelFurtherAnalysis = {cancelFurtherAnalysis}", ex, analysisRequest.SessionId); }
private static string CacheFileInTempDirectory(AnalysisRequest request) { string outputPath = MonitoringSessionController.TempFilePath; FileSystemHelpers.CreateDirectoryIfNotExists(outputPath); string dumpFileInTempDirectory = Path.Combine(outputPath, Path.GetFileName(request.LogFileName)); Logger.LogCpuMonitoringVerboseEvent($"Caching file {request.LogFileName} to {dumpFileInTempDirectory}", request.SessionId); if (!FileSystemHelpers.FileExists(dumpFileInTempDirectory)) { Logger.LogCpuMonitoringVerboseEvent($"File {dumpFileInTempDirectory} does not exist. Copying it locally", request.SessionId); if (!string.IsNullOrWhiteSpace(request.BlobSasUri)) { try { string filePath = Path.Combine("Monitoring", "Logs", request.SessionId, Path.GetFileName(request.LogFileName)); var blob = BlobController.GetBlobForFile(filePath, request.BlobSasUri); var ms = new MemoryStream(); blob.DownloadToStream(ms); ms.Position = 0; using (FileStream file = new FileStream(dumpFileInTempDirectory, FileMode.Create, FileAccess.Write)) { byte[] bytes = new byte[ms.Length]; ms.Read(bytes, 0, (int)ms.Length); file.Write(bytes, 0, bytes.Length); ms.Close(); } Logger.LogCpuMonitoringVerboseEvent($"Copied file from {request.LogFileName} to {dumpFileInTempDirectory} ", request.SessionId); } catch (Exception ex) { Logger.LogCpuMonitoringErrorEvent("Failed while copying file from Blob", ex, request.SessionId); } } else { FileSystemHelpers.CopyFile(request.LogFileName, dumpFileInTempDirectory); Logger.LogCpuMonitoringVerboseEvent($"Copied file from {request.LogFileName} to {dumpFileInTempDirectory} ", request.SessionId); } } return(dumpFileInTempDirectory); }
private static string AnalyzeDumpFile(AnalysisRequest analysisRequest, string inprogressFile, CancellationToken token) { var blobSasUri = analysisRequest.BlobSasUri; string cpuMonitorPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Logs); var diagnosticToolsPath = Infrastructure.Settings.GetDiagnosticToolsPath(); string outputPath = Path.Combine(cpuMonitorPath, analysisRequest.SessionId); string inputFile = CacheFileInTempDirectory(analysisRequest); string args = $@"-File ""{diagnosticToolsPath}\DumpAnalyzer.ps1"" ""{inputFile}"" ""{outputPath}"""; var command = @"D:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"; Logger.LogCpuMonitoringVerboseEvent($"Powershell started with args [{args}]", analysisRequest.SessionId); double secondsWaited = 0; var processStartTime = DateTime.UtcNow; var toolProcess = Infrastructure.RunProcess(command, args, analysisRequest.SessionId); while (!toolProcess.HasExited) { // // Keep updating the Expiration time while we are waiting for the DumpAnalyzer to finish // analysisRequest.ExpirationTime = DateTime.UtcNow.AddMinutes(ANALYSIS_HEARTBEAT_EXPIRATION_IN_MINUTES); analysisRequest.ToJsonFile(inprogressFile); Thread.Sleep(10 * 1000); secondsWaited = secondsWaited + 10; if (secondsWaited > 120) { secondsWaited = 0; Logger.LogCpuMonitoringVerboseEvent($"Waiting for Analysis process {command} {args} to finish. Process running for {DateTime.UtcNow.Subtract(processStartTime).TotalSeconds} seconds", analysisRequest.SessionId); } if (token != CancellationToken.None && token.IsCancellationRequested) { Logger.LogCpuMonitoringVerboseEvent($"Kill tool process [{command} {args}] because cancellation is requested", analysisRequest.SessionId); toolProcess.SafeKillProcess(); foreach (var dumpAnalyzer in Process.GetProcesses().Where(x => x.ProcessName.Equals("DumpAnalyzer", StringComparison.OrdinalIgnoreCase))) { Logger.LogCpuMonitoringVerboseEvent($"Going to kill [DumpAnalyzer ({dumpAnalyzer.Id})] because cancellation is requested", analysisRequest.SessionId); dumpAnalyzer.SafeKillProcess(); } token.ThrowIfCancellationRequested(); } } // Delete the file in the temp directory once analysis is done FileSystemHelpers.DeleteFileSafe(inputFile); if (toolProcess.ExitCode != 0) { throw new Exception($"Analysis process exited with error code {toolProcess.ExitCode}"); } var reportNamePattern = Path.GetFileNameWithoutExtension(analysisRequest.LogFileName) + "*.mht"; var reportFilePath = FileSystemHelpers.GetFilesInDirectory(outputPath, reportNamePattern).FirstOrDefault(); Logger.LogCpuMonitoringVerboseEvent($"DumpAnalyzer completed and reportPath is [{reportFilePath}]", analysisRequest.SessionId); return(reportFilePath); }