public static void QueueAnalysisRequest(string sessionId, string logFileName, string blobSasUri)
{
try
{
Logger.LogCpuMonitoringVerboseEvent($"Going to Queue Analysis request for {sessionId} for {logFileName}", sessionId);
int retryCount = 0;
retryLabel:
string analysisFolderPath = GetAnalysisFolderPath(out bool errorEncountered);
if (errorEncountered && retryCount < 5)
{
Thread.Sleep(60 * 1000);
++retryCount;
Logger.LogCpuMonitoringVerboseEvent($"Encountered error while getting analysis folder path. RetryCount is {retryCount}", sessionId);
goto retryLabel;
}
if (retryCount == 5)
{
throw new Exception("Storage not ready or in-accessible");
}
if (!errorEncountered)
{
var request = new AnalysisRequest()
{
StartTime = DateTime.UtcNow,
SessionId = sessionId,
LogFileName = logFileName,
ExpirationTime = DateTime.MaxValue,
BlobSasUri = blobSasUri,
RetryCount = 0
};
var fileName = $"{ sessionId }_{ Path.GetFileNameWithoutExtension(logFileName)}";
var requestFile = $"{fileName}.request";
requestFile = Path.Combine(analysisFolderPath, requestFile);
var existingRequests = FileSystemHelpers.GetFilesInDirectory(analysisFolderPath, $"{fileName}*", false, SearchOption.TopDirectoryOnly);
if (existingRequests.Count > 0)
{
//
// There is already and existing analysis request for the same file which is
// either in progress or currently submitted so no need to submit a new one
//
Logger.LogCpuMonitoringVerboseEvent($"An existing request for the same file exists - [{string.Join(",", existingRequests)}]", sessionId);
}
else
{
Logger.LogCpuMonitoringVerboseEvent($"Queued {requestFile} for Analysis", sessionId);
request.ToJsonFile(requestFile);
}
}
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failed while queuing analysis request", ex, sessionId);
}
}
private static void HandleErrorsDuringAnalysis(Exception ex, AnalysisRequest analysisRequest, string inprogressFile, bool cancelFurtherAnalysis = false)
{
List <string> errors = new List <string> {
ex.Message
};
bool shouldUpdateAnalysisStatus = (analysisRequest.RetryCount >= MAX_ANALYSIS_RETRY_COUNT) || (cancelFurtherAnalysis);
UpdateSession(analysisRequest.SessionId, analysisRequest.LogFileName, "", errors, shouldUpdateAnalysisStatus);
// Delete the in-progress file as we have retried too many times or operation was cancelled
if (shouldUpdateAnalysisStatus)
{
FileSystemHelpers.DeleteFileSafe(inprogressFile);
}
Logger.LogCpuMonitoringErrorEvent($"HandleErrorsDuringAnalysis, retry count is {analysisRequest.RetryCount}, cancelFurtherAnalysis = {cancelFurtherAnalysis}", ex, analysisRequest.SessionId);
}
private static string CacheFileInTempDirectory(AnalysisRequest request)
{
string outputPath = MonitoringSessionController.TempFilePath;
FileSystemHelpers.CreateDirectoryIfNotExists(outputPath);
string dumpFileInTempDirectory = Path.Combine(outputPath, Path.GetFileName(request.LogFileName));
Logger.LogCpuMonitoringVerboseEvent($"Caching file {request.LogFileName} to {dumpFileInTempDirectory}", request.SessionId);
if (!FileSystemHelpers.FileExists(dumpFileInTempDirectory))
{
Logger.LogCpuMonitoringVerboseEvent($"File {dumpFileInTempDirectory} does not exist. Copying it locally", request.SessionId);
if (!string.IsNullOrWhiteSpace(request.BlobSasUri))
{
try
{
string filePath = Path.Combine("Monitoring", "Logs", request.SessionId, Path.GetFileName(request.LogFileName));
var blob = BlobController.GetBlobForFile(filePath, request.BlobSasUri);
var ms = new MemoryStream();
blob.DownloadToStream(ms);
ms.Position = 0;
using (FileStream file = new FileStream(dumpFileInTempDirectory, FileMode.Create, FileAccess.Write))
{
byte[] bytes = new byte[ms.Length];
ms.Read(bytes, 0, (int)ms.Length);
file.Write(bytes, 0, bytes.Length);
ms.Close();
}
Logger.LogCpuMonitoringVerboseEvent($"Copied file from {request.LogFileName} to {dumpFileInTempDirectory} ", request.SessionId);
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failed while copying file from Blob", ex, request.SessionId);
}
}
else
{
FileSystemHelpers.CopyFile(request.LogFileName, dumpFileInTempDirectory);
Logger.LogCpuMonitoringVerboseEvent($"Copied file from {request.LogFileName} to {dumpFileInTempDirectory} ", request.SessionId);
}
}
return(dumpFileInTempDirectory);
}
private static string AnalyzeDumpFile(AnalysisRequest analysisRequest, string inprogressFile, CancellationToken token)
{
var blobSasUri = analysisRequest.BlobSasUri;
string cpuMonitorPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Logs);
var diagnosticToolsPath = Infrastructure.Settings.GetDiagnosticToolsPath();
string outputPath = Path.Combine(cpuMonitorPath, analysisRequest.SessionId);
string inputFile = CacheFileInTempDirectory(analysisRequest);
string args = $@"-File ""{diagnosticToolsPath}\DumpAnalyzer.ps1"" ""{inputFile}"" ""{outputPath}""";
var command = @"D:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe";
Logger.LogCpuMonitoringVerboseEvent($"Powershell started with args [{args}]", analysisRequest.SessionId);
double secondsWaited = 0;
var processStartTime = DateTime.UtcNow;
var toolProcess = Infrastructure.RunProcess(command, args, analysisRequest.SessionId);
while (!toolProcess.HasExited)
{
//
// Keep updating the Expiration time while we are waiting for the DumpAnalyzer to finish
//
analysisRequest.ExpirationTime = DateTime.UtcNow.AddMinutes(ANALYSIS_HEARTBEAT_EXPIRATION_IN_MINUTES);
analysisRequest.ToJsonFile(inprogressFile);
Thread.Sleep(10 * 1000);
secondsWaited = secondsWaited + 10;
if (secondsWaited > 120)
{
secondsWaited = 0;
Logger.LogCpuMonitoringVerboseEvent($"Waiting for Analysis process {command} {args} to finish. Process running for {DateTime.UtcNow.Subtract(processStartTime).TotalSeconds} seconds", analysisRequest.SessionId);
}
if (token != CancellationToken.None && token.IsCancellationRequested)
{
Logger.LogCpuMonitoringVerboseEvent($"Kill tool process [{command} {args}] because cancellation is requested", analysisRequest.SessionId);
toolProcess.SafeKillProcess();
foreach (var dumpAnalyzer in Process.GetProcesses().Where(x => x.ProcessName.Equals("DumpAnalyzer", StringComparison.OrdinalIgnoreCase)))
{
Logger.LogCpuMonitoringVerboseEvent($"Going to kill [DumpAnalyzer ({dumpAnalyzer.Id})] because cancellation is requested", analysisRequest.SessionId);
dumpAnalyzer.SafeKillProcess();
}
token.ThrowIfCancellationRequested();
}
}
// Delete the file in the temp directory once analysis is done
FileSystemHelpers.DeleteFileSafe(inputFile);
if (toolProcess.ExitCode != 0)
{
throw new Exception($"Analysis process exited with error code {toolProcess.ExitCode}");
}
var reportNamePattern = Path.GetFileNameWithoutExtension(analysisRequest.LogFileName) + "*.mht";
var reportFilePath = FileSystemHelpers.GetFilesInDirectory(outputPath, reportNamePattern).FirstOrDefault();
Logger.LogCpuMonitoringVerboseEvent($"DumpAnalyzer completed and reportPath is [{reportFilePath}]", analysisRequest.SessionId);
return(reportFilePath);
}