public IEnumerable <MonitoringLogsPerInstance> GetActiveSessionMonitoringLogs()
{
var logs = new List <MonitoringLogsPerInstance>();
string cpuMonitorPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
var activeInstances = HeartBeats.HeartBeatController.GetLiveInstances();
if (GetActiveSession() != null)
{
foreach (var logFile in FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "RD*.log", false, SearchOption.TopDirectoryOnly))
{
string instanceName = Path.GetFileNameWithoutExtension(logFile);
if (activeInstances.Any(x => x.Name.Equals(instanceName, StringComparison.OrdinalIgnoreCase)))
{
string logContent = ReadEndTokens(logFile, 10, Encoding.Default, Environment.NewLine);
logs.Add(new MonitoringLogsPerInstance()
{
Instance = instanceName,
Logs = logContent
});
}
}
}
return(logs);
}
public List <MonitoringFile> GetCollectedLogsForSession(string sessionId, string blobSasUri)
{
var filesCollected = new List <MonitoringFile>();
if (string.IsNullOrWhiteSpace(blobSasUri))
{
string folderName = CpuMonitoring.GetLogsFolderForSession(sessionId);
if (FileSystemHelpers.DirectoryExists(folderName))
{
var logFiles = FileSystemHelpers.GetFilesInDirectory(folderName, "*.dmp", false, SearchOption.TopDirectoryOnly);
foreach (var fileName in logFiles)
{
string relativePath = MonitoringFile.GetRelativePath(sessionId, Path.GetFileName(fileName));
filesCollected.Add(new MonitoringFile(fileName, relativePath));
}
}
}
else
{
string directoryPath = Path.Combine("Monitoring", "Logs", sessionId);
List <string> files = new List <string>();
var dir = BlobController.GetBlobDirectory(directoryPath, blobSasUri);
foreach (
IListBlobItem item in
dir.ListBlobs(useFlatBlobListing: true))
{
var relativePath = item.Uri.ToString().Replace(item.Container.Uri.ToString() + "/", "");
string fileName = item.Uri.Segments.Last();
filesCollected.Add(new MonitoringFile(fileName, relativePath));
}
}
return(filesCollected);
}
private int GetTotalCustomActionsCompleted(string sessionId)
{
var sessionDirectory = GetLogsFolderForSession(sessionId);
var dumpCount = FileSystemHelpers.GetFilesInDirectory(sessionDirectory, "*.customactioncompleted", false, SearchOption.TopDirectoryOnly).Count;
return(dumpCount);
}
private void AppendToMonitoringLog(string message, bool logInKusto = false)
{
Interlocked.Increment(ref _loggerCount);
string cpuMonitorPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
string logFilePath = Path.Combine(cpuMonitorPath, Environment.MachineName + ".log");
string logMessage = $"[{DateTime.UtcNow.ToShortDateString()} {DateTime.UtcNow.ToString("hh:mm:ss")}] {message}{Environment.NewLine}";
if (_loggerCount > MAX_LINES_IN_LOGFILE)
{
var sessionDirectory = GetLogsFolderForSession(_sessionId);
var existingFileCount = FileSystemHelpers.GetFilesInDirectory(sessionDirectory, $"{Environment.MachineName}*.log", false, SearchOption.TopDirectoryOnly).Count;
var newFileName = $"{Environment.MachineName}_{existingFileCount}.log";
newFileName = Path.Combine(sessionDirectory, newFileName);
FileSystemHelpers.MoveFile(logFilePath, newFileName);
Interlocked.Exchange(ref _loggerCount, 0);
}
try
{
FileSystemHelpers.AppendAllTextToFile(logFilePath, logMessage);
}
catch (Exception)
{
}
if (logInKusto)
{
Logger.LogCpuMonitoringEvent(message, _sessionId);
}
}
public static void DequeueAnalysisRequest()
{
string analysisFolderPath = GetAnalysisFolderPath(out bool errorEncountered);
if (!errorEncountered)
{
try
{
string requestFile = FileSystemHelpers.GetFilesInDirectory(analysisFolderPath, "*.request", false, SearchOption.TopDirectoryOnly).FirstOrDefault();
if (requestFile != null)
{
var analysisRequest = FileSystemHelpers.FromJsonFile <AnalysisRequest>(requestFile);
var isRequestFileFromSameInstance = analysisRequest.LogFileName.StartsWith(Environment.MachineName);
var exceededAnalysisTimelimit = analysisRequest.StartTime != null && DateTime.UtcNow.Subtract(analysisRequest.StartTime).TotalMinutes > MAX_ANALYSISTIME_ON_SAME_INSTANCE;
Logger.LogCpuMonitoringVerboseEvent($"Checking AnalysisRequest LogFileName={analysisRequest.LogFileName}, MachineName={Environment.MachineName}, exceededAnalysisTimelimit={exceededAnalysisTimelimit}, isRequestFileFromSameInstance={isRequestFileFromSameInstance}", analysisRequest.SessionId);
if (exceededAnalysisTimelimit || isRequestFileFromSameInstance)
{
var inprogressFile = Path.ChangeExtension(requestFile, ".inprogress");
FileSystemHelpers.MoveFile(requestFile, inprogressFile);
AnalyzeKeepingExpirationAlive(inprogressFile);
}
}
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failure in DequeueAnalysisRequest", ex, string.Empty);
}
}
}
public static void QueueAnalysisRequest(string sessionId, string logFileName, string blobSasUri)
{
try
{
Logger.LogCpuMonitoringVerboseEvent($"Going to Queue Analysis request for {sessionId} for {logFileName}", sessionId);
int retryCount = 0;
retryLabel:
string analysisFolderPath = GetAnalysisFolderPath(out bool errorEncountered);
if (errorEncountered && retryCount < 5)
{
Thread.Sleep(60 * 1000);
++retryCount;
Logger.LogCpuMonitoringVerboseEvent($"Encountered error while getting analysis folder path. RetryCount is {retryCount}", sessionId);
goto retryLabel;
}
if (retryCount == 5)
{
throw new Exception("Storage not ready or in-accessible");
}
if (!errorEncountered)
{
var request = new AnalysisRequest()
{
StartTime = DateTime.UtcNow,
SessionId = sessionId,
LogFileName = logFileName,
ExpirationTime = DateTime.MaxValue,
BlobSasUri = blobSasUri,
RetryCount = 0
};
var fileName = $"{ sessionId }_{ Path.GetFileNameWithoutExtension(logFileName)}";
var requestFile = $"{fileName}.request";
requestFile = Path.Combine(analysisFolderPath, requestFile);
var existingRequests = FileSystemHelpers.GetFilesInDirectory(analysisFolderPath, $"{fileName}*", false, SearchOption.TopDirectoryOnly);
if (existingRequests.Count > 0)
{
//
// There is already and existing analysis request for the same file which is
// either in progress or currently submitted so no need to submit a new one
//
Logger.LogCpuMonitoringVerboseEvent($"An existing request for the same file exists - [{string.Join(",", existingRequests)}]", sessionId);
}
else
{
Logger.LogCpuMonitoringVerboseEvent($"Queued {requestFile} for Analysis", sessionId);
request.ToJsonFile(requestFile);
}
}
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failed while queuing analysis request", ex, sessionId);
}
}
public bool StopMonitoringSession()
{
Logger.LogCpuMonitoringVerboseEvent($"Inside the StopMonitoringSession method of MonitoringSessionController", string.Empty);
string cpuMonitoringActivePath = GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitoringActivePath, "*.json", false, SearchOption.TopDirectoryOnly);
if (existingFiles.Count > 0)
{
var monitoringSession = FileSystemHelpers.FromJsonFile <MonitoringSession>(existingFiles.FirstOrDefault());
Logger.LogCpuMonitoringVerboseEvent($"Stopping an active session {existingFiles.FirstOrDefault()}", monitoringSession.SessionId);
var canwriteToFileSystem = CheckAndWaitTillFileSystemWritable(monitoringSession.SessionId);
if (!canwriteToFileSystem)
{
return(false);
}
try
{
monitoringSession.EndDate = DateTime.UtcNow;
string cpuMonitorCompletedPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Completed);
cpuMonitorCompletedPath = Path.Combine(cpuMonitorCompletedPath, monitoringSession.SessionId + ".json");
if (!FileSystemHelpers.FileExists(cpuMonitorCompletedPath))
{
monitoringSession.FilesCollected = GetCollectedLogsForSession(monitoringSession.SessionId, monitoringSession.BlobSasUri);
Logger.LogCpuMonitoringVerboseEvent($"Found {monitoringSession.FilesCollected.Count} files collected by CPU monitoring", monitoringSession.SessionId);
SaveSession(monitoringSession);
MoveMonitoringLogsToSession(monitoringSession.SessionId);
}
else
{
// some other instance probably ended up writing the file
// lets hope that finishes and files get moved properly
}
//
// Now delete the Active Session File
//
try
{
FileSystemHelpers.DeleteFileSafe(existingFiles.FirstOrDefault(), false);
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failed while deleting the Active session file", ex, monitoringSession.SessionId);
return(false);
}
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failed while marking a session as Complete", ex, monitoringSession.SessionId);
return(false);
}
}
return(true);
}
public void TerminateActiveMonitoringSession()
{
string cpuMonitorPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "*.json", false, SearchOption.TopDirectoryOnly);
if (existingFiles.Count > 0)
{
FileSystemHelpers.DeleteDirectoryContentsSafe(cpuMonitorPath, true);
}
}
public static void CleanRemainingLogsIfAny()
{
string cpuMonitorPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
if (FileSystemHelpers.DirectoryExists(cpuMonitorPath))
{
foreach (var log in FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "*.log", false, SearchOption.TopDirectoryOnly))
{
FileSystemHelpers.DeleteFileSafe(log);
}
}
}
public List <MonitoringSession> GetAllCompletedSessions()
{
var sessions = new List <MonitoringSession>();
string cpuMonitorPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Completed);
var existingSessions = FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "*.json", false, SearchOption.TopDirectoryOnly);
foreach (var session in existingSessions)
{
var monitoringSession = FileSystemHelpers.FromJsonFile <MonitoringSession>(session);
sessions.Add(monitoringSession);
}
return(sessions);
}
private void MoveMonitoringLogsToSession(string sessionId)
{
string logsFolderPath = CpuMonitoring.GetLogsFolderForSession(sessionId);
string monitoringFolderActive = GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
var filesCollected = FileSystemHelpers.GetFilesInDirectory(monitoringFolderActive, "*.log", false, SearchOption.TopDirectoryOnly);
foreach (string monitoringLog in filesCollected)
{
string fileName = Path.GetFileName(monitoringLog);
fileName = Path.Combine(logsFolderPath, fileName);
Logger.LogCpuMonitoringVerboseEvent($"Moving {monitoringLog} to {fileName}", sessionId);
RetryHelper.RetryOnException("Moving monitoring log to logs folder...", () =>
{
FileSystemHelpers.MoveFile(monitoringLog, fileName);
}, TimeSpan.FromSeconds(5), 5);
}
}
public MonitoringSession GetActiveSession()
{
var sessions = new List <MonitoringSession>();
string cpuMonitorPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "*.json", false, SearchOption.TopDirectoryOnly);
if (existingFiles.Count > 0)
{
Logger.LogDiagnostic($"Found an active monitoring session {existingFiles.FirstOrDefault()}");
var session = FileSystemHelpers.FromJsonFile <MonitoringSession>(existingFiles.FirstOrDefault());
return(session);
}
else
{
Logger.LogDiagnostic($"Found no active monitoring session");
return(null);
}
}
public static void ReSubmitExpiredRequests()
{
string analysisFolderPath = GetAnalysisFolderPath(out bool errorEncountered);
if (!errorEncountered)
{
var inProgressRequests = FileSystemHelpers.GetFilesInDirectory(analysisFolderPath, "*.inprogress", false, SearchOption.TopDirectoryOnly);
foreach (var inprogressFile in inProgressRequests)
{
var analysisRequest = FileSystemHelpers.FromJsonFile <AnalysisRequest>(inprogressFile);
if (analysisRequest.ExpirationTime < DateTime.UtcNow)
{
Logger.LogCpuMonitoringVerboseEvent($"Found an expired analysis request {inprogressFile} that expired {DateTime.UtcNow.Subtract(analysisRequest.ExpirationTime).TotalSeconds} seconds ago", string.Empty);
if (analysisRequest.RetryCount < MAX_ANALYSIS_RETRY_COUNT)
{
try
{
++analysisRequest.RetryCount;
analysisRequest.ExpirationTime = DateTime.MaxValue;
var requestFile = Path.ChangeExtension(inprogressFile, ".request");
FileSystemHelpers.DeleteFileSafe(inprogressFile);
analysisRequest.ToJsonFile(requestFile);
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failed while deleting an expired analysis request", ex, string.Empty);
}
}
else
{
FileSystemHelpers.DeleteFileSafe(inprogressFile);
Logger.LogCpuMonitoringVerboseEvent($"Deleting {inprogressFile} because the analysis retry count was reached", string.Empty);
}
}
}
}
}
public MonitoringSession CreateSession(MonitoringSession monitoringSession)
{
string cpuMonitoringActive = GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitoringActive, "*.json", false, SearchOption.TopDirectoryOnly);
if (existingFiles.Count > 0)
{
throw new ApplicationException("Another monitoring session is already in progress");
}
else
{
ValidateSessionParameters(monitoringSession);
FileSystemHelpers.DeleteDirectoryContentsSafe(cpuMonitoringActive);
monitoringSession.StartDate = DateTime.UtcNow;
monitoringSession.EndDate = DateTime.MinValue.ToUniversalTime();
monitoringSession.SessionId = monitoringSession.StartDate.ToString(SessionConstants.SessionFileNameFormat);
cpuMonitoringActive = Path.Combine(cpuMonitoringActive, monitoringSession.SessionId + ".json");
monitoringSession.ToJsonFile(cpuMonitoringActive);
Logger.LogNewCpuMonitoringSession(monitoringSession);
}
return(monitoringSession);
}
private static string AnalyzeDumpFile(AnalysisRequest analysisRequest, string inprogressFile, CancellationToken token)
{
var blobSasUri = analysisRequest.BlobSasUri;
string cpuMonitorPath = MonitoringSessionController.GetCpuMonitoringPath(MonitoringSessionDirectories.Logs);
var diagnosticToolsPath = Infrastructure.Settings.GetDiagnosticToolsPath();
string outputPath = Path.Combine(cpuMonitorPath, analysisRequest.SessionId);
string inputFile = CacheFileInTempDirectory(analysisRequest);
string args = $@"-File ""{diagnosticToolsPath}\DumpAnalyzer.ps1"" ""{inputFile}"" ""{outputPath}""";
var command = @"D:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe";
Logger.LogCpuMonitoringVerboseEvent($"Powershell started with args [{args}]", analysisRequest.SessionId);
double secondsWaited = 0;
var processStartTime = DateTime.UtcNow;
var toolProcess = Infrastructure.RunProcess(command, args, analysisRequest.SessionId);
while (!toolProcess.HasExited)
{
//
// Keep updating the Expiration time while we are waiting for the DumpAnalyzer to finish
//
analysisRequest.ExpirationTime = DateTime.UtcNow.AddMinutes(ANALYSIS_HEARTBEAT_EXPIRATION_IN_MINUTES);
analysisRequest.ToJsonFile(inprogressFile);
Thread.Sleep(10 * 1000);
secondsWaited = secondsWaited + 10;
if (secondsWaited > 120)
{
secondsWaited = 0;
Logger.LogCpuMonitoringVerboseEvent($"Waiting for Analysis process {command} {args} to finish. Process running for {DateTime.UtcNow.Subtract(processStartTime).TotalSeconds} seconds", analysisRequest.SessionId);
}
if (token != CancellationToken.None && token.IsCancellationRequested)
{
Logger.LogCpuMonitoringVerboseEvent($"Kill tool process [{command} {args}] because cancellation is requested", analysisRequest.SessionId);
toolProcess.SafeKillProcess();
foreach (var dumpAnalyzer in Process.GetProcesses().Where(x => x.ProcessName.Equals("DumpAnalyzer", StringComparison.OrdinalIgnoreCase)))
{
Logger.LogCpuMonitoringVerboseEvent($"Going to kill [DumpAnalyzer ({dumpAnalyzer.Id})] because cancellation is requested", analysisRequest.SessionId);
dumpAnalyzer.SafeKillProcess();
}
token.ThrowIfCancellationRequested();
}
}
// Delete the file in the temp directory once analysis is done
FileSystemHelpers.DeleteFileSafe(inputFile);
if (toolProcess.ExitCode != 0)
{
throw new Exception($"Analysis process exited with error code {toolProcess.ExitCode}");
}
var reportNamePattern = Path.GetFileNameWithoutExtension(analysisRequest.LogFileName) + "*.mht";
var reportFilePath = FileSystemHelpers.GetFilesInDirectory(outputPath, reportNamePattern).FirstOrDefault();
Logger.LogCpuMonitoringVerboseEvent($"DumpAnalyzer completed and reportPath is [{reportFilePath}]", analysisRequest.SessionId);
return(reportFilePath);
}