private static void AnalyzeKeepingExpirationAlive(string inprogressFile)
{
var maxTimeInMinutes = Infrastructure.Settings.MaxAnalyzerTimeInMinutes;
var analysisStartTime = DateTime.UtcNow;
var analysisRequest = FileSystemHelpers.FromJsonFile <AnalysisRequest>(inprogressFile);
try
{
var cts = new CancellationTokenSource(TimeSpan.FromMinutes(maxTimeInMinutes));
string reportFilePath = AnalyzeDumpFile(analysisRequest, inprogressFile, cts.Token);
if (!string.IsNullOrWhiteSpace(reportFilePath))
{
UpdateSession(analysisRequest.SessionId, analysisRequest.LogFileName, reportFilePath);
FileSystemHelpers.DeleteFileSafe(inprogressFile);
}
else
{
HandleErrorsDuringAnalysis(new Exception("DumpAnalyzer did not generate any report files"), analysisRequest, inprogressFile);
}
}
catch (OperationCanceledException ex)
{
HandleErrorsDuringAnalysis(ex, analysisRequest, inprogressFile, true);
}
catch (Exception ex)
{
HandleErrorsDuringAnalysis(ex, analysisRequest, inprogressFile);
}
}
public static void DequeueAnalysisRequest()
{
string analysisFolderPath = GetAnalysisFolderPath(out bool errorEncountered);
if (!errorEncountered)
{
try
{
string requestFile = FileSystemHelpers.GetFilesInDirectory(analysisFolderPath, "*.request", false, SearchOption.TopDirectoryOnly).FirstOrDefault();
if (requestFile != null)
{
var analysisRequest = FileSystemHelpers.FromJsonFile <AnalysisRequest>(requestFile);
var isRequestFileFromSameInstance = analysisRequest.LogFileName.StartsWith(Environment.MachineName);
var exceededAnalysisTimelimit = analysisRequest.StartTime != null && DateTime.UtcNow.Subtract(analysisRequest.StartTime).TotalMinutes > MAX_ANALYSISTIME_ON_SAME_INSTANCE;
Logger.LogCpuMonitoringVerboseEvent($"Checking AnalysisRequest LogFileName={analysisRequest.LogFileName}, MachineName={Environment.MachineName}, exceededAnalysisTimelimit={exceededAnalysisTimelimit}, isRequestFileFromSameInstance={isRequestFileFromSameInstance}", analysisRequest.SessionId);
if (exceededAnalysisTimelimit || isRequestFileFromSameInstance)
{
var inprogressFile = Path.ChangeExtension(requestFile, ".inprogress");
FileSystemHelpers.MoveFile(requestFile, inprogressFile);
AnalyzeKeepingExpirationAlive(inprogressFile);
}
}
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failure in DequeueAnalysisRequest", ex, string.Empty);
}
}
}
public bool StopMonitoringSession()
{
Logger.LogCpuMonitoringVerboseEvent($"Inside the StopMonitoringSession method of MonitoringSessionController", string.Empty);
string cpuMonitoringActivePath = GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitoringActivePath, "*.json", false, SearchOption.TopDirectoryOnly);
if (existingFiles.Count > 0)
{
var monitoringSession = FileSystemHelpers.FromJsonFile <MonitoringSession>(existingFiles.FirstOrDefault());
Logger.LogCpuMonitoringVerboseEvent($"Stopping an active session {existingFiles.FirstOrDefault()}", monitoringSession.SessionId);
var canwriteToFileSystem = CheckAndWaitTillFileSystemWritable(monitoringSession.SessionId);
if (!canwriteToFileSystem)
{
return(false);
}
try
{
monitoringSession.EndDate = DateTime.UtcNow;
string cpuMonitorCompletedPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Completed);
cpuMonitorCompletedPath = Path.Combine(cpuMonitorCompletedPath, monitoringSession.SessionId + ".json");
if (!FileSystemHelpers.FileExists(cpuMonitorCompletedPath))
{
monitoringSession.FilesCollected = GetCollectedLogsForSession(monitoringSession.SessionId, monitoringSession.BlobSasUri);
Logger.LogCpuMonitoringVerboseEvent($"Found {monitoringSession.FilesCollected.Count} files collected by CPU monitoring", monitoringSession.SessionId);
SaveSession(monitoringSession);
MoveMonitoringLogsToSession(monitoringSession.SessionId);
}
else
{
// some other instance probably ended up writing the file
// lets hope that finishes and files get moved properly
}
//
// Now delete the Active Session File
//
try
{
FileSystemHelpers.DeleteFileSafe(existingFiles.FirstOrDefault(), false);
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failed while deleting the Active session file", ex, monitoringSession.SessionId);
return(false);
}
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failed while marking a session as Complete", ex, monitoringSession.SessionId);
return(false);
}
}
return(true);
}
public List <MonitoringSession> GetAllCompletedSessions()
{
var sessions = new List <MonitoringSession>();
string cpuMonitorPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Completed);
var existingSessions = FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "*.json", false, SearchOption.TopDirectoryOnly);
foreach (var session in existingSessions)
{
var monitoringSession = FileSystemHelpers.FromJsonFile <MonitoringSession>(session);
sessions.Add(monitoringSession);
}
return(sessions);
}
public MonitoringSession GetSession(string sessionId)
{
string cpuMonitoringCompleted = GetCpuMonitoringPath(MonitoringSessionDirectories.Completed);
var sessionFilePath = Path.Combine(cpuMonitoringCompleted, sessionId + ".json");
if (FileSystemHelpers.FileExists(sessionFilePath))
{
var monitoringSession = FileSystemHelpers.FromJsonFile <MonitoringSession>(sessionFilePath);
return(monitoringSession);
}
else
{
return(null);
}
}
public MonitoringSession GetActiveSession()
{
var sessions = new List <MonitoringSession>();
string cpuMonitorPath = GetCpuMonitoringPath(MonitoringSessionDirectories.Active);
var existingFiles = FileSystemHelpers.GetFilesInDirectory(cpuMonitorPath, "*.json", false, SearchOption.TopDirectoryOnly);
if (existingFiles.Count > 0)
{
Logger.LogDiagnostic($"Found an active monitoring session {existingFiles.FirstOrDefault()}");
var session = FileSystemHelpers.FromJsonFile <MonitoringSession>(existingFiles.FirstOrDefault());
return(session);
}
else
{
Logger.LogDiagnostic($"Found no active monitoring session");
return(null);
}
}
public static void ReSubmitExpiredRequests()
{
string analysisFolderPath = GetAnalysisFolderPath(out bool errorEncountered);
if (!errorEncountered)
{
var inProgressRequests = FileSystemHelpers.GetFilesInDirectory(analysisFolderPath, "*.inprogress", false, SearchOption.TopDirectoryOnly);
foreach (var inprogressFile in inProgressRequests)
{
var analysisRequest = FileSystemHelpers.FromJsonFile <AnalysisRequest>(inprogressFile);
if (analysisRequest.ExpirationTime < DateTime.UtcNow)
{
Logger.LogCpuMonitoringVerboseEvent($"Found an expired analysis request {inprogressFile} that expired {DateTime.UtcNow.Subtract(analysisRequest.ExpirationTime).TotalSeconds} seconds ago", string.Empty);
if (analysisRequest.RetryCount < MAX_ANALYSIS_RETRY_COUNT)
{
try
{
++analysisRequest.RetryCount;
analysisRequest.ExpirationTime = DateTime.MaxValue;
var requestFile = Path.ChangeExtension(inprogressFile, ".request");
FileSystemHelpers.DeleteFileSafe(inprogressFile);
analysisRequest.ToJsonFile(requestFile);
}
catch (Exception ex)
{
Logger.LogCpuMonitoringErrorEvent("Failed while deleting an expired analysis request", ex, string.Empty);
}
}
else
{
FileSystemHelpers.DeleteFileSafe(inprogressFile);
Logger.LogCpuMonitoringVerboseEvent($"Deleting {inprogressFile} because the analysis retry count was reached", string.Empty);
}
}
}
}
}