/// <summary> /// Finalizes report files. /// Thread-safe. /// </summary> private void FinalizeReport(bool bAdded, DirectoryInfo DirInfo, FGenericCrashContext NewContext) { // Only remove if we added the report, otherwise leave all files to further investigation. if (bAdded) { // Remove the report files as we're done with them. CleanReport(DirInfo); } else { MoveReportToInvalid(NewContext.CrashDirectory, NewContext.GetAsFilename()); } }
/// <summary> Enqueues a new crash. </summary> void EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return; } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bFromWER = false; if (Context == null || !Context.HasProcessedData()) { WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { FReportData ReportData = new FReportData(MetaData, NewReportPath); Context = ConvertMetadataToCrashContext(MetaData, NewReportPath); bFromWER = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); } else { if (GenericContext != null && GenericContext.PrimaryCrashProperties.ErrorMessage.Length > 0) { // Get error message from the crash context and fix value in the metadata. Context.PrimaryCrashProperties.ErrorMessage = GenericContext.PrimaryCrashProperties.ErrorMessage; } Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // If based on WER, save to the file. if (bFromWER && GenericContext == null) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteFailure("% NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.MoveReportToInvalid(NewReportPath, Context.GetAsFilename()); return; } if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteFailure("% UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.CleanReport(NewReportPath); return; } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteFailure("! BROKEN0 : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.CleanReport(NewReportPath); return; } lock (NewReportsLock) { NewCrashContexts.Add(Context); } CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); } }
/// <summary> /// Builds MDD command line args, waits for a MDD task slot, runs MDD and blocks for the result. Writes diag text into DiagnosticsPath folder if successful. /// </summary> /// <param name="DiagnosticsPath">Path of the minidump file</param> /// <param name="Context">The crash context</param> /// <param name="ProcessorIndex">Processor thread index for logging purposes</param> /// <returns>True, if successful</returns> public bool Run(string DiagnosticsPath, FGenericCrashContext Context, int ProcessorIndex) { if (!File.Exists(Config.Default.MDDExecutablePath)) { CrashReporterProcessServicer.WriteEvent("Symbolicator.Run() file not found " + Config.Default.MDDExecutablePath); return(false); } // Use MinidumpDiagnostics from MDDExecutablePath. CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.MDDExecutablePath, Config.Default.DiskSpaceAlertPercent); // Don't purge logs // TODO: make this clear to logs once a day or something (without letting MDD check on every run!) string PurgeLogsDays = "-1"; FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); // Pass Windows variants (Win32/64) to MinidumpDiagnostics string PlatformVariant = Context.PrimaryCrashProperties.PlatformName; if (PlatformVariant != null && Context.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS")) { if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win32") || Context.PrimaryCrashProperties.PlatformFullName.Contains("32b")) { PlatformVariant = "Win32"; } else if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win64") || Context.PrimaryCrashProperties.PlatformFullName.Contains("64b")) { PlatformVariant = "Win64"; } } // Build the absolute log file path for MinidumpDiagnostics string BaseFolder = CrashReporterProcessServicer.SymbolicatorLogFolder; DateTime WriteTime = DateTime.UtcNow; string DateFolder = WriteTime.ToString("yyyy_MM_dd"); string HourFolder = WriteTime.ToString("HH"); string Folder = Path.Combine(BaseFolder, DateFolder, HourFolder); string AbsLogPath = Path.Combine(Folder, Context.GetAsFilename() + ".log"); Directory.CreateDirectory(Folder); List <string> MinidumpDiagnosticsParams = new List <string> ( new string[] { "\"" + DiagnosticsPath + "\"", "-BranchName=" + EngineVersion.Branch, // Backward compatibility "-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility "-GameName=" + Context.PrimaryCrashProperties.GameName, "-EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion, "-BuildVersion=" + (string.IsNullOrWhiteSpace(Context.PrimaryCrashProperties.BuildVersion) ? string.Format("{0}-CL-{1}", EngineVersion.Branch, EngineVersion.Changelist).Replace('/', '+') : Context.PrimaryCrashProperties.BuildVersion), "-PlatformName=" + Context.PrimaryCrashProperties.PlatformName, "-PlatformVariantName=" + PlatformVariant, "-bUsePDBCache=true", "-PDBCacheDepotRoot=" + Config.Default.DepotRoot, "-PDBCachePath=" + Config.Default.MDDPDBCachePath, "-PDBCacheSizeGB=" + Config.Default.MDDPDBCacheSizeGB, "-PDBCacheMinFreeSpaceGB=" + Config.Default.MDDPDBCacheMinFreeSpaceGB, "-PDBCacheFileDeleteDays=" + Config.Default.MDDPDBCacheFileDeleteDays, "-MutexPDBCache", "-PDBCacheLock=CrashReportProcessPDBCacheLock", "-NoTrimCallstack", "-SyncSymbols", "-NoP4Symbols", "-ForceUsePDBCache", "-MutexSourceSync", "-SourceSyncLock=CrashReportProcessSourceSyncLock", "-SyncMicrosoftSymbols", "-unattended", "-AbsLog=" + AbsLogPath, "-DepotIndex=" + Config.Default.DepotIndex, "-P4User="******"-P4Client=" + Config.Default.P4Client, "-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays + ",[LogFiles]:MaxLogFilesOnDisk=-1", "-LOGTIMESINCESTART" } ); LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null; if (Environment.UserInteractive) { CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD; } else { MinidumpDiagnosticsParams.AddRange ( new[] { "-buildmachine" } ); // Write some debugging message. CrashReporterProcessServicer.WriteMDD("MinidumpDiagnostics Params: " + String.Join(" ", MinidumpDiagnosticsParams)); } Task <bool> NewSymbolicatorTask = Task.FromResult(false); Stopwatch WaitSW = Stopwatch.StartNew(); double WaitForLockTime = 0.0; lock (Tasks) { int TaskIdx = Task.WaitAny(Tasks); Tasks[TaskIdx] = NewSymbolicatorTask = Task <bool> .Factory.StartNew(() => { LaunchProcess ReportParser = new LaunchProcess(Config.Default.MDDExecutablePath, Path.GetDirectoryName(Config.Default.MDDExecutablePath), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray()); return(ReportParser.WaitForExit(Config.Default.MDDTimeoutMinutes * 1000 * 60) == EWaitResult.Ok); }); WaitForLockTime = WaitSW.Elapsed.TotalSeconds; } NewSymbolicatorTask.Wait(); double TotalMDDTime = WaitSW.Elapsed.TotalSeconds; double MDDRunTime = TotalMDDTime - WaitForLockTime; CrashReporterProcessServicer.StatusReporter.AddToMeanCounter(StatusReportingPerfMeanNames.MinidumpDiagnostics, (int)(MDDRunTime * 1000)); CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Symbolicator.Run: Thread blocked for {0:N1}s then MDD ran for {1:N1}s", WaitForLockTime, MDDRunTime)); return(NewSymbolicatorTask.Result); }
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext) { // Use the latest MinidumpDiagnostics from the main branch. string Win64BinariesDirectory = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine", "Binaries", "Win64"); #if DEBUG // Note: the debug executable must be built locally or synced from Perforce manually string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe"); //string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe"); #else string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe"); #endif // Purge logs every 2048 processed crashes string PurgeLogsDays = ProcessedReports % 2048 == 0 ? "2" : "-1"; FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion); List <string> MinidumpDiagnosticsParams = new List <string> ( new string[] { "\"" + DiagnosticsPath + "\"", "-BranchName=" + EngineVersion.Branch, // Backward compatibility "-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility "-GameName=" + NewContext.PrimaryCrashProperties.GameName, "-EngineVersion=" + NewContext.PrimaryCrashProperties.EngineVersion, "-bUsePDBCache=true", "-Annotate", "-SyncSymbols", "-SyncMicrosoftSymbols", "-unattended", "-Log=" + NewContext.GetAsFilename() + "-backup-.log", "-DepotIndex=" + Properties.Settings.Default.DepotIndex, "-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays, "-LOGTIMESINCESTART" } ); LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null; if (Environment.UserInteractive) { CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD; } else { MinidumpDiagnosticsParams.AddRange ( new[] { "-buildmachine", "-forcelogflush" } ); // Write some debugging message. CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams)); } LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray()); if (ReportParser.WaitForExit(MinidumpDiagnosticsTimeoutSeconds * 1000) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure("ProcessDumpFile: Timed out running MinidumpDiagnostics"); } ReadDiagnosticsFile(NewContext); }
/// <summary> /// Finalizes report files. /// Thread-safe. /// </summary> private void FinalizeReport(AddReportResult AddResult, DirectoryInfo DirInfo, FGenericCrashContext NewContext) { // Only remove if we added the report, otherwise leave all files to further investigation. if (AddResult == AddReportResult.Added) { // Remove the report files as we're done with them. CleanReport(DirInfo); } else if (AddResult == AddReportResult.Failed) { MoveReportToInvalid(NewContext.CrashDirectory, ReportQueueBase.GetSafeFilename(NewContext.GetAsFilename())); } else // AddResult == AddReportResult.Cancelled { // Remove report from index of completed reports and leave on disk CrashReporterProcessServicer.ReportIndex.TryRemoveReport(NewContext.CrashDirectory); } }
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext) { // Use the latest MinidumpDiagnostics from the main branch. string Win64BinariesDirectory = Path.Combine(Config.Default.DepotRoot, Config.Default.MDDBinariesFolderInDepot); #if DEBUG // Note: the debug executable must be built locally or synced from Perforce manually string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe"); #else string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe"); #endif // Purge logs every 2048 processed crashes string PurgeLogsDays = ProcessedReports % 2048 == 0 ? "2" : "-1"; FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion); // Pass Windows variants (Win32/64) to MinidumpDiagnostics string PlatformVariant = NewContext.PrimaryCrashProperties.PlatformName; if (PlatformVariant != null && NewContext.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS")) { if (NewContext.PrimaryCrashProperties.PlatformFullName.Contains("Win32") || NewContext.PrimaryCrashProperties.PlatformFullName.Contains("32b")) { PlatformVariant = "Win32"; } else if (NewContext.PrimaryCrashProperties.PlatformFullName.Contains("Win64") || NewContext.PrimaryCrashProperties.PlatformFullName.Contains("64b")) { PlatformVariant = "Win64"; } } List <string> MinidumpDiagnosticsParams = new List <string> ( new string[] { "\"" + DiagnosticsPath + "\"", "-BranchName=" + EngineVersion.Branch, // Backward compatibility "-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility "-GameName=" + NewContext.PrimaryCrashProperties.GameName, "-EngineVersion=" + NewContext.PrimaryCrashProperties.EngineVersion, "-PlatformName=" + NewContext.PrimaryCrashProperties.PlatformName, "-PlatformVariantName=" + PlatformVariant, "-bUsePDBCache=true", "-Annotate", "-SyncSymbols", "-SyncMicrosoftSymbols", "-unattended", "-Log=" + NewContext.GetAsFilename() + "-backup-.log", "-DepotIndex=" + Config.Default.DepotIndex, "-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays, "-LOGTIMESINCESTART" } ); LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null; if (Environment.UserInteractive) { CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD; } else { MinidumpDiagnosticsParams.AddRange ( new[] { "-buildmachine", "-forcelogflush" } ); // Write some debugging message. CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams)); } Stopwatch WaitSW = Stopwatch.StartNew(); lock (MinidumpDiagnosticsLock) { Double WaitForLockTime = WaitSW.Elapsed.TotalSeconds; LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray()); if (ReportParser.WaitForExit(MinidumpDiagnosticsTimeoutMilliseconds) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Timed out running MinidumpDiagnostics"); } Double TotalMDDTime = WaitSW.Elapsed.TotalSeconds; CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("ProcessDumpFile: Total MDD exec time {0:N1}s (blocked for {1:N1}s)", TotalMDDTime, WaitForLockTime)); } ReadDiagnosticsFile(NewContext); }