/// <summary>
/// Finalizes report files.
/// Thread-safe.
/// </summary>
private void FinalizeReport(bool bAdded, DirectoryInfo DirInfo, FGenericCrashContext NewContext)
{
// Only remove if we added the report, otherwise leave all files to further investigation.
if (bAdded)
{
// Remove the report files as we're done with them.
CleanReport(DirInfo);
}
else
{
MoveReportToInvalid(NewContext.CrashDirectory, NewContext.GetAsFilename());
}
}
/// <summary> Enqueues a new crash. </summary>
void EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return;
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bFromWER = false;
if (Context == null || !Context.HasProcessedData())
{
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
FReportData ReportData = new FReportData(MetaData, NewReportPath);
Context = ConvertMetadataToCrashContext(MetaData, NewReportPath);
bFromWER = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
}
else
{
if (GenericContext != null && GenericContext.PrimaryCrashProperties.ErrorMessage.Length > 0)
{
// Get error message from the crash context and fix value in the metadata.
Context.PrimaryCrashProperties.ErrorMessage = GenericContext.PrimaryCrashProperties.ErrorMessage;
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// If based on WER, save to the file.
if (bFromWER && GenericContext == null)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteFailure("% NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.MoveReportToInvalid(NewReportPath, Context.GetAsFilename());
return;
}
if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteFailure("% UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.CleanReport(NewReportPath);
return;
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteFailure("! BROKEN0 : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.CleanReport(NewReportPath);
return;
}
lock (NewReportsLock)
{
NewCrashContexts.Add(Context);
}
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
}
}
/// <summary>
/// Builds MDD command line args, waits for a MDD task slot, runs MDD and blocks for the result. Writes diag text into DiagnosticsPath folder if successful.
/// </summary>
/// <param name="DiagnosticsPath">Path of the minidump file</param>
/// <param name="Context">The crash context</param>
/// <param name="ProcessorIndex">Processor thread index for logging purposes</param>
/// <returns>True, if successful</returns>
public bool Run(string DiagnosticsPath, FGenericCrashContext Context, int ProcessorIndex)
{
if (!File.Exists(Config.Default.MDDExecutablePath))
{
CrashReporterProcessServicer.WriteEvent("Symbolicator.Run() file not found " + Config.Default.MDDExecutablePath);
return(false);
}
// Use MinidumpDiagnostics from MDDExecutablePath.
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.MDDExecutablePath, Config.Default.DiskSpaceAlertPercent);
// Don't purge logs
// TODO: make this clear to logs once a day or something (without letting MDD check on every run!)
string PurgeLogsDays = "-1";
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
// Pass Windows variants (Win32/64) to MinidumpDiagnostics
string PlatformVariant = Context.PrimaryCrashProperties.PlatformName;
if (PlatformVariant != null && Context.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS"))
{
if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win32") ||
Context.PrimaryCrashProperties.PlatformFullName.Contains("32b"))
{
PlatformVariant = "Win32";
}
else if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win64") ||
Context.PrimaryCrashProperties.PlatformFullName.Contains("64b"))
{
PlatformVariant = "Win64";
}
}
// Build the absolute log file path for MinidumpDiagnostics
string BaseFolder = CrashReporterProcessServicer.SymbolicatorLogFolder;
DateTime WriteTime = DateTime.UtcNow;
string DateFolder = WriteTime.ToString("yyyy_MM_dd");
string HourFolder = WriteTime.ToString("HH");
string Folder = Path.Combine(BaseFolder, DateFolder, HourFolder);
string AbsLogPath = Path.Combine(Folder, Context.GetAsFilename() + ".log");
Directory.CreateDirectory(Folder);
List <string> MinidumpDiagnosticsParams = new List <string>
(
new string[]
{
"\"" + DiagnosticsPath + "\"",
"-BranchName=" + EngineVersion.Branch, // Backward compatibility
"-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility
"-GameName=" + Context.PrimaryCrashProperties.GameName,
"-EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion,
"-BuildVersion=" + (string.IsNullOrWhiteSpace(Context.PrimaryCrashProperties.BuildVersion) ?
string.Format("{0}-CL-{1}", EngineVersion.Branch, EngineVersion.Changelist).Replace('/', '+') :
Context.PrimaryCrashProperties.BuildVersion),
"-PlatformName=" + Context.PrimaryCrashProperties.PlatformName,
"-PlatformVariantName=" + PlatformVariant,
"-bUsePDBCache=true",
"-PDBCacheDepotRoot=" + Config.Default.DepotRoot,
"-PDBCachePath=" + Config.Default.MDDPDBCachePath,
"-PDBCacheSizeGB=" + Config.Default.MDDPDBCacheSizeGB,
"-PDBCacheMinFreeSpaceGB=" + Config.Default.MDDPDBCacheMinFreeSpaceGB,
"-PDBCacheFileDeleteDays=" + Config.Default.MDDPDBCacheFileDeleteDays,
"-MutexPDBCache",
"-PDBCacheLock=CrashReportProcessPDBCacheLock",
"-NoTrimCallstack",
"-SyncSymbols",
"-NoP4Symbols",
"-ForceUsePDBCache",
"-MutexSourceSync",
"-SourceSyncLock=CrashReportProcessSourceSyncLock",
"-SyncMicrosoftSymbols",
"-unattended",
"-AbsLog=" + AbsLogPath,
"-DepotIndex=" + Config.Default.DepotIndex,
"-P4User="******"-P4Client=" + Config.Default.P4Client,
"-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays + ",[LogFiles]:MaxLogFilesOnDisk=-1",
"-LOGTIMESINCESTART"
}
);
LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null;
if (Environment.UserInteractive)
{
CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD;
}
else
{
MinidumpDiagnosticsParams.AddRange
(
new[]
{
"-buildmachine"
}
);
// Write some debugging message.
CrashReporterProcessServicer.WriteMDD("MinidumpDiagnostics Params: " + String.Join(" ", MinidumpDiagnosticsParams));
}
Task <bool> NewSymbolicatorTask = Task.FromResult(false);
Stopwatch WaitSW = Stopwatch.StartNew();
double WaitForLockTime = 0.0;
lock (Tasks)
{
int TaskIdx = Task.WaitAny(Tasks);
Tasks[TaskIdx] = NewSymbolicatorTask = Task <bool> .Factory.StartNew(() =>
{
LaunchProcess ReportParser = new LaunchProcess(Config.Default.MDDExecutablePath, Path.GetDirectoryName(Config.Default.MDDExecutablePath), CaptureMessageDelegate,
MinidumpDiagnosticsParams.ToArray());
return(ReportParser.WaitForExit(Config.Default.MDDTimeoutMinutes * 1000 * 60) == EWaitResult.Ok);
});
WaitForLockTime = WaitSW.Elapsed.TotalSeconds;
}
NewSymbolicatorTask.Wait();
double TotalMDDTime = WaitSW.Elapsed.TotalSeconds;
double MDDRunTime = TotalMDDTime - WaitForLockTime;
CrashReporterProcessServicer.StatusReporter.AddToMeanCounter(StatusReportingPerfMeanNames.MinidumpDiagnostics, (int)(MDDRunTime * 1000));
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Symbolicator.Run: Thread blocked for {0:N1}s then MDD ran for {1:N1}s", WaitForLockTime, MDDRunTime));
return(NewSymbolicatorTask.Result);
}
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext)
{
// Use the latest MinidumpDiagnostics from the main branch.
string Win64BinariesDirectory = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine", "Binaries", "Win64");
#if DEBUG
// Note: the debug executable must be built locally or synced from Perforce manually
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe");
//string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe");
#else
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe");
#endif
// Purge logs every 2048 processed crashes
string PurgeLogsDays = ProcessedReports % 2048 == 0 ? "2" : "-1";
FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion);
List <string> MinidumpDiagnosticsParams = new List <string>
(
new string[]
{
"\"" + DiagnosticsPath + "\"",
"-BranchName=" + EngineVersion.Branch, // Backward compatibility
"-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility
"-GameName=" + NewContext.PrimaryCrashProperties.GameName,
"-EngineVersion=" + NewContext.PrimaryCrashProperties.EngineVersion,
"-bUsePDBCache=true",
"-Annotate",
"-SyncSymbols",
"-SyncMicrosoftSymbols",
"-unattended",
"-Log=" + NewContext.GetAsFilename() + "-backup-.log",
"-DepotIndex=" + Properties.Settings.Default.DepotIndex,
"-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays,
"-LOGTIMESINCESTART"
}
);
LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null;
if (Environment.UserInteractive)
{
CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD;
}
else
{
MinidumpDiagnosticsParams.AddRange
(
new[]
{
"-buildmachine",
"-forcelogflush"
}
);
// Write some debugging message.
CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams));
}
LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray());
if (ReportParser.WaitForExit(MinidumpDiagnosticsTimeoutSeconds * 1000) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure("ProcessDumpFile: Timed out running MinidumpDiagnostics");
}
ReadDiagnosticsFile(NewContext);
}
/// <summary>
/// Finalizes report files.
/// Thread-safe.
/// </summary>
private void FinalizeReport(AddReportResult AddResult, DirectoryInfo DirInfo, FGenericCrashContext NewContext)
{
// Only remove if we added the report, otherwise leave all files to further investigation.
if (AddResult == AddReportResult.Added)
{
// Remove the report files as we're done with them.
CleanReport(DirInfo);
}
else if (AddResult == AddReportResult.Failed)
{
MoveReportToInvalid(NewContext.CrashDirectory, ReportQueueBase.GetSafeFilename(NewContext.GetAsFilename()));
}
else // AddResult == AddReportResult.Cancelled
{
// Remove report from index of completed reports and leave on disk
CrashReporterProcessServicer.ReportIndex.TryRemoveReport(NewContext.CrashDirectory);
}
}
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext)
{
// Use the latest MinidumpDiagnostics from the main branch.
string Win64BinariesDirectory = Path.Combine(Config.Default.DepotRoot, Config.Default.MDDBinariesFolderInDepot);
#if DEBUG
// Note: the debug executable must be built locally or synced from Perforce manually
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe");
#else
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe");
#endif
// Purge logs every 2048 processed crashes
string PurgeLogsDays = ProcessedReports % 2048 == 0 ? "2" : "-1";
FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion);
// Pass Windows variants (Win32/64) to MinidumpDiagnostics
string PlatformVariant = NewContext.PrimaryCrashProperties.PlatformName;
if (PlatformVariant != null && NewContext.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS"))
{
if (NewContext.PrimaryCrashProperties.PlatformFullName.Contains("Win32") ||
NewContext.PrimaryCrashProperties.PlatformFullName.Contains("32b"))
{
PlatformVariant = "Win32";
}
else if (NewContext.PrimaryCrashProperties.PlatformFullName.Contains("Win64") ||
NewContext.PrimaryCrashProperties.PlatformFullName.Contains("64b"))
{
PlatformVariant = "Win64";
}
}
List <string> MinidumpDiagnosticsParams = new List <string>
(
new string[]
{
"\"" + DiagnosticsPath + "\"",
"-BranchName=" + EngineVersion.Branch, // Backward compatibility
"-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility
"-GameName=" + NewContext.PrimaryCrashProperties.GameName,
"-EngineVersion=" + NewContext.PrimaryCrashProperties.EngineVersion,
"-PlatformName=" + NewContext.PrimaryCrashProperties.PlatformName,
"-PlatformVariantName=" + PlatformVariant,
"-bUsePDBCache=true",
"-Annotate",
"-SyncSymbols",
"-SyncMicrosoftSymbols",
"-unattended",
"-Log=" + NewContext.GetAsFilename() + "-backup-.log",
"-DepotIndex=" + Config.Default.DepotIndex,
"-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays,
"-LOGTIMESINCESTART"
}
);
LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null;
if (Environment.UserInteractive)
{
CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD;
}
else
{
MinidumpDiagnosticsParams.AddRange
(
new[]
{
"-buildmachine",
"-forcelogflush"
}
);
// Write some debugging message.
CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams));
}
Stopwatch WaitSW = Stopwatch.StartNew();
lock (MinidumpDiagnosticsLock)
{
Double WaitForLockTime = WaitSW.Elapsed.TotalSeconds;
LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray());
if (ReportParser.WaitForExit(MinidumpDiagnosticsTimeoutMilliseconds) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Timed out running MinidumpDiagnostics");
}
Double TotalMDDTime = WaitSW.Elapsed.TotalSeconds;
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("ProcessDumpFile: Total MDD exec time {0:N1}s (blocked for {1:N1}s)", TotalMDDTime, WaitForLockTime));
}
ReadDiagnosticsFile(NewContext);
}