/// <summary>
/// Builds MDD command line args, waits for a MDD task slot, runs MDD and blocks for the result. Writes diag text into DiagnosticsPath folder if successful.
/// </summary>
/// <param name="DiagnosticsPath">Path of the minidump file</param>
/// <param name="Context">The crash context</param>
/// <param name="ProcessorIndex">Processor thread index for logging purposes</param>
/// <returns>True, if successful</returns>
public bool Run(string DiagnosticsPath, FGenericCrashContext Context, int ProcessorIndex)
{
// Use the latest MinidumpDiagnostics from the main branch.
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.DepotRoot, Config.Default.DiskSpaceAlertPercent);
string Win64BinariesDirectory = Path.Combine(Config.Default.DepotRoot, Config.Default.MDDBinariesFolderInDepot);
#if DEBUG
// Note: the debug executable must be built locally or synced from Perforce manually
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe");
#else
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe");
#endif
// Don't purge logs
// TODO: make this clear to logs once a day or something (without letting MDD check on every run!)
string PurgeLogsDays = "-1";
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
// Pass Windows variants (Win32/64) to MinidumpDiagnostics
string PlatformVariant = Context.PrimaryCrashProperties.PlatformName;
if (PlatformVariant != null && Context.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS"))
{
if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win32") ||
Context.PrimaryCrashProperties.PlatformFullName.Contains("32b"))
{
PlatformVariant = "Win32";
}
else if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win64") ||
Context.PrimaryCrashProperties.PlatformFullName.Contains("64b"))
{
PlatformVariant = "Win64";
}
}
// Build the absolute log file path for MinidumpDiagnostics
string BaseFolder = CrashReporterProcessServicer.SymbolicatorLogFolder;
string SubFolder = DateTime.UtcNow.ToString("yyyy_MM_dd");
string Folder = Path.Combine(BaseFolder, SubFolder);
string AbsLogPath = Path.Combine(Folder, Context.GetAsFilename() + ".log");
Directory.CreateDirectory(Folder);
List<string> MinidumpDiagnosticsParams = new List<string>
(
new string[]
{
"\""+DiagnosticsPath+"\"",
"-BranchName=" + EngineVersion.Branch, // Backward compatibility
"-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility
"-GameName=" + Context.PrimaryCrashProperties.GameName,
"-EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion,
"-BuildVersion=" + (string.IsNullOrWhiteSpace(Context.PrimaryCrashProperties.BuildVersion) ?
string.Format("{0}-CL-{1}", EngineVersion.Branch, EngineVersion.Changelist).Replace('/', '+') :
Context.PrimaryCrashProperties.BuildVersion),
"-PlatformName=" + Context.PrimaryCrashProperties.PlatformName,
"-PlatformVariantName=" + PlatformVariant,
"-bUsePDBCache=true",
"-PDBCacheDepotRoot=" + Config.Default.DepotRoot,
"-PDBCachePath=" + Config.Default.MDDPDBCachePath,
"-PDBCacheSizeGB=" + Config.Default.MDDPDBCacheSizeGB,
"-PDBCacheMinFreeSpaceGB=" + Config.Default.MDDPDBCacheMinFreeSpaceGB,
"-PDBCacheFileDeleteDays=" + Config.Default.MDDPDBCacheFileDeleteDays,
"-MutexPDBCache",
"-PDBCacheLock=CrashReportProcessPDBCacheLock",
"-NoTrimCallstack",
"-SyncSymbols",
"-NoP4Symbols",
"-ForceUsePDBCache",
"-MutexSourceSync",
"-SourceSyncLock=CrashReportProcessSourceSyncLock",
"-SyncMicrosoftSymbols",
"-unattended",
"-AbsLog=" + AbsLogPath,
"-DepotIndex=" + Config.Default.DepotIndex,
"-P4User="******"-P4Client=" + Config.Default.P4Client,
"-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays + ",[LogFiles]:MaxLogFilesOnDisk=-1",
"-LOGTIMESINCESTART"
}
);
LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null;
if (Environment.UserInteractive)
{
CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD;
}
else
{
MinidumpDiagnosticsParams.AddRange
(
new[]
{
"-buildmachine",
"-forcelogflush"
}
);
// Write some debugging message.
CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams));
}
Task<bool> NewSymbolicatorTask = Task.FromResult(false);
Stopwatch WaitSW = Stopwatch.StartNew();
Double WaitForLockTime = 0.0;
lock (Tasks)
{
int TaskIdx = Task.WaitAny(Tasks);
Tasks[TaskIdx] = NewSymbolicatorTask = Task<bool>.Factory.StartNew(() =>
{
LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate,
MinidumpDiagnosticsParams.ToArray());
return ReportParser.WaitForExit(Config.Default.MDDTimeoutMinutes*1000*60) == EWaitResult.Ok;
});
WaitForLockTime = WaitSW.Elapsed.TotalSeconds;
}
NewSymbolicatorTask.Wait();
Double TotalMDDTime = WaitSW.Elapsed.TotalSeconds;
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Symbolicator.Run: Thread blocked for {0:N1}s then MDD ran for {1:N1}s", WaitForLockTime, TotalMDDTime - WaitForLockTime));
return NewSymbolicatorTask.Result;
}
/// <summary>
/// Create an Xml payload representing a new crash.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="bHasVideoFile">Whether the report contains a video file.</param>
/// <param name="bHasLog">Whether the report contains a log file.</param>
/// <param name="bHasMinidump">Whether the report contains a minidump.</param>
/// <returns>A string of Xml payload representing the newly found crash report.</returns>
private string CreateCrash( DirectoryInfo DirInfo, FGenericCrashContext NewContext, bool bHasVideoFile, bool bHasLog, bool bHasMinidump )
{
string XmlPayload = "";
try
{
FEngineVersion EngineVersion = new FEngineVersion( NewContext.PrimaryCrashProperties.EngineVersion );
// Create a new crash description for uploading
var NewCrash = new CrashDescription();
NewCrash.CrashType = NewContext.PrimaryCrashProperties.GetCrashType();
NewCrash.BranchName = EngineVersion.GetCleanedBranch();
NewCrash.GameName = NewContext.PrimaryCrashProperties.GameName;
NewCrash.Platform = NewContext.PrimaryCrashProperties.PlatformFullName;
NewCrash.EngineMode = NewContext.PrimaryCrashProperties.EngineMode;
NewCrash.EngineVersion = EngineVersion.VersionNumber;
NewCrash.BuildVersion = NewContext.PrimaryCrashProperties.BuildVersion;
NewCrash.CommandLine = NewContext.PrimaryCrashProperties.CommandLine;
NewCrash.BaseDir = NewContext.PrimaryCrashProperties.BaseDir;
NewCrash.bProcessorFailed = !string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.ProcessorFailedMessage);
NewCrash.Language = NewContext.PrimaryCrashProperties.AppDefaultLocale;
// // Create a locate and get the system language.
// int SystemLanguageCode = 0;
// int.TryParse( ReportData.SystemLanguage, out SystemLanguageCode );
// try
// {
// if( SystemLanguageCode > 0 )
// {
// CultureInfo SystemLanguageCI = new CultureInfo( SystemLanguageCode );
// NewCrash.SystemLanguage = SystemLanguageCI.Name;
// }
// }
// catch( System.Exception )
// {
// // Default to en-US
// NewCrash.SystemLanguage = "en-US";
// }
NewCrash.MachineGuid = NewContext.PrimaryCrashProperties.MachineId; // Valid for all kind of builds, previously only for UE4 releases.
NewCrash.UserName = NewContext.PrimaryCrashProperties.UserName; // Only valid for non-UE4 releases.
NewCrash.EpicAccountId = NewContext.PrimaryCrashProperties.EpicAccountId; // Only valid for UE4 releases.
NewCrash.CallStack = NewContext.PrimaryCrashProperties.GetCallstack();
NewCrash.SourceContext = NewContext.PrimaryCrashProperties.GetSourceContext();
NewCrash.ErrorMessage = NewContext.PrimaryCrashProperties.GetErrorMessage();
NewCrash.UserDescription = NewContext.PrimaryCrashProperties.GetUserDescription();
NewCrash.UserActivityHint = NewContext.PrimaryCrashProperties.UserActivityHint;
NewCrash.CrashGUID = NewContext.PrimaryCrashProperties.CrashGUID;
// Iterate through all files and find a file with the earliest date.
DateTime TimeOfCrash = DateTime.UtcNow;
foreach( var File in DirInfo.GetFiles() )
{
if( File.CreationTimeUtc < TimeOfCrash )
{
TimeOfCrash = File.CreationTimeUtc;
}
}
//NewCrash.TimeofCrash = NewContext.PrimaryCrashProperties.TimeofCrash;
NewCrash.TimeofCrash = TimeOfCrash;
NewCrash.bHasMiniDump = bHasMinidump;
NewCrash.bHasLog = bHasLog;
NewCrash.bHasVideo = bHasVideoFile;
NewCrash.BuiltFromCL = (int)EngineVersion.Changelist;
NewCrash.bAllowToBeContacted = NewContext.PrimaryCrashProperties.bAllowToBeContacted;
// Ignore any callstack that is shorter than expected, usually the callstack is invalid.
if( NewCrash.CallStack.Length <= CrashReporterConstants.MinCallstackDepth )
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "! BadStack: BuiltFromCL=" + string.Format( "{0,7}", NewContext.PrimaryCrashProperties.EngineVersion ) + " Path=" + NewContext.CrashDirectory );
NewContext.PrimaryCrashProperties.ProcessorFailedMessage = string.Format("Callstack was too small. {0} lines (minimum {1})", NewCrash.CallStack.Length, CrashReporterConstants.MinCallstackDepth);
NewContext.ToFile();
}
XmlPayload = XmlHandler.ToXmlString<CrashDescription>( NewCrash );
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "CreateCrash: " + Ex.ToString());
}
return XmlPayload;
}
// From CrashUpload.cpp
/*
struct FCompressedCrashFile : FNoncopyable
{
int32 CurrentFileIndex; // 4 bytes for file index
FString Filename; // 4 bytes for length + 260 bytes for char data
TArray<uint8> Filedata; // 4 bytes for length + N bytes for data
}
*/
/// <summary> Enqueues a new crash. </summary>
private bool EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml<FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return false;
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bContextDirty = false;
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
if (Context == null)
{
// Missing crash context
FReportData ReportData = new FReportData(MetaData, NewReportPath);
ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context);
bContextDirty = true;
}
else if (!Context.HasProcessedData())
{
// Missing data - try to get from WER metadata
FReportData ReportData = new FReportData(MetaData, NewReportPath);
GetErrorMessageFromMetadata(ReportData, Context);
bContextDirty = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return false;
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// Added data from WER, save to the crash context file.
if (bContextDirty)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
Context.ToFile();
}
else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
Context.ToFile();
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
Context.ToFile();
}
// Check static reports index for duplicated reports
// This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router)
// This can be safely discontinued when all crashes come from the same SQS
// This DOES NOT scale to multiple CRP instances
lock (ReportIndexLock)
{
if (!ReportIndex.TryAddNewReport(ReportName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return false;
}
}
if (ShouldDecimateNextReport())
{
CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return false;
}
lock (NewReportsLock)
{
NewCrashContexts.Enqueue(Context);
}
EnqueueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent);
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
return true;
}