private static void GetErrorMessageFromMetadata(FReportData ReportData, FGenericCrashContext CrashContext)
{
if (string.IsNullOrEmpty(CrashContext.PrimaryCrashProperties.ErrorMessage))
{
CrashContext.PrimaryCrashProperties.ErrorMessage = string.Join("\n", ReportData.ErrorMessage);
}
}
void ProcessDumpFile(string DumpPath, FGenericCrashContext NewContext)
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DumpPath);
if (TrySimulateSymbolication(NewContext))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Simulated symbolication (skipped MDD) " + NewContext.CrashDirectory);
}
else if (CrashReporterProcessServicer.Symbolicator.Run(DumpPath, NewContext, ProcessorIndex))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DumpPath);
ReadDiagnosticsFile(NewContext);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DumpPath);
}
if (string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.CallStack))
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationFailedEvent);
}
else
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationSucceededEvent);
}
}
/// <summary> Looks for the CrashContext.runtime-xml, if found, will return a new instance of the FGenericCrashContext. </summary>
private FGenericCrashContext FindCrashContext(string NewReportPath)
{
string CrashContextPath = Path.Combine(NewReportPath, FGenericCrashContext.CrashContextRuntimeXMLName);
bool bExist = File.Exists(CrashContextPath);
if (bExist)
{
return(FGenericCrashContext.FromFile(CrashContextPath));
}
return(null);
}
/// <summary>
/// Main processing thread.
/// </summary>
/// <remarks>All exceptions are caught and written to the event log.</remarks>
private void Init()
{
ProcessorTask = new Task(() =>
{
// parse out hte list of blacklisted games.
string[] GameNamesToBlacklist = (Config.Default.GameNamesToBlacklist == null ? "" : Config.Default.GameNamesToBlacklist).ToLowerInvariant().Split(',');
while (!CancelSource.IsCancellationRequested)
{
try
{
bool bIdle = true;
foreach (var Queue in Watcher.ReportQueues)
{
FGenericCrashContext NewContext = null;
if (Queue.TryDequeueReport(out NewContext))
{
bool bIsBlacklistedGame = GameNamesToBlacklist.Contains(NewContext.PrimaryCrashProperties.GameName.ToLowerInvariant());
// if it's blacklisted, skip it, else process it.
if (bIsBlacklistedGame)
{
CrashReporterProcessServicer.WriteEvent(string.Format("Discarding crash from blacklisted GameName '{0}'", NewContext.PrimaryCrashProperties.GameName));
// delete the report from disk since we don't care about it.
FinalizeReport(AddReportResult.Added, new DirectoryInfo(NewContext.CrashDirectory), NewContext);
}
else
{
ProcessReport(NewContext);
}
// The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest
bIdle = false;
break;
}
}
if (bIdle)
{
// Don't use the CPU if we don't need.
Thread.Sleep(1000);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessNewReports: " + Ex, Ex);
}
TickStatic(Watcher);
}
});
}
/// <summary>
/// Finalizes report files.
/// Thread-safe.
/// </summary>
private void FinalizeReport(bool bAdded, DirectoryInfo DirInfo, FGenericCrashContext NewContext)
{
// Only remove if we added the report, otherwise leave all files to further investigation.
if (bAdded)
{
// Remove the report files as we're done with them.
CleanReport(DirInfo);
}
else
{
MoveReportToInvalid(NewContext.CrashDirectory, NewContext.GetAsFilename());
}
}
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext)
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DiagnosticsPath);
if (CrashReporterProcessServicer.Symbolicator.Run(DiagnosticsPath, NewContext, ProcessorIndex))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DiagnosticsPath);
ReadDiagnosticsFile(NewContext);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DiagnosticsPath);
}
}
bool TrySimulateSymbolication(FGenericCrashContext NewContext)
{
// Check for crashes we don't want to attempt to symbolicate
if (!string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.ErrorMessage) &&
NewContext.PrimaryCrashProperties.ErrorMessage.Contains("Hang detected on"))
{
// Hangs from FThreadHeartBeat::Run() start wth "Hang detected on" and have a useful callstack already in the log
// Callstack and source from MDD will be useless so simulate a callstack that points the user to the log.
NewContext.PrimaryCrashProperties.CallStack = "See log for callstack from hang detection code()\nSee log for callstack from hang detection code()\nSee log for callstack from hang detection code()";
return(true);
}
return(false);
}
/// <summary> Reads callstack, source context and error message from the diagnostics file, if exists. </summary>
private void ReadDiagnosticsFile(FGenericCrashContext NewContext)
{
// Read the callstack and the source context from the diagnostics files.
string CrashDiagnosticsPath = Path.Combine(NewContext.CrashDirectory, CrashReporterConstants.DiagnosticsFileName);
if (File.Exists(CrashDiagnosticsPath))
{
NewContext.PrimaryCrashProperties.CallStack = FGenericCrashContext.EscapeXMLString(string.Join("\n", FReportData.GetCallStack(CrashDiagnosticsPath)));
NewContext.PrimaryCrashProperties.SourceContext = FGenericCrashContext.EscapeXMLString(string.Join("\n", FReportData.GetSourceContext(CrashDiagnosticsPath)));
if (NewContext.PrimaryCrashProperties.ErrorMessage.Length == 0)
{
NewContext.PrimaryCrashProperties.ErrorMessage = FGenericCrashContext.EscapeXMLString(string.Join("\n", FReportData.GetExceptionDescription(CrashDiagnosticsPath)));
}
}
}
/// <summary>If available, will read CrashContext.runtime-xml.</summary>
public void ReadCrashContextIfAvailable()
{
try
{
//\\epicgames.net\Root\Projects\Paragon\QA_CrashReports
bool bHasCrashContext = HasCrashContextFile();
if (bHasCrashContext)
{
//_CrashContext = FGenericCrashContext.FromFile(SitePath + GetCrashContextUrl());
_CrashContext = Settings.Default.DownloadFromS3 ?
FGenericCrashContext.FromUrl(GetCrashContextUrl()) :
null;
bool bTest = _CrashContext != null && !string.IsNullOrEmpty(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation);
if (bTest)
{
_bUseFullMinidumpPath = true;
//Some temporary code to redirect to the new file location for fulldumps for paragon.
//Consider removing this once fulldumps stop appearing in the old location.
if (_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation.ToLower()
.Contains("\\\\epicgames.net\\root\\dept\\gameqa\\paragon\\paragon_launcherCrashdumps"))
{
//Files from old versions of the client may end up in the old location. Check for files there first.
if (!System.IO.Directory.Exists(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation))
{
var suffix =
_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation.Substring("\\\\epicgames.net\\root\\dept\\gameqa\\paragon\\paragon_launcherCrashdumps".Length);
_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation = String.Format("\\\\epicgames.net\\Root\\Projects\\Paragon\\QA_CrashReports{0}", suffix);
//If the file doesn't exist in the new location either then don't use the full minidump path.
_bUseFullMinidumpPath =
System.IO.Directory.Exists(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation);
}
}
//End of temporary code.
FLogger.Global.WriteEvent("ReadCrashContextIfAvailable " + _CrashContext.PrimaryCrashProperties.FullCrashDumpLocation + " is " + _bUseFullMinidumpPath);
}
}
}
catch (Exception Ex)
{
Debug.WriteLine("Exception in ReadCrashContextIfAvailable: " + Ex.ToString());
}
}
/// <summary> Tries to dequeue a report from the list. </summary>
public bool TryDequeueReport(out FGenericCrashContext Context)
{
lock (NewReportsLock)
{
if (NewCrashContexts.Count > 0)
{
Context = NewCrashContexts.Dequeue();
DequeueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(QueueProcessingStartedEventName);
CrashReporterProcessServicer.WriteEvent(string.Format("- Dequeued: {0:N1}/minute BuiltFromCL={1,7} Path={2}", DequeueCounter.EventsPerSecond * 60, Context.PrimaryCrashProperties.EngineVersion, Context.CrashDirectory));
return(true);
}
}
Context = null;
return(false);
}
/// <summary>
/// A function to process a newly landed crash report.
/// </summary>
/// <param name="NewContext">An instance of the generic crash context</param>
private void ProcessReport(FGenericCrashContext NewContext)
{
Stopwatch ProcessReportSW = Stopwatch.StartNew();
// Just to verify that the report is still there.
DirectoryInfo DirInfo = new DirectoryInfo(NewContext.CrashDirectory);
if (!DirInfo.Exists)
{
// Something very odd happened.
CrashReporterProcessServicer.WriteEvent("ProcessReport: Directory not found: " + NewContext.CrashDirectory);
return;
}
double WaitTime = ReadProcessAddReport(DirInfo, NewContext);
// Make sure any log messages have been written to disk
CrashReporterProcessServicer.WriteEvent(string.Format("ProcessReportTime={0} WaitTime={1}", ProcessReportSW.Elapsed.TotalSeconds.ToString("0.0"), WaitTime.ToString("0.00")));
}
/// <summary> Tries to dequeue a report from the list. </summary>
public bool TryDequeueReport(out FGenericCrashContext Context)
{
lock ( NewReportsLock )
{
int LastIndex = NewCrashContexts.Count - 1;
if (LastIndex >= 0)
{
Context = NewCrashContexts[LastIndex];
NewCrashContexts.RemoveAt(LastIndex);
CrashReporterProcessServicer.WriteEvent("- Dequeued: BuiltFromCL=" + string.Format("{0,7}", Context.PrimaryCrashProperties.EngineVersion) + " Path=" + Context.CrashDirectory);
return(true);
}
else
{
Context = null;
return(false);
}
}
}
/// <summary>If available, will read CrashContext.runtime-xml.</summary>
public void ReadCrashContextIfAvailable()
{
try
{
bool bHasCrashContext = HasCrashContextFile();
if (bHasCrashContext)
{
CrashContext = FGenericCrashContext.FromFile(SitePath + GetCrashContextUrl());
bool bTest = CrashContext != null && !string.IsNullOrEmpty(CrashContext.PrimaryCrashProperties.FullCrashDumpLocation);
if (bTest)
{
bUseFullMinidumpPath = true; // System.IO.Directory.Exists( CrashContext.PrimaryCrashProperties.FullCrashDumpLocation ); Doesn't work, probably due to some permissions.
FLogger.Global.WriteEvent("ReadCrashContextIfAvailable " + CrashContext.PrimaryCrashProperties.FullCrashDumpLocation + " is " + bUseFullMinidumpPath);
}
}
}
catch (Exception Ex)
{
Debug.WriteLine("Exception in ReadCrashContextIfAvailable: " + Ex.ToString());
}
}
/// <summary>
/// Main processing thread.
/// </summary>
/// <remarks>All exceptions are caught and written to the event log.</remarks>
private void Init()
{
var Cancel = CancelSource.Token;
ProcessorTask = new Task(() =>
{
while (!Cancel.IsCancellationRequested)
{
try
{
bool bIdle = true;
foreach (var Queue in Watcher.ReportQueues)
{
FGenericCrashContext NewContext = null;
if (Queue.TryDequeueReport(out NewContext))
{
ProcessReport(NewContext);
// The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest
bIdle = false;
break;
}
}
if (bIdle)
{
// Don't use the CPU if we don't need.
Thread.Sleep(1000);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessNewReports: " + Ex, Ex);
}
TickStatic(Watcher);
}
});
}
// From CrashUpload.cpp
/*
* struct FCompressedCrashFile : FNoncopyable
* {
* int32 CurrentFileIndex; // 4 bytes for file index
* FString Filename; // 4 bytes for length + 260 bytes for char data
* TArray<uint8> Filedata; // 4 bytes for length + N bytes for data
* }
*/
/// <summary> Enqueues a new crash. </summary>
private bool EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return(false);
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bContextDirty = false;
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
if (Context == null)
{
// Missing crash context
FReportData ReportData = new FReportData(MetaData, NewReportPath);
ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context);
bContextDirty = true;
}
else if (!Context.HasProcessedData())
{
// Missing data - try to get from WER metadata
FReportData ReportData = new FReportData(MetaData, NewReportPath);
GetErrorMessageFromMetadata(ReportData, Context);
bContextDirty = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// Added data from WER, save to the crash context file.
if (bContextDirty)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
Context.ToFile();
}
else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
Context.ToFile();
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
Context.ToFile();
}
// Check static reports index for duplicated reports
// This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router)
// This can be safely discontinued when all crashes come from the same SQS
// This DOES NOT scale to multiple CRP instances
// ReportIndex can be disabled by leaving the path empty in config
if (!CrashReporterProcessServicer.ReportIndex.TryAddNewReport(ReportName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
if (ShouldDecimateNextReport())
{
CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
lock (NewReportsLock)
{
NewCrashContexts.Enqueue(Context);
}
EnqueueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent);
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
return(true);
}
/// <summary> Converts WER metadata xml file to the crash context. </summary>
private static void ConvertMetadataToCrashContext(FReportData ReportData, string NewReportPath, ref FGenericCrashContext OutCrashContext)
{
if (OutCrashContext == null)
{
OutCrashContext = new FGenericCrashContext();
}
OutCrashContext.PrimaryCrashProperties.CrashVersion = (int)ECrashDescVersions.VER_1_NewCrashFormat;
//OutCrashContext.PrimaryCrashProperties.ProcessId = 0; don't overwrite valid ids, zero is default anyway
OutCrashContext.PrimaryCrashProperties.CrashGUID = new DirectoryInfo(NewReportPath).Name;
// OutCrashContext.PrimaryCrashProperties.IsInternalBuild
// OutCrashContext.PrimaryCrashProperties.IsPerforceBuild
// OutCrashContext.PrimaryCrashProperties.IsSourceDistribution
// OutCrashContext.PrimaryCrashProperties.SecondsSinceStart
OutCrashContext.PrimaryCrashProperties.GameName = ReportData.GameName;
// OutCrashContext.PrimaryCrashProperties.ExecutableName
// OutCrashContext.PrimaryCrashProperties.BuildConfiguration
// OutCrashContext.PrimaryCrashProperties.PlatformName
// OutCrashContext.PrimaryCrashProperties.PlatformNameIni
OutCrashContext.PrimaryCrashProperties.PlatformFullName = ReportData.Platform;
OutCrashContext.PrimaryCrashProperties.EngineMode = ReportData.EngineMode;
OutCrashContext.PrimaryCrashProperties.EngineVersion = ReportData.GetEngineVersion();
OutCrashContext.PrimaryCrashProperties.BuildVersion = ReportData.BuildVersion;
OutCrashContext.PrimaryCrashProperties.CommandLine = ReportData.CommandLine;
// OutCrashContext.PrimaryCrashProperties.LanguageLCID
// Create a locate and get the language.
int LanguageCode = 0;
int.TryParse(ReportData.Language, out LanguageCode);
try
{
if (LanguageCode > 0)
{
CultureInfo LanguageCI = new CultureInfo(LanguageCode);
OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = LanguageCI.Name;
}
}
catch (Exception)
{
OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = null;
}
if (string.IsNullOrEmpty(OutCrashContext.PrimaryCrashProperties.AppDefaultLocale))
{
// Default to en-US
OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = "en-US";
}
// OutCrashContext.PrimaryCrashProperties.IsUE4Release
string WERUserName = ReportData.UserName;
if (!string.IsNullOrEmpty(WERUserName) || string.IsNullOrEmpty(OutCrashContext.PrimaryCrashProperties.UserName))
{
OutCrashContext.PrimaryCrashProperties.UserName = WERUserName;
}
OutCrashContext.PrimaryCrashProperties.BaseDir = ReportData.BaseDir;
// OutCrashContext.PrimaryCrashProperties.RootDir
OutCrashContext.PrimaryCrashProperties.MachineId = ReportData.MachineId;
OutCrashContext.PrimaryCrashProperties.LoginId = ReportData.LoginId;
if (string.IsNullOrWhiteSpace(OutCrashContext.PrimaryCrashProperties.MachineId))
{
// Set MachineId from LoginId if the metadata is too new to contain MachineId
OutCrashContext.PrimaryCrashProperties.MachineId = OutCrashContext.PrimaryCrashProperties.LoginId;
}
OutCrashContext.PrimaryCrashProperties.EpicAccountId = ReportData.EpicAccountId;
// OutCrashContext.PrimaryCrashProperties.CallStack
// OutCrashContext.PrimaryCrashProperties.SourceContext
OutCrashContext.PrimaryCrashProperties.UserDescription = string.Join("\n", ReportData.UserDescription);
// OutCrashContext.PrimaryCrashProperties.CrashDumpMode
// OutCrashContext.PrimaryCrashProperties.Misc.NumberOfCores
// OutCrashContext.PrimaryCrashProperties.Misc.NumberOfCoresIncludingHyperthreads
// OutCrashContext.PrimaryCrashProperties.Misc.Is64bitOperatingSystem
// OutCrashContext.PrimaryCrashProperties.Misc.CPUVendor
// OutCrashContext.PrimaryCrashProperties.Misc.CPUBrand
// OutCrashContext.PrimaryCrashProperties.Misc.PrimaryGPUBrand
// OutCrashContext.PrimaryCrashProperties.Misc.OSVersionMajor
// OutCrashContext.PrimaryCrashProperties.Misc.OSVersionMinor
// OutCrashContext.PrimaryCrashProperties.Misc.AppDiskTotalNumberOfBytes
// OutCrashContext.PrimaryCrashProperties.Misc.AppDiskNumberOfFreeBytes
// OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysical
// OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalVirtual
// OutCrashContext.PrimaryCrashProperties.MemoryStats.PageSize
// OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysicalGB
// OutCrashContext.PrimaryCrashProperties.MemoryStats.AvailablePhysical
// OutCrashContext.PrimaryCrashProperties.MemoryStats.AvailableVirtual
// OutCrashContext.PrimaryCrashProperties.MemoryStats.UsedPhysical
// OutCrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedPhysical
// OutCrashContext.PrimaryCrashProperties.MemoryStats.UsedVirtual
// OutCrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedVirtual
// OutCrashContext.PrimaryCrashProperties.MemoryStats.bIsOOM
// OutCrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationSize
// OutCrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationAlignment
OutCrashContext.PrimaryCrashProperties.TimeofCrash = new DateTime(ReportData.Ticks);
OutCrashContext.PrimaryCrashProperties.bAllowToBeContacted = ReportData.AllowToBeContacted;
OutCrashContext.PrimaryCrashProperties.DeploymentName = ReportData.DeploymentName;
OutCrashContext.PrimaryCrashProperties.IsEnsure = ReportData.IsEnsure;
OutCrashContext.PrimaryCrashProperties.IsAssert = ReportData.IsAssert;
OutCrashContext.PrimaryCrashProperties.CrashType = ReportData.CrashType;
GetErrorMessageFromMetadata(ReportData, OutCrashContext);
}
/// <summary>
/// A function to add a report to the database, and rename the relevant files.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="LogFileName">The file name of the log file in the report.</param>
/// <param name="DumpFileName">The file name of the minidump in the report.</param>
/// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param>
private AddReportResult AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName)
{
try
{
Stopwatch AddReportTime = Stopwatch.StartNew();
// Create an XML representation of a crash
string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null);
if (CrashDetails == "")
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "! CreateCrash no payload: Path=" + NewContext.CrashDirectory);
return(AddReportResult.Failed);
}
// Upload the crash to the database, and retrieve the new row id
int ReportID = UploadCrash(CrashDetails);
if (ReportID <= 0)
{
if (CancelSource.IsCancellationRequested)
{
// Cancelled upload
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "CancelUpload: Path=" + NewContext.CrashDirectory);
return(AddReportResult.Cancelled);
}
else
{
// Upload failed
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "! NoUpload: Path=" + NewContext.CrashDirectory);
return(AddReportResult.Failed);
}
}
bool bToS3 = Config.Default.CrashFilesToAWS && CrashReporterProcessServicer.OutputAWS.IsS3Valid;
bool bToDisk = Config.Default.CrashFilesToDisk;
// Use the row id to name and move the files the way the web site requires
string IDThenUnderscore = string.Format("{0}_", ReportID);
int ReportIDSegment = (ReportID / 10000) * 10000;
string S3IDPrefix = string.Format("/{0}/{1}/{1}_", ReportIDSegment, ReportID);
string DestinationFolder = Path.Combine(Config.Default.ProcessedReports, IDThenUnderscore);
Stopwatch WriteToS3Timer = new Stopwatch();
Stopwatch WriteToDiskTimer = new Stopwatch();
if (bToDisk)
{
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(DestinationFolder, Config.Default.DiskSpaceAlertPercent);
}
// Save log file
if (LogFileName != null)
{
LogFileName = Path.Combine(DirInfo.FullName, LogFileName);
FileInfo LogInfo = new FileInfo(LogFileName);
if (LogInfo.Exists)
{
if (bToS3)
{
WriteToS3Timer.Start();
UploadFileToS3(LogInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + "Launch.log", Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix);
WriteToS3Timer.Stop();
}
if (bToDisk)
{
WriteToDiskTimer.Start();
LogInfo.MoveTo(DestinationFolder + "Launch.log");
WriteToDiskTimer.Stop();
}
}
}
// Save crash context file
string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName);
FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName);
if (CrashContextInfo.Exists)
{
if (bToS3)
{
WriteToS3Timer.Start();
UploadFileToS3(CrashContextInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + FGenericCrashContext.CrashContextRuntimeXMLName, false);
WriteToS3Timer.Stop();
}
if (bToDisk)
{
WriteToDiskTimer.Start();
CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName);
WriteToDiskTimer.Stop();
}
}
if (DumpFileName != null)
{
DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName);
FileInfo DumpInfo = new FileInfo(DumpFileName);
if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/)
{
if (bToS3)
{
WriteToS3Timer.Start();
UploadFileToS3(DumpInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + "MiniDump.dmp", Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix);
WriteToS3Timer.Stop();
}
if (bToDisk)
{
WriteToDiskTimer.Start();
DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp");
WriteToDiskTimer.Stop();
}
}
}
// Move the video (if it exists) to an alternate store
if (VideoFileName != null)
{
DestinationFolder = Path.Combine(Config.Default.ProcessedVideos, IDThenUnderscore);
VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName);
FileInfo VideoInfo = new FileInfo(VideoFileName);
if (VideoInfo.Exists)
{
if (bToS3)
{
WriteToS3Timer.Start();
UploadFileToS3(VideoInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + CrashReporterConstants.VideoFileName, Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix);
WriteToS3Timer.Stop();
}
if (bToDisk)
{
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(DestinationFolder, Config.Default.DiskSpaceAlertPercent);
WriteToDiskTimer.Start();
VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName);
WriteToDiskTimer.Stop();
}
}
}
string TimeTakenString = string.Empty;
if (bToS3)
{
TimeTakenString = string.Format("S3UploadTime={0:F1} ", WriteToS3Timer.Elapsed.TotalSeconds);
}
if (bToDisk)
{
TimeTakenString += string.Format("DiskMoveTime={0:F1} ", WriteToDiskTimer.Elapsed.TotalSeconds);
}
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} AddReport: ReportID={1,8} {2}Path={3}", ProcessorIndex, ReportID, TimeTakenString, NewContext.CrashDirectory));
UpdateProcessedReports();
WebAddCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingSucceededEvent);
double Ratio = (double)WebAddCounter.TotalEvents / (double)ProcessedReportCount * 100;
double AddedPerDay = (double)WebAddCounter.TotalEvents / Timer.Elapsed.TotalDays;
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) +
string.Format(
"AddReport: Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4} AddedPerMinute={5:N1}", (int)Ratio,
ProcessedReportCount, WebAddCounter.TotalEvents, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay, WebAddCounter.EventsPerSecond * 60));
return(AddReportResult.Added);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "AddReport: " + DirInfo.Name + "\n\n" + Ex, Ex);
}
return(AddReportResult.Failed);
}
/// <summary> Converts WER metadata xml file to the crash context. </summary>
private FGenericCrashContext ConvertMetadataToCrashContext(WERReportMetadata Metadata, string NewReportPath)
{
FGenericCrashContext CrashContext = new FGenericCrashContext();
FReportData ReportData = new FReportData(Metadata, NewReportPath);
CrashContext.PrimaryCrashProperties.CrashVersion = (int)ECrashDescVersions.VER_1_NewCrashFormat;
CrashContext.PrimaryCrashProperties.ProcessId = 0;
CrashContext.PrimaryCrashProperties.CrashGUID = new DirectoryInfo(NewReportPath).Name;
// CrashContext.PrimaryCrashProperties.IsInternalBuild
// CrashContext.PrimaryCrashProperties.IsPerforceBuild
// CrashContext.PrimaryCrashProperties.IsSourceDistribution
// CrashContext.PrimaryCrashProperties.SecondsSinceStart
CrashContext.PrimaryCrashProperties.GameName = ReportData.GameName;
// CrashContext.PrimaryCrashProperties.ExecutableName
// CrashContext.PrimaryCrashProperties.BuildConfiguration
// CrashContext.PrimaryCrashProperties.PlatformName
// CrashContext.PrimaryCrashProperties.PlatformNameIni
CrashContext.PrimaryCrashProperties.PlatformFullName = ReportData.Platform;
CrashContext.PrimaryCrashProperties.EngineMode = ReportData.EngineMode;
CrashContext.PrimaryCrashProperties.EngineVersion = ReportData.GetEngineVersion();
CrashContext.PrimaryCrashProperties.CommandLine = ReportData.CommandLine;
// CrashContext.PrimaryCrashProperties.LanguageLCID
// Create a locate and get the language.
int LanguageCode = 0;
int.TryParse(ReportData.Language, out LanguageCode);
try
{
if (LanguageCode > 0)
{
CultureInfo LanguageCI = new CultureInfo(LanguageCode);
CrashContext.PrimaryCrashProperties.AppDefaultLocale = LanguageCI.Name;
}
}
catch (System.Exception)
{
// Default to en-US
CrashContext.PrimaryCrashProperties.AppDefaultLocale = "en-US";
}
// CrashContext.PrimaryCrashProperties.IsUE4Release
CrashContext.PrimaryCrashProperties.UserName = ReportData.UserName;
CrashContext.PrimaryCrashProperties.BaseDir = ReportData.BaseDir;
// CrashContext.PrimaryCrashProperties.RootDir
CrashContext.PrimaryCrashProperties.MachineId = ReportData.MachineId;
CrashContext.PrimaryCrashProperties.EpicAccountId = ReportData.EpicAccountId;
// CrashContext.PrimaryCrashProperties.CallStack
// CrashContext.PrimaryCrashProperties.SourceContext
CrashContext.PrimaryCrashProperties.UserDescription = string.Join("\n", ReportData.UserDescription);
CrashContext.PrimaryCrashProperties.ErrorMessage = string.Join("\n", ReportData.ErrorMessage);
// CrashContext.PrimaryCrashProperties.CrashDumpMode
// CrashContext.PrimaryCrashProperties.Misc.NumberOfCores
// CrashContext.PrimaryCrashProperties.Misc.NumberOfCoresIncludingHyperthreads
// CrashContext.PrimaryCrashProperties.Misc.Is64bitOperatingSystem
// CrashContext.PrimaryCrashProperties.Misc.CPUVendor
// CrashContext.PrimaryCrashProperties.Misc.CPUBrand
// CrashContext.PrimaryCrashProperties.Misc.PrimaryGPUBrand
// CrashContext.PrimaryCrashProperties.Misc.OSVersionMajor
// CrashContext.PrimaryCrashProperties.Misc.OSVersionMinor
// CrashContext.PrimaryCrashProperties.Misc.AppDiskTotalNumberOfBytes
// CrashContext.PrimaryCrashProperties.Misc.AppDiskNumberOfFreeBytes
// CrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysical
// CrashContext.PrimaryCrashProperties.MemoryStats.TotalVirtual
// CrashContext.PrimaryCrashProperties.MemoryStats.PageSize
// CrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysicalGB
// CrashContext.PrimaryCrashProperties.MemoryStats.AvailablePhysical
// CrashContext.PrimaryCrashProperties.MemoryStats.AvailableVirtual
// CrashContext.PrimaryCrashProperties.MemoryStats.UsedPhysical
// CrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedPhysical
// CrashContext.PrimaryCrashProperties.MemoryStats.UsedVirtual
// CrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedVirtual
// CrashContext.PrimaryCrashProperties.MemoryStats.bIsOOM
// CrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationSize
// CrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationAlignment
CrashContext.PrimaryCrashProperties.TimeofCrash = new DateTime(ReportData.Ticks);
CrashContext.PrimaryCrashProperties.bAllowToBeContacted = ReportData.AllowToBeContacted;
return(CrashContext);
}
double ReadProcessAddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext)
{
try
{
string DumpFileName = null;
string LogFileName = null;
string DiagnosticsFileName = "";
string VideoFileName = null;
string CrashContextName = "";
foreach (FileInfo Info in DirInfo.GetFiles())
{
switch (Info.Extension.ToLower())
{
case ".avi":
case ".mp4":
VideoFileName = Info.Name;
break;
case ".runtime-xml":
CrashContextName = Info.Name;
break;
case ".log":
LogFileName = Info.Name;
break;
case ".txt":
if (Info.Name == CrashReporterConstants.DiagnosticsFileName)
{
DiagnosticsFileName = Info.Name;
ReadDiagnosticsFile(NewContext);
}
break;
case ".dmp":
case ".mdmp":
DumpFileName = Info.Name;
// Check to see if this has been processed locally
FileInfo DiagnosticsInfo = new FileInfo(Path.Combine(DirInfo.FullName, CrashReporterConstants.DiagnosticsFileName));
if (!DiagnosticsInfo.Exists && !NewContext.HasProcessedData())
{
ProcessDumpFile(Info.FullName, NewContext);
}
// Check to see if a new one has been created
DiagnosticsInfo.Refresh();
if (DiagnosticsInfo.Exists)
{
DiagnosticsFileName = DiagnosticsInfo.Name;
}
break;
default:
break;
}
}
// Check if the new context has processed data.
if (NewContext.HasProcessedData())
{
Stopwatch WaitSW = Stopwatch.StartNew();
// Wait for previous task, should not really happen.
if (AddReportTask != null)
{
AddReportTask.Wait();
}
double WaitTime = WaitSW.Elapsed.TotalSeconds;
//bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName );
// Save/update crash context to the file.
NewContext.ToFile();
AddReportTask = Task.Factory.StartNew(() =>
{
bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName);
FinalizeReport(bAdded, DirInfo, NewContext);
});
return(WaitTime);
}
else
{
CrashReporterProcessServicer.WriteFailure("! Invalid : BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory);
FinalizeReport(false, DirInfo, NewContext);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("ProcessReport: " + NewContext.CrashDirectory + "\n\n: " + Ex.ToString());
}
return(0.0);
}
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext)
{
// Use the latest MinidumpDiagnostics from the main branch.
string Win64BinariesDirectory = Path.Combine(Config.Default.DepotRoot, Config.Default.MDDBinariesFolderInDepot);
#if DEBUG
// Note: the debug executable must be built locally or synced from Perforce manually
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe");
#else
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe");
#endif
// Purge logs every 2048 processed crashes
string PurgeLogsDays = ProcessedReports % 2048 == 0 ? "2" : "-1";
FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion);
// Pass Windows variants (Win32/64) to MinidumpDiagnostics
string PlatformVariant = NewContext.PrimaryCrashProperties.PlatformName;
if (PlatformVariant != null && NewContext.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS"))
{
if (NewContext.PrimaryCrashProperties.PlatformFullName.Contains("Win32") ||
NewContext.PrimaryCrashProperties.PlatformFullName.Contains("32b"))
{
PlatformVariant = "Win32";
}
else if (NewContext.PrimaryCrashProperties.PlatformFullName.Contains("Win64") ||
NewContext.PrimaryCrashProperties.PlatformFullName.Contains("64b"))
{
PlatformVariant = "Win64";
}
}
List <string> MinidumpDiagnosticsParams = new List <string>
(
new string[]
{
"\"" + DiagnosticsPath + "\"",
"-BranchName=" + EngineVersion.Branch, // Backward compatibility
"-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility
"-GameName=" + NewContext.PrimaryCrashProperties.GameName,
"-EngineVersion=" + NewContext.PrimaryCrashProperties.EngineVersion,
"-PlatformName=" + NewContext.PrimaryCrashProperties.PlatformName,
"-PlatformVariantName=" + PlatformVariant,
"-bUsePDBCache=true",
"-Annotate",
"-SyncSymbols",
"-SyncMicrosoftSymbols",
"-unattended",
"-Log=" + NewContext.GetAsFilename() + "-backup-.log",
"-DepotIndex=" + Config.Default.DepotIndex,
"-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays,
"-LOGTIMESINCESTART"
}
);
LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null;
if (Environment.UserInteractive)
{
CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD;
}
else
{
MinidumpDiagnosticsParams.AddRange
(
new[]
{
"-buildmachine",
"-forcelogflush"
}
);
// Write some debugging message.
CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams));
}
Stopwatch WaitSW = Stopwatch.StartNew();
lock (MinidumpDiagnosticsLock)
{
Double WaitForLockTime = WaitSW.Elapsed.TotalSeconds;
LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray());
if (ReportParser.WaitForExit(MinidumpDiagnosticsTimeoutMilliseconds) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Timed out running MinidumpDiagnostics");
}
Double TotalMDDTime = WaitSW.Elapsed.TotalSeconds;
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("ProcessDumpFile: Total MDD exec time {0:N1}s (blocked for {1:N1}s)", TotalMDDTime, WaitForLockTime));
}
ReadDiagnosticsFile(NewContext);
}
/// <summary>
/// A function to add a report to the database, and rename the relevant files.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="LogFileName">The file name of the log file in the report.</param>
/// <param name="DumpFileName">The file name of the minidump in the report.</param>
/// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param>
private bool AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName)
{
try
{
Stopwatch AddReportTime = Stopwatch.StartNew();
// Create an XML representation of a crash
string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null);
if (CrashDetails == "")
{
CrashReporterProcessServicer.WriteFailure("! NoDetail: Path=" + NewContext.CrashDirectory);
return(false);
}
// Upload the crash to the database, and retrieve the new row id
int ReportID = UploadCrash(CrashDetails);
if (ReportID <= 0)
{
CrashReporterProcessServicer.WriteFailure("! NoUpload: Path=" + NewContext.CrashDirectory);
return(false);
}
string IDThenUnderscore = string.Format("{0}_", ReportID);
// Use the row id to name and move the files the way the web site requires
string DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedReports, IDThenUnderscore);
// Move report files to crash reporter file store
if (LogFileName != null)
{
LogFileName = Path.Combine(DirInfo.FullName, LogFileName);
FileInfo LogInfo = new FileInfo(LogFileName);
if (LogInfo.Exists)
{
LogInfo.MoveTo(DestinationFolder + "Launch.log");
}
}
string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName);
FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName);
if (CrashContextInfo.Exists)
{
CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName);
}
// WERMetaDataName = Path.Combine(DirInfo.FullName, WERMetaDataName);
// FileInfo MetaInfo = new FileInfo(WERMetaDataName);
// if (MetaInfo.Exists)
// {
// MetaInfo.MoveTo(DestinationFolder + "WERMeta.xml");
// }
if (DumpFileName != null)
{
DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName);
FileInfo DumpInfo = new FileInfo(DumpFileName);
if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/)
{
DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp");
}
}
// DiagnosticsFileName = Path.Combine(DirInfo.FullName, DiagnosticsFileName);
// FileInfo DiagnosticsInfo = new FileInfo(DiagnosticsFileName);
// if (DiagnosticsInfo.Exists)
// {
// DiagnosticsInfo.MoveTo(DestinationFolder + CrashReporterConstants.DiagnosticsFileName);
// }
// Move the video (if it exists) to an alternate store
if (VideoFileName != null)
{
DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedVideos, IDThenUnderscore);
VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName);
FileInfo VideoInfo = new FileInfo(VideoFileName);
if (VideoInfo.Exists)
{
VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName);
}
}
CrashReporterProcessServicer.WriteEvent("# WebAdded: ReportID =" + string.Format("{0,7}", ReportID) + " Path=" + NewContext.CrashDirectory);
UpdateProcessedReports();
WebAddedReports++;
double Ratio = (double)WebAddedReports / (double)ProcessedReports * 100;
double AddedPerDay = (double)WebAddedReports / Timer.Elapsed.TotalDays;
CrashReporterProcessServicer.WriteEvent(string.Format("Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4}", (int)Ratio, ProcessedReports, WebAddedReports, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay));
return(true);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("AddReport: " + DirInfo.Name + "\n\n" + Ex.ToString());
}
return(false);
}
/// <summary>
/// Create an Xml payload representing a new crash.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="bHasVideoFile">Whether the report contains a video file.</param>
/// <param name="bHasLog">Whether the report contains a log file.</param>
/// <param name="bHasMinidump">Whether the report contains a minidump.</param>
/// <returns>A string of Xml payload representing the newly found crash report.</returns>
private string CreateCrash(DirectoryInfo DirInfo, FGenericCrashContext NewContext, bool bHasVideoFile, bool bHasLog, bool bHasMinidump)
{
string XmlPayload = "";
try
{
FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion);
// Create a new crash description for uploading
CrashDescription NewCrash = new CrashDescription();
NewCrash.BranchName = EngineVersion.GetCleanedBranch();
NewCrash.GameName = NewContext.PrimaryCrashProperties.GameName;
NewCrash.Platform = NewContext.PrimaryCrashProperties.PlatformFullName;
NewCrash.EngineMode = NewContext.PrimaryCrashProperties.EngineMode;
NewCrash.BuildVersion = EngineVersion.VersionNumber;
NewCrash.CommandLine = NewContext.PrimaryCrashProperties.CommandLine;
NewCrash.BaseDir = NewContext.PrimaryCrashProperties.BaseDir;
NewCrash.Language = NewContext.PrimaryCrashProperties.AppDefaultLocale;
// // Create a locate and get the system language.
// int SystemLanguageCode = 0;
// int.TryParse( ReportData.SystemLanguage, out SystemLanguageCode );
// try
// {
// if( SystemLanguageCode > 0 )
// {
// CultureInfo SystemLanguageCI = new CultureInfo( SystemLanguageCode );
// NewCrash.SystemLanguage = SystemLanguageCI.Name;
// }
// }
// catch( System.Exception )
// {
// // Default to en-US
// NewCrash.SystemLanguage = "en-US";
// }
NewCrash.MachineGuid = NewContext.PrimaryCrashProperties.MachineId; // Valid for all kind of builds, previously only for UE4 releases.
NewCrash.UserName = NewContext.PrimaryCrashProperties.UserName; // Only valid for non-UE4 releases.
NewCrash.EpicAccountId = NewContext.PrimaryCrashProperties.EpicAccountId; // Only valid for UE4 releases.
NewCrash.CallStack = NewContext.PrimaryCrashProperties.GetCallstack();
NewCrash.SourceContext = NewContext.PrimaryCrashProperties.GetSourceContext();
NewCrash.ErrorMessage = NewContext.PrimaryCrashProperties.GetErrorMessage();
NewCrash.UserDescription = NewContext.PrimaryCrashProperties.GetUserDescription();
// Iterate through all files and find a file with the earliest date.
DateTime TimeOfCrash = DateTime.UtcNow;
foreach (var File in DirInfo.GetFiles())
{
if (File.CreationTimeUtc < TimeOfCrash)
{
TimeOfCrash = File.CreationTimeUtc;
}
}
//NewCrash.TimeofCrash = NewContext.PrimaryCrashProperties.TimeofCrash;
NewCrash.TimeofCrash = TimeOfCrash;
NewCrash.bHasMiniDump = bHasMinidump;
NewCrash.bHasLog = bHasLog;
NewCrash.bHasVideo = bHasVideoFile;
NewCrash.BuiltFromCL = (int)EngineVersion.Changelist;
NewCrash.bAllowToBeContacted = NewContext.PrimaryCrashProperties.bAllowToBeContacted;
// Ignore any callstack that is shorter than expected, usually the callstack is invalid.
if (NewCrash.CallStack.Length <= CrashReporterConstants.MinCallstackDepth)
{
CrashReporterProcessServicer.WriteFailure("! BadStack: BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory);
}
else
{
XmlPayload = XmlHandler.ToXmlString <CrashDescription>(NewCrash);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CreateCrash: " + Ex.ToString());
}
return(XmlPayload);
}
/// <summary>
/// Finalizes report files.
/// Thread-safe.
/// </summary>
private void FinalizeReport(AddReportResult AddResult, DirectoryInfo DirInfo, FGenericCrashContext NewContext)
{
// Only remove if we added the report, otherwise leave all files to further investigation.
if (AddResult == AddReportResult.Added)
{
// Remove the report files as we're done with them.
CleanReport(DirInfo);
}
else if (AddResult == AddReportResult.Failed)
{
MoveReportToInvalid(NewContext.CrashDirectory, ReportQueueBase.GetSafeFilename(NewContext.GetAsFilename()));
}
else // AddResult == AddReportResult.Cancelled
{
// Remove report from index of completed reports and leave on disk
CrashReporterProcessServicer.ReportIndex.TryRemoveReport(NewContext.CrashDirectory);
}
}
/// <summary>
/// Builds MDD command line args, waits for a MDD task slot, runs MDD and blocks for the result. Writes diag text into DiagnosticsPath folder if successful.
/// </summary>
/// <param name="DiagnosticsPath">Path of the minidump file</param>
/// <param name="Context">The crash context</param>
/// <param name="ProcessorIndex">Processor thread index for logging purposes</param>
/// <returns>True, if successful</returns>
public bool Run(string DiagnosticsPath, FGenericCrashContext Context, int ProcessorIndex)
{
if (!File.Exists(Config.Default.MDDExecutablePath))
{
CrashReporterProcessServicer.WriteEvent("Symbolicator.Run() file not found " + Config.Default.MDDExecutablePath);
return(false);
}
// Use MinidumpDiagnostics from MDDExecutablePath.
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.MDDExecutablePath, Config.Default.DiskSpaceAlertPercent);
// Don't purge logs
// TODO: make this clear to logs once a day or something (without letting MDD check on every run!)
string PurgeLogsDays = "-1";
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
// Pass Windows variants (Win32/64) to MinidumpDiagnostics
string PlatformVariant = Context.PrimaryCrashProperties.PlatformName;
if (PlatformVariant != null && Context.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS"))
{
if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win32") ||
Context.PrimaryCrashProperties.PlatformFullName.Contains("32b"))
{
PlatformVariant = "Win32";
}
else if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win64") ||
Context.PrimaryCrashProperties.PlatformFullName.Contains("64b"))
{
PlatformVariant = "Win64";
}
}
// Build the absolute log file path for MinidumpDiagnostics
string BaseFolder = CrashReporterProcessServicer.SymbolicatorLogFolder;
DateTime WriteTime = DateTime.UtcNow;
string DateFolder = WriteTime.ToString("yyyy_MM_dd");
string HourFolder = WriteTime.ToString("HH");
string Folder = Path.Combine(BaseFolder, DateFolder, HourFolder);
string AbsLogPath = Path.Combine(Folder, Context.GetAsFilename() + ".log");
Directory.CreateDirectory(Folder);
List <string> MinidumpDiagnosticsParams = new List <string>
(
new string[]
{
"\"" + DiagnosticsPath + "\"",
"-BranchName=" + EngineVersion.Branch, // Backward compatibility
"-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility
"-GameName=" + Context.PrimaryCrashProperties.GameName,
"-EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion,
"-BuildVersion=" + (string.IsNullOrWhiteSpace(Context.PrimaryCrashProperties.BuildVersion) ?
string.Format("{0}-CL-{1}", EngineVersion.Branch, EngineVersion.Changelist).Replace('/', '+') :
Context.PrimaryCrashProperties.BuildVersion),
"-PlatformName=" + Context.PrimaryCrashProperties.PlatformName,
"-PlatformVariantName=" + PlatformVariant,
"-bUsePDBCache=true",
"-PDBCacheDepotRoot=" + Config.Default.DepotRoot,
"-PDBCachePath=" + Config.Default.MDDPDBCachePath,
"-PDBCacheSizeGB=" + Config.Default.MDDPDBCacheSizeGB,
"-PDBCacheMinFreeSpaceGB=" + Config.Default.MDDPDBCacheMinFreeSpaceGB,
"-PDBCacheFileDeleteDays=" + Config.Default.MDDPDBCacheFileDeleteDays,
"-MutexPDBCache",
"-PDBCacheLock=CrashReportProcessPDBCacheLock",
"-NoTrimCallstack",
"-SyncSymbols",
"-NoP4Symbols",
"-ForceUsePDBCache",
"-MutexSourceSync",
"-SourceSyncLock=CrashReportProcessSourceSyncLock",
"-SyncMicrosoftSymbols",
"-unattended",
"-AbsLog=" + AbsLogPath,
"-DepotIndex=" + Config.Default.DepotIndex,
"-P4User="******"-P4Client=" + Config.Default.P4Client,
"-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays + ",[LogFiles]:MaxLogFilesOnDisk=-1",
"-LOGTIMESINCESTART"
}
);
LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null;
if (Environment.UserInteractive)
{
CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD;
}
else
{
MinidumpDiagnosticsParams.AddRange
(
new[]
{
"-buildmachine"
}
);
// Write some debugging message.
CrashReporterProcessServicer.WriteMDD("MinidumpDiagnostics Params: " + String.Join(" ", MinidumpDiagnosticsParams));
}
Task <bool> NewSymbolicatorTask = Task.FromResult(false);
Stopwatch WaitSW = Stopwatch.StartNew();
double WaitForLockTime = 0.0;
lock (Tasks)
{
int TaskIdx = Task.WaitAny(Tasks);
Tasks[TaskIdx] = NewSymbolicatorTask = Task <bool> .Factory.StartNew(() =>
{
LaunchProcess ReportParser = new LaunchProcess(Config.Default.MDDExecutablePath, Path.GetDirectoryName(Config.Default.MDDExecutablePath), CaptureMessageDelegate,
MinidumpDiagnosticsParams.ToArray());
return(ReportParser.WaitForExit(Config.Default.MDDTimeoutMinutes * 1000 * 60) == EWaitResult.Ok);
});
WaitForLockTime = WaitSW.Elapsed.TotalSeconds;
}
NewSymbolicatorTask.Wait();
double TotalMDDTime = WaitSW.Elapsed.TotalSeconds;
double MDDRunTime = TotalMDDTime - WaitForLockTime;
CrashReporterProcessServicer.StatusReporter.AddToMeanCounter(StatusReportingPerfMeanNames.MinidumpDiagnostics, (int)(MDDRunTime * 1000));
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Symbolicator.Run: Thread blocked for {0:N1}s then MDD ran for {1:N1}s", WaitForLockTime, MDDRunTime));
return(NewSymbolicatorTask.Result);
}
/// <summary>If available, will read CrashContext.runtime-xml.</summary>
public void ReadCrashContextIfAvailable()
{
try
{
//\\epicgames.net\Root\Projects\Paragon\QA_CrashReports
bool bHasCrashContext = HasCrashContextFile();
Plugins = new List <string>();
if (bHasCrashContext)
{
//_CrashContext = FGenericCrashContext.FromFile(SitePath + GetCrashContextUrl());
_CrashContext = FGenericCrashContext.FromUrl(GetCrashContextUrl());
bool bTest = _CrashContext != null && !string.IsNullOrEmpty(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation);
if (bTest)
{
_bUseFullMinidumpPath = true;
//Some temporary code to redirect to the new file location for fulldumps for paragon.
//Consider removing this once fulldumps stop appearing in the old location.
if (_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation.ToLower()
.Contains("\\\\epicgames.net\\root\\dept\\gameqa\\paragon\\paragon_launcherCrashdumps"))
{
//Files from old versions of the client may end up in the old location. Check for files there first.
if (!System.IO.Directory.Exists(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation))
{
var suffix =
_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation.Substring("\\\\epicgames.net\\root\\dept\\gameqa\\paragon\\paragon_launcherCrashdumps".Length);
_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation = String.Format("\\\\epicgames.net\\Root\\Projects\\Paragon\\QA_CrashReports{0}", suffix);
//If the file doesn't exist in the new location either then don't use the full minidump path.
_bUseFullMinidumpPath =
System.IO.Directory.Exists(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation);
}
}
//End of temporary code.
FLogger.Global.WriteEvent("ReadCrashContextIfAvailable " + _CrashContext.PrimaryCrashProperties.FullCrashDumpLocation + " is " + _bUseFullMinidumpPath);
}
if (_CrashContext.UntypedEnabledPlugins != null)
{
XmlNode[] nodes = _CrashContext.UntypedEnabledPlugins as XmlNode[];
if (nodes != null)
{
foreach (var node in nodes)
{
try
{
dynamic data = Json.Decode(node.InnerText);
String printName = data.FriendlyName + " " + data.VersionName;
System.Diagnostics.Debug.WriteLine(printName);
Plugins.Add(printName);
}
catch (Exception e)
{
Debug.WriteLine("Error trying to add a plugin entry: " + e.ToString());
}
}
}
}
}
}
catch (Exception Ex)
{
Debug.WriteLine("Exception in ReadCrashContextIfAvailable: " + Ex.ToString());
}
}
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext)
{
// Use the latest MinidumpDiagnostics from the main branch.
string Win64BinariesDirectory = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine", "Binaries", "Win64");
#if DEBUG
// Note: the debug executable must be built locally or synced from Perforce manually
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe");
//string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe");
#else
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe");
#endif
// Purge logs every 2048 processed crashes
string PurgeLogsDays = ProcessedReports % 2048 == 0 ? "2" : "-1";
FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion);
List <string> MinidumpDiagnosticsParams = new List <string>
(
new string[]
{
"\"" + DiagnosticsPath + "\"",
"-BranchName=" + EngineVersion.Branch, // Backward compatibility
"-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility
"-GameName=" + NewContext.PrimaryCrashProperties.GameName,
"-EngineVersion=" + NewContext.PrimaryCrashProperties.EngineVersion,
"-bUsePDBCache=true",
"-Annotate",
"-SyncSymbols",
"-SyncMicrosoftSymbols",
"-unattended",
"-Log=" + NewContext.GetAsFilename() + "-backup-.log",
"-DepotIndex=" + Properties.Settings.Default.DepotIndex,
"-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays,
"-LOGTIMESINCESTART"
}
);
LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null;
if (Environment.UserInteractive)
{
CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD;
}
else
{
MinidumpDiagnosticsParams.AddRange
(
new[]
{
"-buildmachine",
"-forcelogflush"
}
);
// Write some debugging message.
CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams));
}
LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray());
if (ReportParser.WaitForExit(MinidumpDiagnosticsTimeoutSeconds * 1000) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure("ProcessDumpFile: Timed out running MinidumpDiagnostics");
}
ReadDiagnosticsFile(NewContext);
}
/// <summary> Enqueues a new crash. </summary>
void EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return;
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bFromWER = false;
if (Context == null || !Context.HasProcessedData())
{
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
FReportData ReportData = new FReportData(MetaData, NewReportPath);
Context = ConvertMetadataToCrashContext(MetaData, NewReportPath);
bFromWER = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
}
else
{
if (GenericContext != null && GenericContext.PrimaryCrashProperties.ErrorMessage.Length > 0)
{
// Get error message from the crash context and fix value in the metadata.
Context.PrimaryCrashProperties.ErrorMessage = GenericContext.PrimaryCrashProperties.ErrorMessage;
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// If based on WER, save to the file.
if (bFromWER && GenericContext == null)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteFailure("% NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.MoveReportToInvalid(NewReportPath, Context.GetAsFilename());
return;
}
if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteFailure("% UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.CleanReport(NewReportPath);
return;
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteFailure("! BROKEN0 : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.CleanReport(NewReportPath);
return;
}
lock (NewReportsLock)
{
NewCrashContexts.Add(Context);
}
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
}
}
/// <summary>
/// Main processing thread.
/// </summary>
/// <remarks>All exceptions are caught and written to the event log.</remarks>
private void ProcessNewReports()
{
// Use the latest MinidumpDiagnostics from the main branch.
string Win64BinariesDirectory = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine", "Binaries", "Win64");
#if !DEBUG
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics*");
if (!SyncRequiredFiles(MinidumpDiagnosticsName))
{
return;
}
#endif
DateTime LastCleanupTime = DateTime.Now;
int LastDay = DateTime.UtcNow.Day;
var Cancel = CancelSource.Token;
ProcessorTask = Task.Factory.StartNew(() =>
{
while (!Cancel.IsCancellationRequested)
{
try
{
foreach (var Queue in Watcher.ReportQueues)
{
FGenericCrashContext NewContext = null;
if (Queue.TryDequeueReport(out NewContext))
{
ProcessReport(NewContext);
int CurrentDay = DateTime.UtcNow.Day;
if (CurrentDay > LastDay)
{
// Check the log and create a new one for a new day.
CrashReporterProcessServicer.Log.CreateNewLogFile();
LastDay = CurrentDay;
}
// The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest
break;
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("ProcessNewReports: " + Ex.ToString());
}
// Remove all folders older than n days to prevent unbounded growth
if ((DateTime.Now - LastCleanupTime) > TimeSpan.FromMinutes(15))
{
CleanRepository();
LastCleanupTime = DateTime.Now;
}
// Don't use the CPU if we don't need.
Thread.Sleep(100);
}
});
}
/// <summary> Enqueues a new crash. </summary>
private void EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return;
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bFromWER = false;
if (Context == null || !Context.HasProcessedData())
{
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
FReportData ReportData = new FReportData(MetaData, NewReportPath);
ConvertMetadataToCrashContext(MetaData, NewReportPath, ref Context);
bFromWER = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
}
else
{
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// Added data from WER, save to the crash context file.
if (bFromWER)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
}
else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
}
lock (NewReportsLock)
{
NewCrashContexts.Enqueue(Context);
}
EnqueueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingConstants.QueuedEvent);
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
}
}
double ReadProcessAddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext)
{
try
{
string DumpFileName = null;
string LogFileName = null;
string DiagnosticsFileName = "";
string VideoFileName = null;
string CrashContextName = "";
foreach (FileInfo Info in DirInfo.GetFiles())
{
switch (Info.Extension.ToLower())
{
case ".avi":
case ".mp4":
VideoFileName = Info.Name;
break;
case ".runtime-xml":
CrashContextName = Info.Name;
break;
case ".log":
LogFileName = Info.Name;
break;
case ".txt":
if (string.Compare(Info.Name, CrashReporterConstants.DiagnosticsFileName, true) == 0)
{
DiagnosticsFileName = Info.Name;
ReadDiagnosticsFile(NewContext);
}
break;
case ".dmp":
case ".mdmp":
DumpFileName = Info.Name;
// Check to see if this has been processed locally
FileInfo DiagnosticsInfo = new FileInfo(Path.Combine(DirInfo.FullName, CrashReporterConstants.DiagnosticsFileName));
if (!DiagnosticsInfo.Exists && !NewContext.HasProcessedData())
{
ProcessDumpFile(Info.FullName, NewContext);
}
// Check to see if a new one has been created
DiagnosticsInfo.Refresh();
if (DiagnosticsInfo.Exists)
{
DiagnosticsFileName = DiagnosticsInfo.Name;
}
break;
default:
break;
}
}
// Check if the new context has processed data.
if (!NewContext.HasProcessedData())
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "% Warning no callstack or error msg : BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory);
NewContext.PrimaryCrashProperties.ProcessorFailedMessage = "No callstack or error message. Diagnostics missing or failed.";
}
Stopwatch WaitSW = Stopwatch.StartNew();
// Wait for previous task, should not really happen.
int AddReportTaskSlot = WaitForFreeAddReportTask();
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Starting AddReportTask running on slot {0} of {1} ({2} active)", AddReportTaskSlot, Config.Default.AddReportsPerProcessor, GetActiveAddReportTasks()));
double WaitTime = WaitSW.Elapsed.TotalSeconds;
//bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName );
// Save/update crash context to the file.
NewContext.ToFile();
AddReportTasks[AddReportTaskSlot] = Task.Factory.StartNew(() =>
{
AddReportResult Result = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName);
FinalizeReport(Result, DirInfo, NewContext);
});
return(WaitTime);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessReport: " + NewContext.CrashDirectory + "\n\n: " + Ex, Ex);
}
return(0.0);
}
