/// <summary> Looks for the WER metadata xml file, if found, will return a new instance of the WERReportMetadata. </summary>
private WERReportMetadata FindMetadata(string NewReportPath)
{
WERReportMetadata MetaData = null;
// Just to verify that the report is still there.
DirectoryInfo DirInfo = new DirectoryInfo(NewReportPath);
if (!DirInfo.Exists)
{
CrashReporterProcessServicer.WriteFailure("FindMetadata: Directory not found " + NewReportPath);
}
else
{
// Check to see if we wish to suppress processing of this report.
foreach (var Info in DirInfo.GetFiles("*.xml"))
{
var MetaDataToCheck = XmlHandler.ReadXml <WERReportMetadata>(Info.FullName);
if (CheckMetaData(MetaDataToCheck))
{
MetaData = MetaDataToCheck;
break;
}
}
}
return(MetaData);
}
/// <summary>
/// Check to see if we wish to reject a report based on the WER meta data.
/// </summary>
/// <param name="Request">A request containing the XML representation of a WERReportMetadata class instance.</param>
/// <returns>true if we do not reject.</returns>
private CrashReporterResult CheckReportDetail(HttpListenerRequest Request)
{
CrashReporterResult ReportResult = new CrashReporterResult();
#if DISABLED_CRR
ReportResult.bSuccess = false;
ReportResult.Message = "CRR disabled";
CrashReporterReceiverServicer.WriteEvent("CheckReportDetail() Report rejected by disabled CRR");
#else
string WERReportMetadataString = GetContentStreamString(Request);
WERReportMetadata WERData = null;
if (WERReportMetadataString.Length > 0)
{
try
{
WERData = XmlHandler.FromXmlString <WERReportMetadata>(WERReportMetadataString);
}
catch (System.Exception Ex)
{
CrashReporterReceiverServicer.WriteEvent("Error during XmlHandler.FromXmlString, probably incorrect encoding, trying to fix: " + Ex.Message);
byte[] StringBytes = System.Text.Encoding.Unicode.GetBytes(WERReportMetadataString);
string ConvertedXML = System.Text.Encoding.UTF8.GetString(StringBytes);
WERData = XmlHandler.FromXmlString <WERReportMetadata>(ConvertedXML);
}
}
if (WERData != null)
{
// Ignore crashes in the minidump parser itself
ReportResult.bSuccess = true;
if (WERData.ProblemSignatures.Parameter0.ToLower() == "MinidumpDiagnostics".ToLower())
{
ReportResult.bSuccess = false;
ReportResult.Message = "Rejecting MinidumpDiagnostics crash";
}
// Ignore Debug and DebugGame crashes
string CrashingModule = WERData.ProblemSignatures.Parameter3.ToLower();
if (CrashingModule.Contains("-debug"))
{
ReportResult.bSuccess = false;
ReportResult.Message = "Rejecting Debug or DebugGame crash";
}
}
#endif
return(ReportResult);
}
/// <summary>
/// Optionally don't process some reports based on the Windows Error report meta data.
/// </summary>
/// <param name="WERData">The Windows Error Report meta data.</param>
/// <returns>false to reject the report.</returns>
private bool CheckMetaData(WERReportMetadata WERData)
{
if (WERData == null)
{
return(false);
}
// Reject any crashes with the invalid metadata.
if (WERData.ProblemSignatures == null || WERData.DynamicSignatures == null || WERData.OSVersionInformation == null)
{
return(false);
}
// Reject any crashes from the minidump processor.
if (WERData.ProblemSignatures.Parameter0 != null && WERData.ProblemSignatures.Parameter0.ToLower() == "MinidumpDiagnostics".ToLower())
{
return(false);
}
return(true);
}
/// <summary> Enqueues a new crash. </summary>
void EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return;
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bFromWER = false;
if (Context == null || !Context.HasProcessedData())
{
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
FReportData ReportData = new FReportData(MetaData, NewReportPath);
Context = ConvertMetadataToCrashContext(MetaData, NewReportPath);
bFromWER = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
}
else
{
if (GenericContext != null && GenericContext.PrimaryCrashProperties.ErrorMessage.Length > 0)
{
// Get error message from the crash context and fix value in the metadata.
Context.PrimaryCrashProperties.ErrorMessage = GenericContext.PrimaryCrashProperties.ErrorMessage;
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// If based on WER, save to the file.
if (bFromWER && GenericContext == null)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteFailure("% NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.MoveReportToInvalid(NewReportPath, Context.GetAsFilename());
return;
}
if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteFailure("% UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.CleanReport(NewReportPath);
return;
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteFailure("! BROKEN0 : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.CleanReport(NewReportPath);
return;
}
lock (NewReportsLock)
{
NewCrashContexts.Add(Context);
}
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
}
}
/// <summary> Converts WER metadata xml file to the crash context. </summary>
private FGenericCrashContext ConvertMetadataToCrashContext(WERReportMetadata Metadata, string NewReportPath)
{
FGenericCrashContext CrashContext = new FGenericCrashContext();
FReportData ReportData = new FReportData(Metadata, NewReportPath);
CrashContext.PrimaryCrashProperties.CrashVersion = (int)ECrashDescVersions.VER_1_NewCrashFormat;
CrashContext.PrimaryCrashProperties.ProcessId = 0;
CrashContext.PrimaryCrashProperties.CrashGUID = new DirectoryInfo(NewReportPath).Name;
// CrashContext.PrimaryCrashProperties.IsInternalBuild
// CrashContext.PrimaryCrashProperties.IsPerforceBuild
// CrashContext.PrimaryCrashProperties.IsSourceDistribution
// CrashContext.PrimaryCrashProperties.SecondsSinceStart
CrashContext.PrimaryCrashProperties.GameName = ReportData.GameName;
// CrashContext.PrimaryCrashProperties.ExecutableName
// CrashContext.PrimaryCrashProperties.BuildConfiguration
// CrashContext.PrimaryCrashProperties.PlatformName
// CrashContext.PrimaryCrashProperties.PlatformNameIni
CrashContext.PrimaryCrashProperties.PlatformFullName = ReportData.Platform;
CrashContext.PrimaryCrashProperties.EngineMode = ReportData.EngineMode;
CrashContext.PrimaryCrashProperties.EngineVersion = ReportData.GetEngineVersion();
CrashContext.PrimaryCrashProperties.CommandLine = ReportData.CommandLine;
// CrashContext.PrimaryCrashProperties.LanguageLCID
// Create a locate and get the language.
int LanguageCode = 0;
int.TryParse(ReportData.Language, out LanguageCode);
try
{
if (LanguageCode > 0)
{
CultureInfo LanguageCI = new CultureInfo(LanguageCode);
CrashContext.PrimaryCrashProperties.AppDefaultLocale = LanguageCI.Name;
}
}
catch (System.Exception)
{
// Default to en-US
CrashContext.PrimaryCrashProperties.AppDefaultLocale = "en-US";
}
// CrashContext.PrimaryCrashProperties.IsUE4Release
CrashContext.PrimaryCrashProperties.UserName = ReportData.UserName;
CrashContext.PrimaryCrashProperties.BaseDir = ReportData.BaseDir;
// CrashContext.PrimaryCrashProperties.RootDir
CrashContext.PrimaryCrashProperties.MachineId = ReportData.MachineId;
CrashContext.PrimaryCrashProperties.EpicAccountId = ReportData.EpicAccountId;
// CrashContext.PrimaryCrashProperties.CallStack
// CrashContext.PrimaryCrashProperties.SourceContext
CrashContext.PrimaryCrashProperties.UserDescription = string.Join("\n", ReportData.UserDescription);
CrashContext.PrimaryCrashProperties.ErrorMessage = string.Join("\n", ReportData.ErrorMessage);
// CrashContext.PrimaryCrashProperties.CrashDumpMode
// CrashContext.PrimaryCrashProperties.Misc.NumberOfCores
// CrashContext.PrimaryCrashProperties.Misc.NumberOfCoresIncludingHyperthreads
// CrashContext.PrimaryCrashProperties.Misc.Is64bitOperatingSystem
// CrashContext.PrimaryCrashProperties.Misc.CPUVendor
// CrashContext.PrimaryCrashProperties.Misc.CPUBrand
// CrashContext.PrimaryCrashProperties.Misc.PrimaryGPUBrand
// CrashContext.PrimaryCrashProperties.Misc.OSVersionMajor
// CrashContext.PrimaryCrashProperties.Misc.OSVersionMinor
// CrashContext.PrimaryCrashProperties.Misc.AppDiskTotalNumberOfBytes
// CrashContext.PrimaryCrashProperties.Misc.AppDiskNumberOfFreeBytes
// CrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysical
// CrashContext.PrimaryCrashProperties.MemoryStats.TotalVirtual
// CrashContext.PrimaryCrashProperties.MemoryStats.PageSize
// CrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysicalGB
// CrashContext.PrimaryCrashProperties.MemoryStats.AvailablePhysical
// CrashContext.PrimaryCrashProperties.MemoryStats.AvailableVirtual
// CrashContext.PrimaryCrashProperties.MemoryStats.UsedPhysical
// CrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedPhysical
// CrashContext.PrimaryCrashProperties.MemoryStats.UsedVirtual
// CrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedVirtual
// CrashContext.PrimaryCrashProperties.MemoryStats.bIsOOM
// CrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationSize
// CrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationAlignment
CrashContext.PrimaryCrashProperties.TimeofCrash = new DateTime(ReportData.Ticks);
CrashContext.PrimaryCrashProperties.bAllowToBeContacted = ReportData.AllowToBeContacted;
return(CrashContext);
}
// From CrashUpload.cpp
/*
* struct FCompressedCrashFile : FNoncopyable
* {
* int32 CurrentFileIndex; // 4 bytes for file index
* FString Filename; // 4 bytes for length + 260 bytes for char data
* TArray<uint8> Filedata; // 4 bytes for length + N bytes for data
* }
*/
/// <summary> Enqueues a new crash. </summary>
private bool EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return(false);
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bContextDirty = false;
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
if (Context == null)
{
// Missing crash context
FReportData ReportData = new FReportData(MetaData, NewReportPath);
ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context);
bContextDirty = true;
}
else if (!Context.HasProcessedData())
{
// Missing data - try to get from WER metadata
FReportData ReportData = new FReportData(MetaData, NewReportPath);
GetErrorMessageFromMetadata(ReportData, Context);
bContextDirty = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// Added data from WER, save to the crash context file.
if (bContextDirty)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
Context.ToFile();
}
else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
Context.ToFile();
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
Context.ToFile();
}
// Check static reports index for duplicated reports
// This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router)
// This can be safely discontinued when all crashes come from the same SQS
// This DOES NOT scale to multiple CRP instances
// ReportIndex can be disabled by leaving the path empty in config
if (!CrashReporterProcessServicer.ReportIndex.TryAddNewReport(ReportName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
if (ShouldDecimateNextReport())
{
CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
lock (NewReportsLock)
{
NewCrashContexts.Enqueue(Context);
}
EnqueueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent);
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
return(true);
}
/// <summary> Enqueues a new crash. </summary>
private void EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return;
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bFromWER = false;
if (Context == null || !Context.HasProcessedData())
{
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
FReportData ReportData = new FReportData(MetaData, NewReportPath);
ConvertMetadataToCrashContext(MetaData, NewReportPath, ref Context);
bFromWER = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
}
else
{
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// Added data from WER, save to the crash context file.
if (bFromWER)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
}
else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
}
lock (NewReportsLock)
{
NewCrashContexts.Enqueue(Context);
}
EnqueueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingConstants.QueuedEvent);
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
}
}
