/// <summary> Enqueues a new crash. </summary> void EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return; } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bFromWER = false; if (Context == null || !Context.HasProcessedData()) { WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { FReportData ReportData = new FReportData(MetaData, NewReportPath); Context = ConvertMetadataToCrashContext(MetaData, NewReportPath); bFromWER = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); } else { if (GenericContext != null && GenericContext.PrimaryCrashProperties.ErrorMessage.Length > 0) { // Get error message from the crash context and fix value in the metadata. Context.PrimaryCrashProperties.ErrorMessage = GenericContext.PrimaryCrashProperties.ErrorMessage; } Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // If based on WER, save to the file. if (bFromWER && GenericContext == null) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteFailure("% NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.MoveReportToInvalid(NewReportPath, Context.GetAsFilename()); return; } if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteFailure("% UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.CleanReport(NewReportPath); return; } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteFailure("! BROKEN0 : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.CleanReport(NewReportPath); return; } lock (NewReportsLock) { NewCrashContexts.Add(Context); } CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); } }
// From CrashUpload.cpp /* * struct FCompressedCrashFile : FNoncopyable * { * int32 CurrentFileIndex; // 4 bytes for file index * FString Filename; // 4 bytes for length + 260 bytes for char data * TArray<uint8> Filedata; // 4 bytes for length + N bytes for data * } */ /// <summary> Enqueues a new crash. </summary> private bool EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return(false); } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bContextDirty = false; WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { if (Context == null) { // Missing crash context FReportData ReportData = new FReportData(MetaData, NewReportPath); ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context); bContextDirty = true; } else if (!Context.HasProcessedData()) { // Missing data - try to get from WER metadata FReportData ReportData = new FReportData(MetaData, NewReportPath); GetErrorMessageFromMetadata(ReportData, Context); bContextDirty = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // Added data from WER, save to the crash context file. if (bContextDirty) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion; Context.ToFile(); } else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName; Context.ToFile(); } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg"; Context.ToFile(); } // Check static reports index for duplicated reports // This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router) // This can be safely discontinued when all crashes come from the same SQS // This DOES NOT scale to multiple CRP instances // ReportIndex can be disabled by leaving the path empty in config if (!CrashReporterProcessServicer.ReportIndex.TryAddNewReport(ReportName)) { // Crash report not accepted by index CrashReporterProcessServicer.WriteEvent(string.Format( "EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } if (ShouldDecimateNextReport()) { CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } lock (NewReportsLock) { NewCrashContexts.Enqueue(Context); } EnqueueCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent); CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); return(true); }
/// <summary> Enqueues a new crash. </summary> private void EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return; } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bFromWER = false; if (Context == null || !Context.HasProcessedData()) { WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { FReportData ReportData = new FReportData(MetaData, NewReportPath); ConvertMetadataToCrashContext(MetaData, NewReportPath, ref Context); bFromWER = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); } else { Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // Added data from WER, save to the crash context file. if (bFromWER) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion; } else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName; } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg"; } lock (NewReportsLock) { NewCrashContexts.Enqueue(Context); } EnqueueCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingConstants.QueuedEvent); CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); } }
double ReadProcessAddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext) { try { string DumpFileName = null; string LogFileName = null; string DiagnosticsFileName = ""; string VideoFileName = null; string CrashContextName = ""; foreach (FileInfo Info in DirInfo.GetFiles()) { switch (Info.Extension.ToLower()) { case ".avi": case ".mp4": VideoFileName = Info.Name; break; case ".runtime-xml": CrashContextName = Info.Name; break; case ".log": LogFileName = Info.Name; break; case ".txt": if (Info.Name == CrashReporterConstants.DiagnosticsFileName) { DiagnosticsFileName = Info.Name; ReadDiagnosticsFile(NewContext); } break; case ".dmp": case ".mdmp": DumpFileName = Info.Name; // Check to see if this has been processed locally FileInfo DiagnosticsInfo = new FileInfo(Path.Combine(DirInfo.FullName, CrashReporterConstants.DiagnosticsFileName)); if (!DiagnosticsInfo.Exists && !NewContext.HasProcessedData()) { ProcessDumpFile(Info.FullName, NewContext); } // Check to see if a new one has been created DiagnosticsInfo.Refresh(); if (DiagnosticsInfo.Exists) { DiagnosticsFileName = DiagnosticsInfo.Name; } break; default: break; } } // Check if the new context has processed data. if (NewContext.HasProcessedData()) { Stopwatch WaitSW = Stopwatch.StartNew(); // Wait for previous task, should not really happen. if (AddReportTask != null) { AddReportTask.Wait(); } double WaitTime = WaitSW.Elapsed.TotalSeconds; //bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName ); // Save/update crash context to the file. NewContext.ToFile(); AddReportTask = Task.Factory.StartNew(() => { bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName); FinalizeReport(bAdded, DirInfo, NewContext); }); return(WaitTime); } else { CrashReporterProcessServicer.WriteFailure("! Invalid : BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory); FinalizeReport(false, DirInfo, NewContext); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("ProcessReport: " + NewContext.CrashDirectory + "\n\n: " + Ex.ToString()); } return(0.0); }
double ReadProcessAddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext) { try { string DumpFileName = null; string LogFileName = null; string DiagnosticsFileName = ""; string VideoFileName = null; string CrashContextName = ""; foreach (FileInfo Info in DirInfo.GetFiles()) { switch (Info.Extension.ToLower()) { case ".avi": case ".mp4": VideoFileName = Info.Name; break; case ".runtime-xml": CrashContextName = Info.Name; break; case ".log": LogFileName = Info.Name; break; case ".txt": if (string.Compare(Info.Name, CrashReporterConstants.DiagnosticsFileName, true) == 0) { DiagnosticsFileName = Info.Name; ReadDiagnosticsFile(NewContext); } break; case ".dmp": case ".mdmp": DumpFileName = Info.Name; // Check to see if this has been processed locally FileInfo DiagnosticsInfo = new FileInfo(Path.Combine(DirInfo.FullName, CrashReporterConstants.DiagnosticsFileName)); if (!DiagnosticsInfo.Exists && !NewContext.HasProcessedData()) { ProcessDumpFile(Info.FullName, NewContext); } // Check to see if a new one has been created DiagnosticsInfo.Refresh(); if (DiagnosticsInfo.Exists) { DiagnosticsFileName = DiagnosticsInfo.Name; } break; default: break; } } // Check if the new context has processed data. if (!NewContext.HasProcessedData()) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "% Warning no callstack or error msg : BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory); NewContext.PrimaryCrashProperties.ProcessorFailedMessage = "No callstack or error message. Diagnostics missing or failed."; } Stopwatch WaitSW = Stopwatch.StartNew(); // Wait for previous task, should not really happen. int AddReportTaskSlot = WaitForFreeAddReportTask(); CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Starting AddReportTask running on slot {0} of {1} ({2} active)", AddReportTaskSlot, Config.Default.AddReportsPerProcessor, GetActiveAddReportTasks())); double WaitTime = WaitSW.Elapsed.TotalSeconds; //bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName ); // Save/update crash context to the file. NewContext.ToFile(); AddReportTasks[AddReportTaskSlot] = Task.Factory.StartNew(() => { AddReportResult Result = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName); FinalizeReport(Result, DirInfo, NewContext); }); return(WaitTime); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessReport: " + NewContext.CrashDirectory + "\n\n: " + Ex, Ex); } return(0.0); }
/// <summary> /// Create an Xml payload representing a new crash. /// </summary> /// <param name="DirInfo">The DirectoryInfo of the report folder.</param> /// <param name="NewContext">The generic crash context.</param> /// <param name="bHasVideoFile">Whether the report contains a video file.</param> /// <param name="bHasLog">Whether the report contains a log file.</param> /// <param name="bHasMinidump">Whether the report contains a minidump.</param> /// <returns>A string of Xml payload representing the newly found crash report.</returns> private string CreateCrash(DirectoryInfo DirInfo, FGenericCrashContext NewContext, bool bHasVideoFile, bool bHasLog, bool bHasMinidump) { string XmlPayload = ""; try { FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion); // Create a new crash description for uploading var NewCrash = new CrashDescription(); NewCrash.CrashType = NewContext.PrimaryCrashProperties.GetCrashType(); NewCrash.BranchName = EngineVersion.GetCleanedBranch(); NewCrash.GameName = NewContext.PrimaryCrashProperties.GameName; NewCrash.Platform = NewContext.PrimaryCrashProperties.PlatformFullName; NewCrash.EngineMode = NewContext.PrimaryCrashProperties.EngineMode; NewCrash.EngineModeEx = NewContext.PrimaryCrashProperties.GetEngineModeEx(); NewCrash.EngineVersion = EngineVersion.VersionNumber; NewCrash.BuildVersion = NewContext.PrimaryCrashProperties.BuildVersion; NewCrash.CommandLine = NewContext.PrimaryCrashProperties.CommandLine; NewCrash.BaseDir = NewContext.PrimaryCrashProperties.BaseDir; NewCrash.bProcessorFailed = !string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.ProcessorFailedMessage); NewCrash.Language = NewContext.PrimaryCrashProperties.AppDefaultLocale; // // Create a locate and get the system language. // int SystemLanguageCode = 0; // int.TryParse( ReportData.SystemLanguage, out SystemLanguageCode ); // try // { // if( SystemLanguageCode > 0 ) // { // CultureInfo SystemLanguageCI = new CultureInfo( SystemLanguageCode ); // NewCrash.SystemLanguage = SystemLanguageCI.Name; // } // } // catch( System.Exception ) // { // // Default to en-US // NewCrash.SystemLanguage = "en-US"; // } NewCrash.MachineGuid = NewContext.PrimaryCrashProperties.MachineId; // Valid for all kind of builds, previously only for UE4 releases. NewCrash.UserName = NewContext.PrimaryCrashProperties.UserName; // Only valid for non-UE4 releases. NewCrash.EpicAccountId = NewContext.PrimaryCrashProperties.EpicAccountId; // Only valid for UE4 releases. NewCrash.CallStack = NewContext.PrimaryCrashProperties.GetCallstack(); NewCrash.SourceContext = NewContext.PrimaryCrashProperties.GetSourceContext(); NewCrash.ErrorMessage = NewContext.PrimaryCrashProperties.GetErrorMessage(); NewCrash.UserDescription = NewContext.PrimaryCrashProperties.GetUserDescription(); NewCrash.UserActivityHint = NewContext.PrimaryCrashProperties.UserActivityHint; NewCrash.CrashGUID = NewContext.PrimaryCrashProperties.CrashGUID; // Iterate through all files and find a file with the earliest date. DateTime TimeOfCrash = DateTime.UtcNow; foreach (var File in DirInfo.GetFiles()) { if (File.CreationTimeUtc < TimeOfCrash) { TimeOfCrash = File.CreationTimeUtc; } } //NewCrash.TimeofCrash = NewContext.PrimaryCrashProperties.TimeofCrash; NewCrash.TimeofCrash = TimeOfCrash; NewCrash.bHasMiniDump = bHasMinidump; NewCrash.bHasLog = bHasLog; NewCrash.bHasVideo = bHasVideoFile; NewCrash.BuiltFromCL = (int)EngineVersion.Changelist; NewCrash.bAllowToBeContacted = NewContext.PrimaryCrashProperties.bAllowToBeContacted; // Ignore any callstack that is shorter than expected, usually the callstack is invalid. if (NewCrash.CallStack.Length <= CrashReporterConstants.MinCallstackDepth) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "! BadStack: BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory); NewContext.PrimaryCrashProperties.ProcessorFailedMessage = string.Format("Callstack was too small. {0} lines (minimum {1})", NewCrash.CallStack.Length, CrashReporterConstants.MinCallstackDepth); NewContext.ToFile(); } XmlPayload = XmlHandler.ToXmlString <CrashDescription>(NewCrash); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "CreateCrash: " + Ex, Ex); } return(XmlPayload); }