private static void GetErrorMessageFromMetadata(FReportData ReportData, FGenericCrashContext CrashContext) { if (string.IsNullOrEmpty(CrashContext.PrimaryCrashProperties.ErrorMessage)) { CrashContext.PrimaryCrashProperties.ErrorMessage = string.Join("\n", ReportData.ErrorMessage); } }
void ProcessDumpFile(string DumpPath, FGenericCrashContext NewContext) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DumpPath); if (TrySimulateSymbolication(NewContext)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Simulated symbolication (skipped MDD) " + NewContext.CrashDirectory); } else if (CrashReporterProcessServicer.Symbolicator.Run(DumpPath, NewContext, ProcessorIndex)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DumpPath); ReadDiagnosticsFile(NewContext); } else { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DumpPath); } if (string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.CallStack)) { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationFailedEvent); } else { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationSucceededEvent); } }
/// <summary> Looks for the CrashContext.runtime-xml, if found, will return a new instance of the FGenericCrashContext. </summary> private FGenericCrashContext FindCrashContext(string NewReportPath) { string CrashContextPath = Path.Combine(NewReportPath, FGenericCrashContext.CrashContextRuntimeXMLName); bool bExist = File.Exists(CrashContextPath); if (bExist) { return(FGenericCrashContext.FromFile(CrashContextPath)); } return(null); }
/// <summary> /// Main processing thread. /// </summary> /// <remarks>All exceptions are caught and written to the event log.</remarks> private void Init() { ProcessorTask = new Task(() => { // parse out hte list of blacklisted games. string[] GameNamesToBlacklist = (Config.Default.GameNamesToBlacklist == null ? "" : Config.Default.GameNamesToBlacklist).ToLowerInvariant().Split(','); while (!CancelSource.IsCancellationRequested) { try { bool bIdle = true; foreach (var Queue in Watcher.ReportQueues) { FGenericCrashContext NewContext = null; if (Queue.TryDequeueReport(out NewContext)) { bool bIsBlacklistedGame = GameNamesToBlacklist.Contains(NewContext.PrimaryCrashProperties.GameName.ToLowerInvariant()); // if it's blacklisted, skip it, else process it. if (bIsBlacklistedGame) { CrashReporterProcessServicer.WriteEvent(string.Format("Discarding crash from blacklisted GameName '{0}'", NewContext.PrimaryCrashProperties.GameName)); // delete the report from disk since we don't care about it. FinalizeReport(AddReportResult.Added, new DirectoryInfo(NewContext.CrashDirectory), NewContext); } else { ProcessReport(NewContext); } // The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest bIdle = false; break; } } if (bIdle) { // Don't use the CPU if we don't need. Thread.Sleep(1000); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessNewReports: " + Ex, Ex); } TickStatic(Watcher); } }); }
/// <summary> /// Finalizes report files. /// Thread-safe. /// </summary> private void FinalizeReport(bool bAdded, DirectoryInfo DirInfo, FGenericCrashContext NewContext) { // Only remove if we added the report, otherwise leave all files to further investigation. if (bAdded) { // Remove the report files as we're done with them. CleanReport(DirInfo); } else { MoveReportToInvalid(NewContext.CrashDirectory, NewContext.GetAsFilename()); } }
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DiagnosticsPath); if (CrashReporterProcessServicer.Symbolicator.Run(DiagnosticsPath, NewContext, ProcessorIndex)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DiagnosticsPath); ReadDiagnosticsFile(NewContext); } else { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DiagnosticsPath); } }
bool TrySimulateSymbolication(FGenericCrashContext NewContext) { // Check for crashes we don't want to attempt to symbolicate if (!string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.ErrorMessage) && NewContext.PrimaryCrashProperties.ErrorMessage.Contains("Hang detected on")) { // Hangs from FThreadHeartBeat::Run() start wth "Hang detected on" and have a useful callstack already in the log // Callstack and source from MDD will be useless so simulate a callstack that points the user to the log. NewContext.PrimaryCrashProperties.CallStack = "See log for callstack from hang detection code()\nSee log for callstack from hang detection code()\nSee log for callstack from hang detection code()"; return(true); } return(false); }
/// <summary> Reads callstack, source context and error message from the diagnostics file, if exists. </summary> private void ReadDiagnosticsFile(FGenericCrashContext NewContext) { // Read the callstack and the source context from the diagnostics files. string CrashDiagnosticsPath = Path.Combine(NewContext.CrashDirectory, CrashReporterConstants.DiagnosticsFileName); if (File.Exists(CrashDiagnosticsPath)) { NewContext.PrimaryCrashProperties.CallStack = FGenericCrashContext.EscapeXMLString(string.Join("\n", FReportData.GetCallStack(CrashDiagnosticsPath))); NewContext.PrimaryCrashProperties.SourceContext = FGenericCrashContext.EscapeXMLString(string.Join("\n", FReportData.GetSourceContext(CrashDiagnosticsPath))); if (NewContext.PrimaryCrashProperties.ErrorMessage.Length == 0) { NewContext.PrimaryCrashProperties.ErrorMessage = FGenericCrashContext.EscapeXMLString(string.Join("\n", FReportData.GetExceptionDescription(CrashDiagnosticsPath))); } } }
/// <summary>If available, will read CrashContext.runtime-xml.</summary> public void ReadCrashContextIfAvailable() { try { //\\epicgames.net\Root\Projects\Paragon\QA_CrashReports bool bHasCrashContext = HasCrashContextFile(); if (bHasCrashContext) { //_CrashContext = FGenericCrashContext.FromFile(SitePath + GetCrashContextUrl()); _CrashContext = Settings.Default.DownloadFromS3 ? FGenericCrashContext.FromUrl(GetCrashContextUrl()) : null; bool bTest = _CrashContext != null && !string.IsNullOrEmpty(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation); if (bTest) { _bUseFullMinidumpPath = true; //Some temporary code to redirect to the new file location for fulldumps for paragon. //Consider removing this once fulldumps stop appearing in the old location. if (_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation.ToLower() .Contains("\\\\epicgames.net\\root\\dept\\gameqa\\paragon\\paragon_launcherCrashdumps")) { //Files from old versions of the client may end up in the old location. Check for files there first. if (!System.IO.Directory.Exists(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation)) { var suffix = _CrashContext.PrimaryCrashProperties.FullCrashDumpLocation.Substring("\\\\epicgames.net\\root\\dept\\gameqa\\paragon\\paragon_launcherCrashdumps".Length); _CrashContext.PrimaryCrashProperties.FullCrashDumpLocation = String.Format("\\\\epicgames.net\\Root\\Projects\\Paragon\\QA_CrashReports{0}", suffix); //If the file doesn't exist in the new location either then don't use the full minidump path. _bUseFullMinidumpPath = System.IO.Directory.Exists(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation); } } //End of temporary code. FLogger.Global.WriteEvent("ReadCrashContextIfAvailable " + _CrashContext.PrimaryCrashProperties.FullCrashDumpLocation + " is " + _bUseFullMinidumpPath); } } } catch (Exception Ex) { Debug.WriteLine("Exception in ReadCrashContextIfAvailable: " + Ex.ToString()); } }
/// <summary> Tries to dequeue a report from the list. </summary> public bool TryDequeueReport(out FGenericCrashContext Context) { lock (NewReportsLock) { if (NewCrashContexts.Count > 0) { Context = NewCrashContexts.Dequeue(); DequeueCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(QueueProcessingStartedEventName); CrashReporterProcessServicer.WriteEvent(string.Format("- Dequeued: {0:N1}/minute BuiltFromCL={1,7} Path={2}", DequeueCounter.EventsPerSecond * 60, Context.PrimaryCrashProperties.EngineVersion, Context.CrashDirectory)); return(true); } } Context = null; return(false); }
/// <summary> /// A function to process a newly landed crash report. /// </summary> /// <param name="NewContext">An instance of the generic crash context</param> private void ProcessReport(FGenericCrashContext NewContext) { Stopwatch ProcessReportSW = Stopwatch.StartNew(); // Just to verify that the report is still there. DirectoryInfo DirInfo = new DirectoryInfo(NewContext.CrashDirectory); if (!DirInfo.Exists) { // Something very odd happened. CrashReporterProcessServicer.WriteEvent("ProcessReport: Directory not found: " + NewContext.CrashDirectory); return; } double WaitTime = ReadProcessAddReport(DirInfo, NewContext); // Make sure any log messages have been written to disk CrashReporterProcessServicer.WriteEvent(string.Format("ProcessReportTime={0} WaitTime={1}", ProcessReportSW.Elapsed.TotalSeconds.ToString("0.0"), WaitTime.ToString("0.00"))); }
/// <summary> Tries to dequeue a report from the list. </summary> public bool TryDequeueReport(out FGenericCrashContext Context) { lock ( NewReportsLock ) { int LastIndex = NewCrashContexts.Count - 1; if (LastIndex >= 0) { Context = NewCrashContexts[LastIndex]; NewCrashContexts.RemoveAt(LastIndex); CrashReporterProcessServicer.WriteEvent("- Dequeued: BuiltFromCL=" + string.Format("{0,7}", Context.PrimaryCrashProperties.EngineVersion) + " Path=" + Context.CrashDirectory); return(true); } else { Context = null; return(false); } } }
/// <summary>If available, will read CrashContext.runtime-xml.</summary> public void ReadCrashContextIfAvailable() { try { bool bHasCrashContext = HasCrashContextFile(); if (bHasCrashContext) { CrashContext = FGenericCrashContext.FromFile(SitePath + GetCrashContextUrl()); bool bTest = CrashContext != null && !string.IsNullOrEmpty(CrashContext.PrimaryCrashProperties.FullCrashDumpLocation); if (bTest) { bUseFullMinidumpPath = true; // System.IO.Directory.Exists( CrashContext.PrimaryCrashProperties.FullCrashDumpLocation ); Doesn't work, probably due to some permissions. FLogger.Global.WriteEvent("ReadCrashContextIfAvailable " + CrashContext.PrimaryCrashProperties.FullCrashDumpLocation + " is " + bUseFullMinidumpPath); } } } catch (Exception Ex) { Debug.WriteLine("Exception in ReadCrashContextIfAvailable: " + Ex.ToString()); } }
/// <summary> /// Main processing thread. /// </summary> /// <remarks>All exceptions are caught and written to the event log.</remarks> private void Init() { var Cancel = CancelSource.Token; ProcessorTask = new Task(() => { while (!Cancel.IsCancellationRequested) { try { bool bIdle = true; foreach (var Queue in Watcher.ReportQueues) { FGenericCrashContext NewContext = null; if (Queue.TryDequeueReport(out NewContext)) { ProcessReport(NewContext); // The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest bIdle = false; break; } } if (bIdle) { // Don't use the CPU if we don't need. Thread.Sleep(1000); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessNewReports: " + Ex, Ex); } TickStatic(Watcher); } }); }
// From CrashUpload.cpp /* * struct FCompressedCrashFile : FNoncopyable * { * int32 CurrentFileIndex; // 4 bytes for file index * FString Filename; // 4 bytes for length + 260 bytes for char data * TArray<uint8> Filedata; // 4 bytes for length + N bytes for data * } */ /// <summary> Enqueues a new crash. </summary> private bool EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return(false); } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bContextDirty = false; WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { if (Context == null) { // Missing crash context FReportData ReportData = new FReportData(MetaData, NewReportPath); ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context); bContextDirty = true; } else if (!Context.HasProcessedData()) { // Missing data - try to get from WER metadata FReportData ReportData = new FReportData(MetaData, NewReportPath); GetErrorMessageFromMetadata(ReportData, Context); bContextDirty = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // Added data from WER, save to the crash context file. if (bContextDirty) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion; Context.ToFile(); } else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName; Context.ToFile(); } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg"; Context.ToFile(); } // Check static reports index for duplicated reports // This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router) // This can be safely discontinued when all crashes come from the same SQS // This DOES NOT scale to multiple CRP instances // ReportIndex can be disabled by leaving the path empty in config if (!CrashReporterProcessServicer.ReportIndex.TryAddNewReport(ReportName)) { // Crash report not accepted by index CrashReporterProcessServicer.WriteEvent(string.Format( "EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } if (ShouldDecimateNextReport()) { CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } lock (NewReportsLock) { NewCrashContexts.Enqueue(Context); } EnqueueCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent); CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); return(true); }
/// <summary> Converts WER metadata xml file to the crash context. </summary> private static void ConvertMetadataToCrashContext(FReportData ReportData, string NewReportPath, ref FGenericCrashContext OutCrashContext) { if (OutCrashContext == null) { OutCrashContext = new FGenericCrashContext(); } OutCrashContext.PrimaryCrashProperties.CrashVersion = (int)ECrashDescVersions.VER_1_NewCrashFormat; //OutCrashContext.PrimaryCrashProperties.ProcessId = 0; don't overwrite valid ids, zero is default anyway OutCrashContext.PrimaryCrashProperties.CrashGUID = new DirectoryInfo(NewReportPath).Name; // OutCrashContext.PrimaryCrashProperties.IsInternalBuild // OutCrashContext.PrimaryCrashProperties.IsPerforceBuild // OutCrashContext.PrimaryCrashProperties.IsSourceDistribution // OutCrashContext.PrimaryCrashProperties.SecondsSinceStart OutCrashContext.PrimaryCrashProperties.GameName = ReportData.GameName; // OutCrashContext.PrimaryCrashProperties.ExecutableName // OutCrashContext.PrimaryCrashProperties.BuildConfiguration // OutCrashContext.PrimaryCrashProperties.PlatformName // OutCrashContext.PrimaryCrashProperties.PlatformNameIni OutCrashContext.PrimaryCrashProperties.PlatformFullName = ReportData.Platform; OutCrashContext.PrimaryCrashProperties.EngineMode = ReportData.EngineMode; OutCrashContext.PrimaryCrashProperties.EngineVersion = ReportData.GetEngineVersion(); OutCrashContext.PrimaryCrashProperties.BuildVersion = ReportData.BuildVersion; OutCrashContext.PrimaryCrashProperties.CommandLine = ReportData.CommandLine; // OutCrashContext.PrimaryCrashProperties.LanguageLCID // Create a locate and get the language. int LanguageCode = 0; int.TryParse(ReportData.Language, out LanguageCode); try { if (LanguageCode > 0) { CultureInfo LanguageCI = new CultureInfo(LanguageCode); OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = LanguageCI.Name; } } catch (Exception) { OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = null; } if (string.IsNullOrEmpty(OutCrashContext.PrimaryCrashProperties.AppDefaultLocale)) { // Default to en-US OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = "en-US"; } // OutCrashContext.PrimaryCrashProperties.IsUE4Release string WERUserName = ReportData.UserName; if (!string.IsNullOrEmpty(WERUserName) || string.IsNullOrEmpty(OutCrashContext.PrimaryCrashProperties.UserName)) { OutCrashContext.PrimaryCrashProperties.UserName = WERUserName; } OutCrashContext.PrimaryCrashProperties.BaseDir = ReportData.BaseDir; // OutCrashContext.PrimaryCrashProperties.RootDir OutCrashContext.PrimaryCrashProperties.MachineId = ReportData.MachineId; OutCrashContext.PrimaryCrashProperties.LoginId = ReportData.LoginId; if (string.IsNullOrWhiteSpace(OutCrashContext.PrimaryCrashProperties.MachineId)) { // Set MachineId from LoginId if the metadata is too new to contain MachineId OutCrashContext.PrimaryCrashProperties.MachineId = OutCrashContext.PrimaryCrashProperties.LoginId; } OutCrashContext.PrimaryCrashProperties.EpicAccountId = ReportData.EpicAccountId; // OutCrashContext.PrimaryCrashProperties.CallStack // OutCrashContext.PrimaryCrashProperties.SourceContext OutCrashContext.PrimaryCrashProperties.UserDescription = string.Join("\n", ReportData.UserDescription); // OutCrashContext.PrimaryCrashProperties.CrashDumpMode // OutCrashContext.PrimaryCrashProperties.Misc.NumberOfCores // OutCrashContext.PrimaryCrashProperties.Misc.NumberOfCoresIncludingHyperthreads // OutCrashContext.PrimaryCrashProperties.Misc.Is64bitOperatingSystem // OutCrashContext.PrimaryCrashProperties.Misc.CPUVendor // OutCrashContext.PrimaryCrashProperties.Misc.CPUBrand // OutCrashContext.PrimaryCrashProperties.Misc.PrimaryGPUBrand // OutCrashContext.PrimaryCrashProperties.Misc.OSVersionMajor // OutCrashContext.PrimaryCrashProperties.Misc.OSVersionMinor // OutCrashContext.PrimaryCrashProperties.Misc.AppDiskTotalNumberOfBytes // OutCrashContext.PrimaryCrashProperties.Misc.AppDiskNumberOfFreeBytes // OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysical // OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalVirtual // OutCrashContext.PrimaryCrashProperties.MemoryStats.PageSize // OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysicalGB // OutCrashContext.PrimaryCrashProperties.MemoryStats.AvailablePhysical // OutCrashContext.PrimaryCrashProperties.MemoryStats.AvailableVirtual // OutCrashContext.PrimaryCrashProperties.MemoryStats.UsedPhysical // OutCrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedPhysical // OutCrashContext.PrimaryCrashProperties.MemoryStats.UsedVirtual // OutCrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedVirtual // OutCrashContext.PrimaryCrashProperties.MemoryStats.bIsOOM // OutCrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationSize // OutCrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationAlignment OutCrashContext.PrimaryCrashProperties.TimeofCrash = new DateTime(ReportData.Ticks); OutCrashContext.PrimaryCrashProperties.bAllowToBeContacted = ReportData.AllowToBeContacted; OutCrashContext.PrimaryCrashProperties.DeploymentName = ReportData.DeploymentName; OutCrashContext.PrimaryCrashProperties.IsEnsure = ReportData.IsEnsure; OutCrashContext.PrimaryCrashProperties.IsAssert = ReportData.IsAssert; OutCrashContext.PrimaryCrashProperties.CrashType = ReportData.CrashType; GetErrorMessageFromMetadata(ReportData, OutCrashContext); }
/// <summary> /// A function to add a report to the database, and rename the relevant files. /// Thread-safe. /// </summary> /// <param name="DirInfo">The DirectoryInfo of the report folder.</param> /// <param name="NewContext">The generic crash context.</param> /// <param name="LogFileName">The file name of the log file in the report.</param> /// <param name="DumpFileName">The file name of the minidump in the report.</param> /// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param> private AddReportResult AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName) { try { Stopwatch AddReportTime = Stopwatch.StartNew(); // Create an XML representation of a crash string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null); if (CrashDetails == "") { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "! CreateCrash no payload: Path=" + NewContext.CrashDirectory); return(AddReportResult.Failed); } // Upload the crash to the database, and retrieve the new row id int ReportID = UploadCrash(CrashDetails); if (ReportID <= 0) { if (CancelSource.IsCancellationRequested) { // Cancelled upload CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "CancelUpload: Path=" + NewContext.CrashDirectory); return(AddReportResult.Cancelled); } else { // Upload failed CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "! NoUpload: Path=" + NewContext.CrashDirectory); return(AddReportResult.Failed); } } bool bToS3 = Config.Default.CrashFilesToAWS && CrashReporterProcessServicer.OutputAWS.IsS3Valid; bool bToDisk = Config.Default.CrashFilesToDisk; // Use the row id to name and move the files the way the web site requires string IDThenUnderscore = string.Format("{0}_", ReportID); int ReportIDSegment = (ReportID / 10000) * 10000; string S3IDPrefix = string.Format("/{0}/{1}/{1}_", ReportIDSegment, ReportID); string DestinationFolder = Path.Combine(Config.Default.ProcessedReports, IDThenUnderscore); Stopwatch WriteToS3Timer = new Stopwatch(); Stopwatch WriteToDiskTimer = new Stopwatch(); if (bToDisk) { CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(DestinationFolder, Config.Default.DiskSpaceAlertPercent); } // Save log file if (LogFileName != null) { LogFileName = Path.Combine(DirInfo.FullName, LogFileName); FileInfo LogInfo = new FileInfo(LogFileName); if (LogInfo.Exists) { if (bToS3) { WriteToS3Timer.Start(); UploadFileToS3(LogInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + "Launch.log", Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix); WriteToS3Timer.Stop(); } if (bToDisk) { WriteToDiskTimer.Start(); LogInfo.MoveTo(DestinationFolder + "Launch.log"); WriteToDiskTimer.Stop(); } } } // Save crash context file string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName); FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName); if (CrashContextInfo.Exists) { if (bToS3) { WriteToS3Timer.Start(); UploadFileToS3(CrashContextInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + FGenericCrashContext.CrashContextRuntimeXMLName, false); WriteToS3Timer.Stop(); } if (bToDisk) { WriteToDiskTimer.Start(); CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName); WriteToDiskTimer.Stop(); } } if (DumpFileName != null) { DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName); FileInfo DumpInfo = new FileInfo(DumpFileName); if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/) { if (bToS3) { WriteToS3Timer.Start(); UploadFileToS3(DumpInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + "MiniDump.dmp", Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix); WriteToS3Timer.Stop(); } if (bToDisk) { WriteToDiskTimer.Start(); DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp"); WriteToDiskTimer.Stop(); } } } // Move the video (if it exists) to an alternate store if (VideoFileName != null) { DestinationFolder = Path.Combine(Config.Default.ProcessedVideos, IDThenUnderscore); VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName); FileInfo VideoInfo = new FileInfo(VideoFileName); if (VideoInfo.Exists) { if (bToS3) { WriteToS3Timer.Start(); UploadFileToS3(VideoInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + CrashReporterConstants.VideoFileName, Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix); WriteToS3Timer.Stop(); } if (bToDisk) { CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(DestinationFolder, Config.Default.DiskSpaceAlertPercent); WriteToDiskTimer.Start(); VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName); WriteToDiskTimer.Stop(); } } } string TimeTakenString = string.Empty; if (bToS3) { TimeTakenString = string.Format("S3UploadTime={0:F1} ", WriteToS3Timer.Elapsed.TotalSeconds); } if (bToDisk) { TimeTakenString += string.Format("DiskMoveTime={0:F1} ", WriteToDiskTimer.Elapsed.TotalSeconds); } CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} AddReport: ReportID={1,8} {2}Path={3}", ProcessorIndex, ReportID, TimeTakenString, NewContext.CrashDirectory)); UpdateProcessedReports(); WebAddCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingSucceededEvent); double Ratio = (double)WebAddCounter.TotalEvents / (double)ProcessedReportCount * 100; double AddedPerDay = (double)WebAddCounter.TotalEvents / Timer.Elapsed.TotalDays; CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format( "AddReport: Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4} AddedPerMinute={5:N1}", (int)Ratio, ProcessedReportCount, WebAddCounter.TotalEvents, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay, WebAddCounter.EventsPerSecond * 60)); return(AddReportResult.Added); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "AddReport: " + DirInfo.Name + "\n\n" + Ex, Ex); } return(AddReportResult.Failed); }
/// <summary> Converts WER metadata xml file to the crash context. </summary> private FGenericCrashContext ConvertMetadataToCrashContext(WERReportMetadata Metadata, string NewReportPath) { FGenericCrashContext CrashContext = new FGenericCrashContext(); FReportData ReportData = new FReportData(Metadata, NewReportPath); CrashContext.PrimaryCrashProperties.CrashVersion = (int)ECrashDescVersions.VER_1_NewCrashFormat; CrashContext.PrimaryCrashProperties.ProcessId = 0; CrashContext.PrimaryCrashProperties.CrashGUID = new DirectoryInfo(NewReportPath).Name; // CrashContext.PrimaryCrashProperties.IsInternalBuild // CrashContext.PrimaryCrashProperties.IsPerforceBuild // CrashContext.PrimaryCrashProperties.IsSourceDistribution // CrashContext.PrimaryCrashProperties.SecondsSinceStart CrashContext.PrimaryCrashProperties.GameName = ReportData.GameName; // CrashContext.PrimaryCrashProperties.ExecutableName // CrashContext.PrimaryCrashProperties.BuildConfiguration // CrashContext.PrimaryCrashProperties.PlatformName // CrashContext.PrimaryCrashProperties.PlatformNameIni CrashContext.PrimaryCrashProperties.PlatformFullName = ReportData.Platform; CrashContext.PrimaryCrashProperties.EngineMode = ReportData.EngineMode; CrashContext.PrimaryCrashProperties.EngineVersion = ReportData.GetEngineVersion(); CrashContext.PrimaryCrashProperties.CommandLine = ReportData.CommandLine; // CrashContext.PrimaryCrashProperties.LanguageLCID // Create a locate and get the language. int LanguageCode = 0; int.TryParse(ReportData.Language, out LanguageCode); try { if (LanguageCode > 0) { CultureInfo LanguageCI = new CultureInfo(LanguageCode); CrashContext.PrimaryCrashProperties.AppDefaultLocale = LanguageCI.Name; } } catch (System.Exception) { // Default to en-US CrashContext.PrimaryCrashProperties.AppDefaultLocale = "en-US"; } // CrashContext.PrimaryCrashProperties.IsUE4Release CrashContext.PrimaryCrashProperties.UserName = ReportData.UserName; CrashContext.PrimaryCrashProperties.BaseDir = ReportData.BaseDir; // CrashContext.PrimaryCrashProperties.RootDir CrashContext.PrimaryCrashProperties.MachineId = ReportData.MachineId; CrashContext.PrimaryCrashProperties.EpicAccountId = ReportData.EpicAccountId; // CrashContext.PrimaryCrashProperties.CallStack // CrashContext.PrimaryCrashProperties.SourceContext CrashContext.PrimaryCrashProperties.UserDescription = string.Join("\n", ReportData.UserDescription); CrashContext.PrimaryCrashProperties.ErrorMessage = string.Join("\n", ReportData.ErrorMessage); // CrashContext.PrimaryCrashProperties.CrashDumpMode // CrashContext.PrimaryCrashProperties.Misc.NumberOfCores // CrashContext.PrimaryCrashProperties.Misc.NumberOfCoresIncludingHyperthreads // CrashContext.PrimaryCrashProperties.Misc.Is64bitOperatingSystem // CrashContext.PrimaryCrashProperties.Misc.CPUVendor // CrashContext.PrimaryCrashProperties.Misc.CPUBrand // CrashContext.PrimaryCrashProperties.Misc.PrimaryGPUBrand // CrashContext.PrimaryCrashProperties.Misc.OSVersionMajor // CrashContext.PrimaryCrashProperties.Misc.OSVersionMinor // CrashContext.PrimaryCrashProperties.Misc.AppDiskTotalNumberOfBytes // CrashContext.PrimaryCrashProperties.Misc.AppDiskNumberOfFreeBytes // CrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysical // CrashContext.PrimaryCrashProperties.MemoryStats.TotalVirtual // CrashContext.PrimaryCrashProperties.MemoryStats.PageSize // CrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysicalGB // CrashContext.PrimaryCrashProperties.MemoryStats.AvailablePhysical // CrashContext.PrimaryCrashProperties.MemoryStats.AvailableVirtual // CrashContext.PrimaryCrashProperties.MemoryStats.UsedPhysical // CrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedPhysical // CrashContext.PrimaryCrashProperties.MemoryStats.UsedVirtual // CrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedVirtual // CrashContext.PrimaryCrashProperties.MemoryStats.bIsOOM // CrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationSize // CrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationAlignment CrashContext.PrimaryCrashProperties.TimeofCrash = new DateTime(ReportData.Ticks); CrashContext.PrimaryCrashProperties.bAllowToBeContacted = ReportData.AllowToBeContacted; return(CrashContext); }
double ReadProcessAddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext) { try { string DumpFileName = null; string LogFileName = null; string DiagnosticsFileName = ""; string VideoFileName = null; string CrashContextName = ""; foreach (FileInfo Info in DirInfo.GetFiles()) { switch (Info.Extension.ToLower()) { case ".avi": case ".mp4": VideoFileName = Info.Name; break; case ".runtime-xml": CrashContextName = Info.Name; break; case ".log": LogFileName = Info.Name; break; case ".txt": if (Info.Name == CrashReporterConstants.DiagnosticsFileName) { DiagnosticsFileName = Info.Name; ReadDiagnosticsFile(NewContext); } break; case ".dmp": case ".mdmp": DumpFileName = Info.Name; // Check to see if this has been processed locally FileInfo DiagnosticsInfo = new FileInfo(Path.Combine(DirInfo.FullName, CrashReporterConstants.DiagnosticsFileName)); if (!DiagnosticsInfo.Exists && !NewContext.HasProcessedData()) { ProcessDumpFile(Info.FullName, NewContext); } // Check to see if a new one has been created DiagnosticsInfo.Refresh(); if (DiagnosticsInfo.Exists) { DiagnosticsFileName = DiagnosticsInfo.Name; } break; default: break; } } // Check if the new context has processed data. if (NewContext.HasProcessedData()) { Stopwatch WaitSW = Stopwatch.StartNew(); // Wait for previous task, should not really happen. if (AddReportTask != null) { AddReportTask.Wait(); } double WaitTime = WaitSW.Elapsed.TotalSeconds; //bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName ); // Save/update crash context to the file. NewContext.ToFile(); AddReportTask = Task.Factory.StartNew(() => { bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName); FinalizeReport(bAdded, DirInfo, NewContext); }); return(WaitTime); } else { CrashReporterProcessServicer.WriteFailure("! Invalid : BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory); FinalizeReport(false, DirInfo, NewContext); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("ProcessReport: " + NewContext.CrashDirectory + "\n\n: " + Ex.ToString()); } return(0.0); }
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext) { // Use the latest MinidumpDiagnostics from the main branch. string Win64BinariesDirectory = Path.Combine(Config.Default.DepotRoot, Config.Default.MDDBinariesFolderInDepot); #if DEBUG // Note: the debug executable must be built locally or synced from Perforce manually string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe"); #else string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe"); #endif // Purge logs every 2048 processed crashes string PurgeLogsDays = ProcessedReports % 2048 == 0 ? "2" : "-1"; FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion); // Pass Windows variants (Win32/64) to MinidumpDiagnostics string PlatformVariant = NewContext.PrimaryCrashProperties.PlatformName; if (PlatformVariant != null && NewContext.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS")) { if (NewContext.PrimaryCrashProperties.PlatformFullName.Contains("Win32") || NewContext.PrimaryCrashProperties.PlatformFullName.Contains("32b")) { PlatformVariant = "Win32"; } else if (NewContext.PrimaryCrashProperties.PlatformFullName.Contains("Win64") || NewContext.PrimaryCrashProperties.PlatformFullName.Contains("64b")) { PlatformVariant = "Win64"; } } List <string> MinidumpDiagnosticsParams = new List <string> ( new string[] { "\"" + DiagnosticsPath + "\"", "-BranchName=" + EngineVersion.Branch, // Backward compatibility "-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility "-GameName=" + NewContext.PrimaryCrashProperties.GameName, "-EngineVersion=" + NewContext.PrimaryCrashProperties.EngineVersion, "-PlatformName=" + NewContext.PrimaryCrashProperties.PlatformName, "-PlatformVariantName=" + PlatformVariant, "-bUsePDBCache=true", "-Annotate", "-SyncSymbols", "-SyncMicrosoftSymbols", "-unattended", "-Log=" + NewContext.GetAsFilename() + "-backup-.log", "-DepotIndex=" + Config.Default.DepotIndex, "-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays, "-LOGTIMESINCESTART" } ); LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null; if (Environment.UserInteractive) { CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD; } else { MinidumpDiagnosticsParams.AddRange ( new[] { "-buildmachine", "-forcelogflush" } ); // Write some debugging message. CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams)); } Stopwatch WaitSW = Stopwatch.StartNew(); lock (MinidumpDiagnosticsLock) { Double WaitForLockTime = WaitSW.Elapsed.TotalSeconds; LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray()); if (ReportParser.WaitForExit(MinidumpDiagnosticsTimeoutMilliseconds) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Timed out running MinidumpDiagnostics"); } Double TotalMDDTime = WaitSW.Elapsed.TotalSeconds; CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("ProcessDumpFile: Total MDD exec time {0:N1}s (blocked for {1:N1}s)", TotalMDDTime, WaitForLockTime)); } ReadDiagnosticsFile(NewContext); }
/// <summary> /// A function to add a report to the database, and rename the relevant files. /// Thread-safe. /// </summary> /// <param name="DirInfo">The DirectoryInfo of the report folder.</param> /// <param name="NewContext">The generic crash context.</param> /// <param name="LogFileName">The file name of the log file in the report.</param> /// <param name="DumpFileName">The file name of the minidump in the report.</param> /// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param> private bool AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName) { try { Stopwatch AddReportTime = Stopwatch.StartNew(); // Create an XML representation of a crash string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null); if (CrashDetails == "") { CrashReporterProcessServicer.WriteFailure("! NoDetail: Path=" + NewContext.CrashDirectory); return(false); } // Upload the crash to the database, and retrieve the new row id int ReportID = UploadCrash(CrashDetails); if (ReportID <= 0) { CrashReporterProcessServicer.WriteFailure("! NoUpload: Path=" + NewContext.CrashDirectory); return(false); } string IDThenUnderscore = string.Format("{0}_", ReportID); // Use the row id to name and move the files the way the web site requires string DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedReports, IDThenUnderscore); // Move report files to crash reporter file store if (LogFileName != null) { LogFileName = Path.Combine(DirInfo.FullName, LogFileName); FileInfo LogInfo = new FileInfo(LogFileName); if (LogInfo.Exists) { LogInfo.MoveTo(DestinationFolder + "Launch.log"); } } string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName); FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName); if (CrashContextInfo.Exists) { CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName); } // WERMetaDataName = Path.Combine(DirInfo.FullName, WERMetaDataName); // FileInfo MetaInfo = new FileInfo(WERMetaDataName); // if (MetaInfo.Exists) // { // MetaInfo.MoveTo(DestinationFolder + "WERMeta.xml"); // } if (DumpFileName != null) { DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName); FileInfo DumpInfo = new FileInfo(DumpFileName); if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/) { DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp"); } } // DiagnosticsFileName = Path.Combine(DirInfo.FullName, DiagnosticsFileName); // FileInfo DiagnosticsInfo = new FileInfo(DiagnosticsFileName); // if (DiagnosticsInfo.Exists) // { // DiagnosticsInfo.MoveTo(DestinationFolder + CrashReporterConstants.DiagnosticsFileName); // } // Move the video (if it exists) to an alternate store if (VideoFileName != null) { DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedVideos, IDThenUnderscore); VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName); FileInfo VideoInfo = new FileInfo(VideoFileName); if (VideoInfo.Exists) { VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName); } } CrashReporterProcessServicer.WriteEvent("# WebAdded: ReportID =" + string.Format("{0,7}", ReportID) + " Path=" + NewContext.CrashDirectory); UpdateProcessedReports(); WebAddedReports++; double Ratio = (double)WebAddedReports / (double)ProcessedReports * 100; double AddedPerDay = (double)WebAddedReports / Timer.Elapsed.TotalDays; CrashReporterProcessServicer.WriteEvent(string.Format("Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4}", (int)Ratio, ProcessedReports, WebAddedReports, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay)); return(true); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("AddReport: " + DirInfo.Name + "\n\n" + Ex.ToString()); } return(false); }
/// <summary> /// Create an Xml payload representing a new crash. /// </summary> /// <param name="DirInfo">The DirectoryInfo of the report folder.</param> /// <param name="NewContext">The generic crash context.</param> /// <param name="bHasVideoFile">Whether the report contains a video file.</param> /// <param name="bHasLog">Whether the report contains a log file.</param> /// <param name="bHasMinidump">Whether the report contains a minidump.</param> /// <returns>A string of Xml payload representing the newly found crash report.</returns> private string CreateCrash(DirectoryInfo DirInfo, FGenericCrashContext NewContext, bool bHasVideoFile, bool bHasLog, bool bHasMinidump) { string XmlPayload = ""; try { FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion); // Create a new crash description for uploading CrashDescription NewCrash = new CrashDescription(); NewCrash.BranchName = EngineVersion.GetCleanedBranch(); NewCrash.GameName = NewContext.PrimaryCrashProperties.GameName; NewCrash.Platform = NewContext.PrimaryCrashProperties.PlatformFullName; NewCrash.EngineMode = NewContext.PrimaryCrashProperties.EngineMode; NewCrash.BuildVersion = EngineVersion.VersionNumber; NewCrash.CommandLine = NewContext.PrimaryCrashProperties.CommandLine; NewCrash.BaseDir = NewContext.PrimaryCrashProperties.BaseDir; NewCrash.Language = NewContext.PrimaryCrashProperties.AppDefaultLocale; // // Create a locate and get the system language. // int SystemLanguageCode = 0; // int.TryParse( ReportData.SystemLanguage, out SystemLanguageCode ); // try // { // if( SystemLanguageCode > 0 ) // { // CultureInfo SystemLanguageCI = new CultureInfo( SystemLanguageCode ); // NewCrash.SystemLanguage = SystemLanguageCI.Name; // } // } // catch( System.Exception ) // { // // Default to en-US // NewCrash.SystemLanguage = "en-US"; // } NewCrash.MachineGuid = NewContext.PrimaryCrashProperties.MachineId; // Valid for all kind of builds, previously only for UE4 releases. NewCrash.UserName = NewContext.PrimaryCrashProperties.UserName; // Only valid for non-UE4 releases. NewCrash.EpicAccountId = NewContext.PrimaryCrashProperties.EpicAccountId; // Only valid for UE4 releases. NewCrash.CallStack = NewContext.PrimaryCrashProperties.GetCallstack(); NewCrash.SourceContext = NewContext.PrimaryCrashProperties.GetSourceContext(); NewCrash.ErrorMessage = NewContext.PrimaryCrashProperties.GetErrorMessage(); NewCrash.UserDescription = NewContext.PrimaryCrashProperties.GetUserDescription(); // Iterate through all files and find a file with the earliest date. DateTime TimeOfCrash = DateTime.UtcNow; foreach (var File in DirInfo.GetFiles()) { if (File.CreationTimeUtc < TimeOfCrash) { TimeOfCrash = File.CreationTimeUtc; } } //NewCrash.TimeofCrash = NewContext.PrimaryCrashProperties.TimeofCrash; NewCrash.TimeofCrash = TimeOfCrash; NewCrash.bHasMiniDump = bHasMinidump; NewCrash.bHasLog = bHasLog; NewCrash.bHasVideo = bHasVideoFile; NewCrash.BuiltFromCL = (int)EngineVersion.Changelist; NewCrash.bAllowToBeContacted = NewContext.PrimaryCrashProperties.bAllowToBeContacted; // Ignore any callstack that is shorter than expected, usually the callstack is invalid. if (NewCrash.CallStack.Length <= CrashReporterConstants.MinCallstackDepth) { CrashReporterProcessServicer.WriteFailure("! BadStack: BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory); } else { XmlPayload = XmlHandler.ToXmlString <CrashDescription>(NewCrash); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("CreateCrash: " + Ex.ToString()); } return(XmlPayload); }
/// <summary> /// Finalizes report files. /// Thread-safe. /// </summary> private void FinalizeReport(AddReportResult AddResult, DirectoryInfo DirInfo, FGenericCrashContext NewContext) { // Only remove if we added the report, otherwise leave all files to further investigation. if (AddResult == AddReportResult.Added) { // Remove the report files as we're done with them. CleanReport(DirInfo); } else if (AddResult == AddReportResult.Failed) { MoveReportToInvalid(NewContext.CrashDirectory, ReportQueueBase.GetSafeFilename(NewContext.GetAsFilename())); } else // AddResult == AddReportResult.Cancelled { // Remove report from index of completed reports and leave on disk CrashReporterProcessServicer.ReportIndex.TryRemoveReport(NewContext.CrashDirectory); } }
/// <summary> /// Builds MDD command line args, waits for a MDD task slot, runs MDD and blocks for the result. Writes diag text into DiagnosticsPath folder if successful. /// </summary> /// <param name="DiagnosticsPath">Path of the minidump file</param> /// <param name="Context">The crash context</param> /// <param name="ProcessorIndex">Processor thread index for logging purposes</param> /// <returns>True, if successful</returns> public bool Run(string DiagnosticsPath, FGenericCrashContext Context, int ProcessorIndex) { if (!File.Exists(Config.Default.MDDExecutablePath)) { CrashReporterProcessServicer.WriteEvent("Symbolicator.Run() file not found " + Config.Default.MDDExecutablePath); return(false); } // Use MinidumpDiagnostics from MDDExecutablePath. CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.MDDExecutablePath, Config.Default.DiskSpaceAlertPercent); // Don't purge logs // TODO: make this clear to logs once a day or something (without letting MDD check on every run!) string PurgeLogsDays = "-1"; FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); // Pass Windows variants (Win32/64) to MinidumpDiagnostics string PlatformVariant = Context.PrimaryCrashProperties.PlatformName; if (PlatformVariant != null && Context.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS")) { if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win32") || Context.PrimaryCrashProperties.PlatformFullName.Contains("32b")) { PlatformVariant = "Win32"; } else if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win64") || Context.PrimaryCrashProperties.PlatformFullName.Contains("64b")) { PlatformVariant = "Win64"; } } // Build the absolute log file path for MinidumpDiagnostics string BaseFolder = CrashReporterProcessServicer.SymbolicatorLogFolder; DateTime WriteTime = DateTime.UtcNow; string DateFolder = WriteTime.ToString("yyyy_MM_dd"); string HourFolder = WriteTime.ToString("HH"); string Folder = Path.Combine(BaseFolder, DateFolder, HourFolder); string AbsLogPath = Path.Combine(Folder, Context.GetAsFilename() + ".log"); Directory.CreateDirectory(Folder); List <string> MinidumpDiagnosticsParams = new List <string> ( new string[] { "\"" + DiagnosticsPath + "\"", "-BranchName=" + EngineVersion.Branch, // Backward compatibility "-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility "-GameName=" + Context.PrimaryCrashProperties.GameName, "-EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion, "-BuildVersion=" + (string.IsNullOrWhiteSpace(Context.PrimaryCrashProperties.BuildVersion) ? string.Format("{0}-CL-{1}", EngineVersion.Branch, EngineVersion.Changelist).Replace('/', '+') : Context.PrimaryCrashProperties.BuildVersion), "-PlatformName=" + Context.PrimaryCrashProperties.PlatformName, "-PlatformVariantName=" + PlatformVariant, "-bUsePDBCache=true", "-PDBCacheDepotRoot=" + Config.Default.DepotRoot, "-PDBCachePath=" + Config.Default.MDDPDBCachePath, "-PDBCacheSizeGB=" + Config.Default.MDDPDBCacheSizeGB, "-PDBCacheMinFreeSpaceGB=" + Config.Default.MDDPDBCacheMinFreeSpaceGB, "-PDBCacheFileDeleteDays=" + Config.Default.MDDPDBCacheFileDeleteDays, "-MutexPDBCache", "-PDBCacheLock=CrashReportProcessPDBCacheLock", "-NoTrimCallstack", "-SyncSymbols", "-NoP4Symbols", "-ForceUsePDBCache", "-MutexSourceSync", "-SourceSyncLock=CrashReportProcessSourceSyncLock", "-SyncMicrosoftSymbols", "-unattended", "-AbsLog=" + AbsLogPath, "-DepotIndex=" + Config.Default.DepotIndex, "-P4User="******"-P4Client=" + Config.Default.P4Client, "-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays + ",[LogFiles]:MaxLogFilesOnDisk=-1", "-LOGTIMESINCESTART" } ); LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null; if (Environment.UserInteractive) { CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD; } else { MinidumpDiagnosticsParams.AddRange ( new[] { "-buildmachine" } ); // Write some debugging message. CrashReporterProcessServicer.WriteMDD("MinidumpDiagnostics Params: " + String.Join(" ", MinidumpDiagnosticsParams)); } Task <bool> NewSymbolicatorTask = Task.FromResult(false); Stopwatch WaitSW = Stopwatch.StartNew(); double WaitForLockTime = 0.0; lock (Tasks) { int TaskIdx = Task.WaitAny(Tasks); Tasks[TaskIdx] = NewSymbolicatorTask = Task <bool> .Factory.StartNew(() => { LaunchProcess ReportParser = new LaunchProcess(Config.Default.MDDExecutablePath, Path.GetDirectoryName(Config.Default.MDDExecutablePath), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray()); return(ReportParser.WaitForExit(Config.Default.MDDTimeoutMinutes * 1000 * 60) == EWaitResult.Ok); }); WaitForLockTime = WaitSW.Elapsed.TotalSeconds; } NewSymbolicatorTask.Wait(); double TotalMDDTime = WaitSW.Elapsed.TotalSeconds; double MDDRunTime = TotalMDDTime - WaitForLockTime; CrashReporterProcessServicer.StatusReporter.AddToMeanCounter(StatusReportingPerfMeanNames.MinidumpDiagnostics, (int)(MDDRunTime * 1000)); CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Symbolicator.Run: Thread blocked for {0:N1}s then MDD ran for {1:N1}s", WaitForLockTime, MDDRunTime)); return(NewSymbolicatorTask.Result); }
/// <summary>If available, will read CrashContext.runtime-xml.</summary> public void ReadCrashContextIfAvailable() { try { //\\epicgames.net\Root\Projects\Paragon\QA_CrashReports bool bHasCrashContext = HasCrashContextFile(); Plugins = new List <string>(); if (bHasCrashContext) { //_CrashContext = FGenericCrashContext.FromFile(SitePath + GetCrashContextUrl()); _CrashContext = FGenericCrashContext.FromUrl(GetCrashContextUrl()); bool bTest = _CrashContext != null && !string.IsNullOrEmpty(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation); if (bTest) { _bUseFullMinidumpPath = true; //Some temporary code to redirect to the new file location for fulldumps for paragon. //Consider removing this once fulldumps stop appearing in the old location. if (_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation.ToLower() .Contains("\\\\epicgames.net\\root\\dept\\gameqa\\paragon\\paragon_launcherCrashdumps")) { //Files from old versions of the client may end up in the old location. Check for files there first. if (!System.IO.Directory.Exists(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation)) { var suffix = _CrashContext.PrimaryCrashProperties.FullCrashDumpLocation.Substring("\\\\epicgames.net\\root\\dept\\gameqa\\paragon\\paragon_launcherCrashdumps".Length); _CrashContext.PrimaryCrashProperties.FullCrashDumpLocation = String.Format("\\\\epicgames.net\\Root\\Projects\\Paragon\\QA_CrashReports{0}", suffix); //If the file doesn't exist in the new location either then don't use the full minidump path. _bUseFullMinidumpPath = System.IO.Directory.Exists(_CrashContext.PrimaryCrashProperties.FullCrashDumpLocation); } } //End of temporary code. FLogger.Global.WriteEvent("ReadCrashContextIfAvailable " + _CrashContext.PrimaryCrashProperties.FullCrashDumpLocation + " is " + _bUseFullMinidumpPath); } if (_CrashContext.UntypedEnabledPlugins != null) { XmlNode[] nodes = _CrashContext.UntypedEnabledPlugins as XmlNode[]; if (nodes != null) { foreach (var node in nodes) { try { dynamic data = Json.Decode(node.InnerText); String printName = data.FriendlyName + " " + data.VersionName; System.Diagnostics.Debug.WriteLine(printName); Plugins.Add(printName); } catch (Exception e) { Debug.WriteLine("Error trying to add a plugin entry: " + e.ToString()); } } } } } } catch (Exception Ex) { Debug.WriteLine("Exception in ReadCrashContextIfAvailable: " + Ex.ToString()); } }
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext) { // Use the latest MinidumpDiagnostics from the main branch. string Win64BinariesDirectory = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine", "Binaries", "Win64"); #if DEBUG // Note: the debug executable must be built locally or synced from Perforce manually string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe"); //string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe"); #else string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe"); #endif // Purge logs every 2048 processed crashes string PurgeLogsDays = ProcessedReports % 2048 == 0 ? "2" : "-1"; FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion); List <string> MinidumpDiagnosticsParams = new List <string> ( new string[] { "\"" + DiagnosticsPath + "\"", "-BranchName=" + EngineVersion.Branch, // Backward compatibility "-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility "-GameName=" + NewContext.PrimaryCrashProperties.GameName, "-EngineVersion=" + NewContext.PrimaryCrashProperties.EngineVersion, "-bUsePDBCache=true", "-Annotate", "-SyncSymbols", "-SyncMicrosoftSymbols", "-unattended", "-Log=" + NewContext.GetAsFilename() + "-backup-.log", "-DepotIndex=" + Properties.Settings.Default.DepotIndex, "-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays, "-LOGTIMESINCESTART" } ); LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null; if (Environment.UserInteractive) { CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD; } else { MinidumpDiagnosticsParams.AddRange ( new[] { "-buildmachine", "-forcelogflush" } ); // Write some debugging message. CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams)); } LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray()); if (ReportParser.WaitForExit(MinidumpDiagnosticsTimeoutSeconds * 1000) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure("ProcessDumpFile: Timed out running MinidumpDiagnostics"); } ReadDiagnosticsFile(NewContext); }
/// <summary> Enqueues a new crash. </summary> void EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return; } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bFromWER = false; if (Context == null || !Context.HasProcessedData()) { WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { FReportData ReportData = new FReportData(MetaData, NewReportPath); Context = ConvertMetadataToCrashContext(MetaData, NewReportPath); bFromWER = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); } else { if (GenericContext != null && GenericContext.PrimaryCrashProperties.ErrorMessage.Length > 0) { // Get error message from the crash context and fix value in the metadata. Context.PrimaryCrashProperties.ErrorMessage = GenericContext.PrimaryCrashProperties.ErrorMessage; } Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // If based on WER, save to the file. if (bFromWER && GenericContext == null) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteFailure("% NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.MoveReportToInvalid(NewReportPath, Context.GetAsFilename()); return; } if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteFailure("% UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.CleanReport(NewReportPath); return; } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteFailure("! BROKEN0 : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.CleanReport(NewReportPath); return; } lock (NewReportsLock) { NewCrashContexts.Add(Context); } CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); } }
/// <summary> /// Main processing thread. /// </summary> /// <remarks>All exceptions are caught and written to the event log.</remarks> private void ProcessNewReports() { // Use the latest MinidumpDiagnostics from the main branch. string Win64BinariesDirectory = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine", "Binaries", "Win64"); #if !DEBUG string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics*"); if (!SyncRequiredFiles(MinidumpDiagnosticsName)) { return; } #endif DateTime LastCleanupTime = DateTime.Now; int LastDay = DateTime.UtcNow.Day; var Cancel = CancelSource.Token; ProcessorTask = Task.Factory.StartNew(() => { while (!Cancel.IsCancellationRequested) { try { foreach (var Queue in Watcher.ReportQueues) { FGenericCrashContext NewContext = null; if (Queue.TryDequeueReport(out NewContext)) { ProcessReport(NewContext); int CurrentDay = DateTime.UtcNow.Day; if (CurrentDay > LastDay) { // Check the log and create a new one for a new day. CrashReporterProcessServicer.Log.CreateNewLogFile(); LastDay = CurrentDay; } // The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest break; } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("ProcessNewReports: " + Ex.ToString()); } // Remove all folders older than n days to prevent unbounded growth if ((DateTime.Now - LastCleanupTime) > TimeSpan.FromMinutes(15)) { CleanRepository(); LastCleanupTime = DateTime.Now; } // Don't use the CPU if we don't need. Thread.Sleep(100); } }); }
/// <summary> Enqueues a new crash. </summary> private void EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return; } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bFromWER = false; if (Context == null || !Context.HasProcessedData()) { WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { FReportData ReportData = new FReportData(MetaData, NewReportPath); ConvertMetadataToCrashContext(MetaData, NewReportPath, ref Context); bFromWER = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); } else { Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // Added data from WER, save to the crash context file. if (bFromWER) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion; } else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName; } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg"; } lock (NewReportsLock) { NewCrashContexts.Enqueue(Context); } EnqueueCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingConstants.QueuedEvent); CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); } }
double ReadProcessAddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext) { try { string DumpFileName = null; string LogFileName = null; string DiagnosticsFileName = ""; string VideoFileName = null; string CrashContextName = ""; foreach (FileInfo Info in DirInfo.GetFiles()) { switch (Info.Extension.ToLower()) { case ".avi": case ".mp4": VideoFileName = Info.Name; break; case ".runtime-xml": CrashContextName = Info.Name; break; case ".log": LogFileName = Info.Name; break; case ".txt": if (string.Compare(Info.Name, CrashReporterConstants.DiagnosticsFileName, true) == 0) { DiagnosticsFileName = Info.Name; ReadDiagnosticsFile(NewContext); } break; case ".dmp": case ".mdmp": DumpFileName = Info.Name; // Check to see if this has been processed locally FileInfo DiagnosticsInfo = new FileInfo(Path.Combine(DirInfo.FullName, CrashReporterConstants.DiagnosticsFileName)); if (!DiagnosticsInfo.Exists && !NewContext.HasProcessedData()) { ProcessDumpFile(Info.FullName, NewContext); } // Check to see if a new one has been created DiagnosticsInfo.Refresh(); if (DiagnosticsInfo.Exists) { DiagnosticsFileName = DiagnosticsInfo.Name; } break; default: break; } } // Check if the new context has processed data. if (!NewContext.HasProcessedData()) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "% Warning no callstack or error msg : BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory); NewContext.PrimaryCrashProperties.ProcessorFailedMessage = "No callstack or error message. Diagnostics missing or failed."; } Stopwatch WaitSW = Stopwatch.StartNew(); // Wait for previous task, should not really happen. int AddReportTaskSlot = WaitForFreeAddReportTask(); CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Starting AddReportTask running on slot {0} of {1} ({2} active)", AddReportTaskSlot, Config.Default.AddReportsPerProcessor, GetActiveAddReportTasks())); double WaitTime = WaitSW.Elapsed.TotalSeconds; //bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName ); // Save/update crash context to the file. NewContext.ToFile(); AddReportTasks[AddReportTaskSlot] = Task.Factory.StartNew(() => { AddReportResult Result = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName); FinalizeReport(Result, DirInfo, NewContext); }); return(WaitTime); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessReport: " + NewContext.CrashDirectory + "\n\n: " + Ex, Ex); } return(0.0); }