void ProcessDumpFile(string DumpPath, FGenericCrashContext NewContext) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DumpPath); if (TrySimulateSymbolication(NewContext)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Simulated symbolication (skipped MDD) " + NewContext.CrashDirectory); } else if (CrashReporterProcessServicer.Symbolicator.Run(DumpPath, NewContext, ProcessorIndex)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DumpPath); ReadDiagnosticsFile(NewContext); } else { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DumpPath); } if (string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.CallStack)) { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationFailedEvent); } else { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationSucceededEvent); } }
/// <summary> /// A thread to watch for new crash reports landing. /// </summary> /// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks> void Start() { CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:"); var Settings = Config.Default; if (!string.IsNullOrEmpty(Settings.InternalLandingZone)) { if (System.IO.Directory.Exists(Settings.InternalLandingZone)) { ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority)", Settings.InternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority) is not accessible", Settings.InternalLandingZone)); } } #if !DEBUG if (!string.IsNullOrEmpty(Settings.ExternalLandingZone)) { if (System.IO.Directory.Exists(Settings.ExternalLandingZone)) { ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0}", Settings.ExternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} is not accessible", Settings.ExternalLandingZone)); } } #endif //!DEBUG // Init queue entries in StatusReporter foreach (var Queue in ReportQueues) { CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath); } var Cancel = CancelSource.Token; WatcherTask = Task.Factory.StartNew(async() => { DateTime LastQueueSizeReport = DateTime.MinValue; while (!Cancel.IsCancellationRequested) { // Check the landing zones for new reports DateTime StartTime = DateTime.Now; foreach (var Queue in ReportQueues) { int QueueSize = Queue.CheckForNewReports(); CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize); } TimeSpan TimeTaken = DateTime.Now - StartTime; CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds)); await Task.Delay(60000, Cancel); } }); }
private static int UnpackRecordsFromDelimitedProtocolBuffers(Stream ProtocolBufferStream, string InLandingZone, string ReadableRequestString) { var UnpackedRecordCount = 0; DataRouterConsumer Consumer = new DataRouterConsumer(); // Expect one or more pairs of varint size + protocol buffer in the stream while (ProtocolBufferStream.Position < ProtocolBufferStream.Length) { DataRouterConsumer.ProtocolBufferRecord Message; if (!Consumer.TryParse(ProtocolBufferStream, out Message)) { string FailString = "! Protocol buffer parse fail in UnpackRecordsFromDelimitedProtocolBuffers(): " + ReadableRequestString; FailString += '\n' + Consumer.LastError; CrashReporterProcessServicer.WriteFailure(FailString); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent); break; } if (DecompressDataRouterContent(Message.Payload, InLandingZone)) { UnpackedRecordCount++; } } return(UnpackedRecordCount); }
protected static bool WriteIncomingFile(BinaryReader BinaryData, int FileIndex, string DestinationFolderPath) { try { Int32 CurrentFileIndex = BinaryData.ReadInt32(); if (CurrentFileIndex != FileIndex) { CrashReporterProcessServicer.WriteFailure("! WriteIncomingFile index mismatch: Required=" + FileIndex + " Read=" + CurrentFileIndex); return(false); } string Filename = FBinaryReaderHelper.ReadFixedSizeString(BinaryData); string SafeFilename = GetSafeFilename(Filename); Int32 FiledataLength = BinaryData.ReadInt32(); byte[] Filedata = BinaryData.ReadBytes(FiledataLength); Directory.CreateDirectory(DestinationFolderPath); File.WriteAllBytes(Path.Combine(DestinationFolderPath, SafeFilename), Filedata); return(true); } catch (Exception Ex) { throw new CrashReporterException(string.Format("! WriteIncomingFile failed writing FileIndex={0} FolderPath={1}", FileIndex, DestinationFolderPath), Ex); } }
/// <summary> Looks for the WER metadata xml file, if found, will return a new instance of the WERReportMetadata. </summary> private WERReportMetadata FindMetadata(string NewReportPath) { WERReportMetadata MetaData = null; // Just to verify that the report is still there. DirectoryInfo DirInfo = new DirectoryInfo(NewReportPath); if (!DirInfo.Exists) { CrashReporterProcessServicer.WriteFailure("FindMetadata: Directory not found " + NewReportPath); } else { // Check to see if we wish to suppress processing of this report. foreach (var Info in DirInfo.GetFiles("*.xml")) { var MetaDataToCheck = XmlHandler.ReadXml <WERReportMetadata>(Info.FullName); if (CheckMetaData(MetaDataToCheck)) { MetaData = MetaDataToCheck; break; } } } return(MetaData); }
private void TryGetNewS3Crashes(int CrashCount) { int NewCrashCount = 0; while (NewCrashCount < CrashCount) { string SQSRecord = "<unset>"; try { if (!DequeueRecordSQS(out SQSRecord)) { // Queue empty break; } var RecordPair = SQSRecord.Split(','); if (RecordPair.Length != 2) { CrashReporterProcessServicer.WriteFailure("TryGetNewS3Crashes: bad SQS message was " + SQSRecord); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent); continue; } string S3BucketName = RecordPair[0]; string S3Key = RecordPair[1]; string ReadableRequestString = "Bucket=" + S3BucketName + " Key=" + S3Key; var ObjectRequest = new GetObjectRequest { BucketName = S3BucketName, Key = S3Key }; using (Stream ProtocolBufferStream = new MemoryStream()) { using (GetObjectResponse ObjectResponse = S3Client.GetObject(ObjectRequest)) { using (Stream ResponseStream = ObjectResponse.ResponseStream) { if (!TryDecompResponseStream(ResponseStream, ProtocolBufferStream)) { CrashReporterProcessServicer.WriteFailure("! GZip fail in DecompResponseStream(): " + ReadableRequestString); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent); continue; } } } NewCrashCount += UnpackRecordsFromDelimitedProtocolBuffers(ProtocolBufferStream, LandingZone, ReadableRequestString); } } catch (Exception ex) { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent); CrashReporterProcessServicer.WriteException("TryGetNewS3Crashes: failure during processing SQS record " + SQSRecord + "\n" + ex); } } }
/// <summary> /// Static init for one-time init jobs /// </summary> static ReportProcessor() { #if !DEBUG if (!SyncRequiredFiles()) { CrashReporterProcessServicer.WriteFailure("ReportProcessor: failed to sync files from Perforce"); } #endif }
public void ReadFromFile() { if (!IsEnabled) { return; } try { Index = new Dictionary <string, DateTime>(); if (!File.Exists(Filepath)) { if (File.Exists(Filepath + ".backup")) { CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read ReportIndex from {0}. Attempting to read from {1}", Filepath, Filepath + ".backup")); CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.ReadFromFile.UsingBackup", string.Format("Failed to read ReportIndex from {0}. Using backup.", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes); File.Move(Filepath + ".backup", Filepath); } else { CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read ReportIndex from {0}. Generating new one.", Filepath)); File.Create(Filepath).Close(); } } using (var Reader = File.OpenText(Filepath)) { string ItemStringRaw; while ((ItemStringRaw = Reader.ReadLine()) != null) { string ItemString = ItemStringRaw.Trim(); if (!string.IsNullOrWhiteSpace(ItemString)) { KeyValuePair <string, DateTime> NewItem; if (!TryParseItemString(ItemString, out NewItem)) { CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read line from ReportIndex: {0}.", ItemString)); continue; } Index.Add(NewItem.Key, NewItem.Value); } } } LastFlush = DateTime.UtcNow; } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("Failed to read ReportIndex from {0}. Exception was: {1}", Filepath, Ex), Ex); CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.ReadFromFile", string.Format("Failed to read ReportIndex from {0}", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes); } }
/// <summary> /// Static init for one-time init jobs /// </summary> static ReportProcessor() { #if !DEBUG if (Config.Default.bSyncMinidumpDiagnostics) { if (!SyncMinidumpDiagnostics()) { CrashReporterProcessServicer.WriteFailure("ReportProcessor: failed to sync files from Perforce"); } } #endif }
/// <summary> /// Decompresses a compressed crash report. /// </summary> unsafe private bool DecompressReport(string CompressedReportPath, FCompressedCrashInformation Meta) { string ReportPath = Path.GetDirectoryName(CompressedReportPath); using (FileStream FileStream = File.OpenRead(CompressedReportPath)) { Int32 UncompressedSize = Meta.GetUncompressedSize(); Int32 CompressedSize = Meta.GetCompressedSize(); if (FileStream.Length != CompressedSize) { return(false); } byte[] UncompressedBufferArray = new byte[UncompressedSize]; byte[] CompressedBufferArray = new byte[CompressedSize]; FileStream.Read(CompressedBufferArray, 0, CompressedSize); fixed(byte *UncompressedBufferPtr = UncompressedBufferArray, CompressedBufferPtr = CompressedBufferArray) { bool bResult = UE4UncompressMemoryZLIB((IntPtr)UncompressedBufferPtr, UncompressedSize, (IntPtr)CompressedBufferPtr, CompressedSize); if (!bResult) { CrashReporterProcessServicer.WriteFailure("! ZLibFail: Path=" + ReportPath); return(false); } } MemoryStream MemoryStream = new MemoryStream(UncompressedBufferArray, false); BinaryReader BinaryData = new BinaryReader(MemoryStream); for (int FileIndex = 0; FileIndex < Meta.GetNumberOfFiles(); FileIndex++) { Int32 CurrentFileIndex = BinaryData.ReadInt32(); if (CurrentFileIndex != FileIndex) { CrashReporterProcessServicer.WriteFailure("! ReadFail: Required=" + FileIndex + " Read=" + CurrentFileIndex); return(false); } string Filename = FBinaryReaderHelper.ReadFixedSizeString(BinaryData); Int32 FiledataLength = BinaryData.ReadInt32(); byte[] Filedata = BinaryData.ReadBytes(FiledataLength); File.WriteAllBytes(Path.Combine(ReportPath, Filename), Filedata); } } return(true); }
/// <summary> /// Call the web service function with an Xml payload to add a crash to the database. /// </summary> /// <param name="Payload">Xml representation of a new crash to upload.</param> /// <returns>The database id of the newly added row.</returns> private int UploadCrash(string Payload) { int NewID = -1; try { // Simple suppression by blanking out the URL for local testing if (Config.Default.CrashReportWebSite.Length > 0) { bool bDebug = false; string RequestString; if (!bDebug) { RequestString = "http://" + Config.Default.CrashReportWebSite + ":80/Crashes/AddCrash/-1"; } else { RequestString = "http://localhost:80/Crashes/AddCrash/-1"; } string ErrorMessage = string.Empty; for (int Retry = 0; Retry < 3; ++Retry) { string ResponseString = SimpleWebRequest.GetWebServiceResponse(RequestString, Payload); if (ResponseString.Length > 0) { // Convert response into a string CrashReporterResult Result = XmlHandler.FromXmlString <CrashReporterResult>(ResponseString); if (Result.ID > 0) { NewID = Result.ID; break; } ErrorMessage = Result.Message; } Thread.Sleep(200); } if (NewID == -1) { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "UploadCrash: " + ErrorMessage); } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "UploadCrash: " + Ex.ToString()); } return(NewID); }
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DiagnosticsPath); if (CrashReporterProcessServicer.Symbolicator.Run(DiagnosticsPath, NewContext, ProcessorIndex)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DiagnosticsPath); ReadDiagnosticsFile(NewContext); } else { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DiagnosticsPath); } }
/// <summary> /// A thread to watch for new crash reports landing. /// </summary> /// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks> void Start() { CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:"); var Settings = Properties.Settings.Default; if (!string.IsNullOrEmpty(Settings.InternalLandingZone)) { if (System.IO.Directory.Exists(Settings.InternalLandingZone)) { ReportQueues.Add(new ReportQueue(Settings.InternalLandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority)", Settings.InternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority) is not accessible", Settings.InternalLandingZone)); } } #if !DEBUG if (!string.IsNullOrEmpty(Settings.ExternalLandingZone)) { if (System.IO.Directory.Exists(Settings.ExternalLandingZone)) { ReportQueues.Add(new ReportQueue(Settings.ExternalLandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0}", Settings.ExternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} is not accessible", Settings.ExternalLandingZone)); } } #endif //!DEBUG var Cancel = CancelSource.Token; WatcherTask = Task.Factory.StartNew(async() => { while (!Cancel.IsCancellationRequested) { // Check the landing zones for new reports foreach (var Queue in ReportQueues) { Queue.CheckForNewReports(); } await Task.Delay(60000, Cancel); } }); }
/// <summary> /// Look for new report folders and add them to the publicly available thread-safe queue /// </summary> public void CheckForNewReports() { try { var NewReportPath = ""; var ReportsInLandingZone = new ReportIdSet(); // Check the landing zones for new reports DirectoryInfo DirInfo = new DirectoryInfo(LandingZone); // Couldn't find a landing zone, skip and try again later. // Crash receiver will recreate them if needed. if (!DirInfo.Exists) { CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone); return; } // Add any new reports foreach (var SubDirInfo in DirInfo.GetDirectories()) { try { if (Directory.Exists(SubDirInfo.FullName)) { NewReportPath = SubDirInfo.FullName; ReportsInLandingZone.Add(NewReportPath); if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath)) { EnqueueNewReport(NewReportPath); } } } catch (System.Exception Ex) { CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex.ToString()); ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks); } } //CrashReporterProcessServicer.WriteEvent( string.Format( "ReportsInLandingZone={0}, ReportsInLandingZoneLastTimeWeChecked={1}", ReportsInLandingZone.Count, ReportsInLandingZoneLastTimeWeChecked.Count ) ); ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone; } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex.ToString()); } }
/// <summary> /// Decompresses a compressed crash report. /// </summary> unsafe static protected bool DecompressReport(string CompressedReportPath, FCompressedCrashInformation Meta) { string ReportPath = Path.GetDirectoryName(CompressedReportPath); using (FileStream FileStream = File.OpenRead(CompressedReportPath)) { Int32 UncompressedSize = Meta.GetUncompressedSize(); Int32 CompressedSize = Meta.GetCompressedSize(); if (FileStream.Length != CompressedSize) { return(false); } byte[] UncompressedBufferArray = new byte[UncompressedSize]; byte[] CompressedBufferArray = new byte[CompressedSize]; FileStream.Read(CompressedBufferArray, 0, CompressedSize); fixed(byte *UncompressedBufferPtr = UncompressedBufferArray, CompressedBufferPtr = CompressedBufferArray) { Int32 UncompressResult = NativeMethods.UE4UncompressMemoryZLIB((IntPtr)UncompressedBufferPtr, UncompressedSize, (IntPtr)CompressedBufferPtr, CompressedSize); if (UncompressResult < 0) { CrashReporterProcessServicer.WriteFailure("! DecompressReport() failed in UE4UncompressMemoryZLIB() with " + NativeMethods.GetUncompressError(UncompressResult)); CrashReporterProcessServicer.WriteFailure("! ZLibFail in DecompressReport(): Path=" + ReportPath); return(false); } } using (BinaryReader BinaryData = new BinaryReader(new MemoryStream(UncompressedBufferArray, false))) { for (int FileIndex = 0; FileIndex < Meta.GetNumberOfFiles(); FileIndex++) { if (!WriteIncomingFile(BinaryData, FileIndex, ReportPath)) { CrashReporterProcessServicer.WriteFailure("! DecompressReport() failed to write file index " + FileIndex + " Path=" + ReportPath); return(false); } } } } return(true); }
protected virtual bool GetCrashesFromLandingZone(out DirectoryInfo[] OutDirectories) { // Check the landing zones for new reports DirectoryInfo DirInfo = new DirectoryInfo(LandingZone); // Couldn't find a landing zone, skip and try again later. // Crash receiver will recreate them if needed. if (!DirInfo.Exists) { CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone); OutDirectories = new DirectoryInfo[0]; return(false); } OutDirectories = DirInfo.GetDirectories().OrderBy(dirinfo => dirinfo.CreationTimeUtc).ToArray(); return(true); }
/// <summary> /// Call the web service function with an Xml payload to add a crash to the database. /// </summary> /// <param name="Payload">Xml representation of a new crash to upload.</param> /// <returns>The database id of the newly added row.</returns> private int UploadCrash(string Payload) { int NewID = -1; try { // Simple suppression by blanking out the URL for local testing if (Properties.Settings.Default.CrashReportWebSite.Length > 0) { bool bDebug = false; string RequestString; if (!bDebug) { RequestString = "http://" + Properties.Settings.Default.CrashReportWebSite + ":80/Crashes/AddCrash/-1"; } else { RequestString = "http://localhost:80/Crashes/AddCrash/-1"; } string ResponseString = SimpleWebRequest.GetWebServiceResponse(RequestString, Payload); if (ResponseString.Length > 0) { // Convert response into a string CrashReporterResult Result = XmlHandler.FromXmlString <CrashReporterResult>(ResponseString); if (Result.ID > 0) { NewID = Result.ID; } else { CrashReporterProcessServicer.WriteFailure("UploadCrash: " + Result.Message); } } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("UploadCrash: " + Ex.ToString()); } return(NewID); }
/// <summary> /// Sync the MinidumpDiagnostics binary and the engine config files to #head. /// </summary> /// <returns>true if all the files synced without issue, false otherwise.</returns> /// <remarks>As MinidumpDiagnostics is synced to #head, it requires the engine config files that match to run properly.</remarks> private static bool SyncMinidumpDiagnostics() { // Use the latest MinidumpDiagnostics from the main branch. CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.DepotRoot, Config.Default.DiskSpaceAlertPercent); string UserString = "-u " + Config.Default.P4User; string ClientString = "-c " + Config.Default.P4Client; string SyncBinariesString = Path.Combine(Config.Default.DepotRoot, Config.Default.SyncBinariesFromDepot); using (var MDDSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, UserString, ClientString, "sync", SyncBinariesString)) { if (MDDSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure("Failed to sync MinidumpDiagnostics " + SyncBinariesString); return(false); } } string SyncConfigString = Path.Combine(Config.Default.DepotRoot, Config.Default.SyncConfigFromDepot); using (var ConfigSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, UserString, ClientString, "sync", SyncConfigString)) { if (ConfigSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure("Failed to sync config files " + SyncConfigString); return(false); } } string SyncThirdPartyString = Path.Combine(Config.Default.DepotRoot, Config.Default.SyncThirdPartyFromDepot); // Required by Perforce using (var MiscSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, UserString, ClientString, "sync", SyncThirdPartyString)) { if (MiscSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure("Failed to sync OpenSSL files " + SyncThirdPartyString); return(false); } } return(true); }
/// <summary> /// Sync the MinidumpDiagnostics binary and the engine config files to #head. /// </summary> /// <param name="MinidumpDiagnosticsName">The name of the binary to sync to process the minidump.</param> /// <returns>true if all the files synced without issue, false otherwise.</returns> /// <remarks>As MinidumpDiagnostics is synced to #head, it requires the engine config files that match to run properly.</remarks> private bool SyncRequiredFiles(string MinidumpDiagnosticsName) { using (var MDDSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, "sync", MinidumpDiagnosticsName)) { if (MDDSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure("Failed to sync MinidumpDiagnostics " + MinidumpDiagnosticsName); return(false); } } string ConfigFiles = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine/Config/..."); using (var ConfigSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, "sync", ConfigFiles)) { if (ConfigSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure("Failed to sync config files " + ConfigFiles); return(false); } } string SSLFiles = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine/Binaries/ThirdParty/OpenSSL/..."); // Required by Perforce using (var MiscSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, "sync", SSLFiles)) { if (MiscSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure("Failed to sync OpenSSL files " + SSLFiles); return(false); } } return(true); }
/// <summary> Enqueues a new crash. </summary> void EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return; } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bFromWER = false; if (Context == null || !Context.HasProcessedData()) { WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { FReportData ReportData = new FReportData(MetaData, NewReportPath); Context = ConvertMetadataToCrashContext(MetaData, NewReportPath); bFromWER = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); } else { if (GenericContext != null && GenericContext.PrimaryCrashProperties.ErrorMessage.Length > 0) { // Get error message from the crash context and fix value in the metadata. Context.PrimaryCrashProperties.ErrorMessage = GenericContext.PrimaryCrashProperties.ErrorMessage; } Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // If based on WER, save to the file. if (bFromWER && GenericContext == null) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteFailure("% NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.MoveReportToInvalid(NewReportPath, Context.GetAsFilename()); return; } if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteFailure("% UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.CleanReport(NewReportPath); return; } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteFailure("! BROKEN0 : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.CleanReport(NewReportPath); return; } lock (NewReportsLock) { NewCrashContexts.Add(Context); } CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); } }
/// <summary> /// A function to add a report to the database, and rename the relevant files. /// Thread-safe. /// </summary> /// <param name="DirInfo">The DirectoryInfo of the report folder.</param> /// <param name="NewContext">The generic crash context.</param> /// <param name="LogFileName">The file name of the log file in the report.</param> /// <param name="DumpFileName">The file name of the minidump in the report.</param> /// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param> private AddReportResult AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName) { try { Stopwatch AddReportTime = Stopwatch.StartNew(); // Create an XML representation of a crash string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null); if (CrashDetails == "") { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "! CreateCrash no payload: Path=" + NewContext.CrashDirectory); return(AddReportResult.Failed); } // Upload the crash to the database, and retrieve the new row id int ReportID = UploadCrash(CrashDetails); if (ReportID <= 0) { if (CancelSource.IsCancellationRequested) { // Cancelled upload CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "CancelUpload: Path=" + NewContext.CrashDirectory); return(AddReportResult.Cancelled); } else { // Upload failed CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "! NoUpload: Path=" + NewContext.CrashDirectory); return(AddReportResult.Failed); } } bool bToS3 = Config.Default.CrashFilesToAWS && CrashReporterProcessServicer.OutputAWS.IsS3Valid; bool bToDisk = Config.Default.CrashFilesToDisk; // Use the row id to name and move the files the way the web site requires string IDThenUnderscore = string.Format("{0}_", ReportID); int ReportIDSegment = (ReportID / 10000) * 10000; string S3IDPrefix = string.Format("/{0}/{1}/{1}_", ReportIDSegment, ReportID); string DestinationFolder = Path.Combine(Config.Default.ProcessedReports, IDThenUnderscore); Stopwatch WriteToS3Timer = new Stopwatch(); Stopwatch WriteToDiskTimer = new Stopwatch(); if (bToDisk) { CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(DestinationFolder, Config.Default.DiskSpaceAlertPercent); } // Save log file if (LogFileName != null) { LogFileName = Path.Combine(DirInfo.FullName, LogFileName); FileInfo LogInfo = new FileInfo(LogFileName); if (LogInfo.Exists) { if (bToS3) { WriteToS3Timer.Start(); UploadFileToS3(LogInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + "Launch.log", Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix); WriteToS3Timer.Stop(); } if (bToDisk) { WriteToDiskTimer.Start(); LogInfo.MoveTo(DestinationFolder + "Launch.log"); WriteToDiskTimer.Stop(); } } } // Save crash context file string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName); FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName); if (CrashContextInfo.Exists) { if (bToS3) { WriteToS3Timer.Start(); UploadFileToS3(CrashContextInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + FGenericCrashContext.CrashContextRuntimeXMLName, false); WriteToS3Timer.Stop(); } if (bToDisk) { WriteToDiskTimer.Start(); CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName); WriteToDiskTimer.Stop(); } } if (DumpFileName != null) { DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName); FileInfo DumpInfo = new FileInfo(DumpFileName); if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/) { if (bToS3) { WriteToS3Timer.Start(); UploadFileToS3(DumpInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + "MiniDump.dmp", Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix); WriteToS3Timer.Stop(); } if (bToDisk) { WriteToDiskTimer.Start(); DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp"); WriteToDiskTimer.Stop(); } } } // Move the video (if it exists) to an alternate store if (VideoFileName != null) { DestinationFolder = Path.Combine(Config.Default.ProcessedVideos, IDThenUnderscore); VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName); FileInfo VideoInfo = new FileInfo(VideoFileName); if (VideoInfo.Exists) { if (bToS3) { WriteToS3Timer.Start(); UploadFileToS3(VideoInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + CrashReporterConstants.VideoFileName, Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix); WriteToS3Timer.Stop(); } if (bToDisk) { CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(DestinationFolder, Config.Default.DiskSpaceAlertPercent); WriteToDiskTimer.Start(); VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName); WriteToDiskTimer.Stop(); } } } string TimeTakenString = string.Empty; if (bToS3) { TimeTakenString = string.Format("S3UploadTime={0:F1} ", WriteToS3Timer.Elapsed.TotalSeconds); } if (bToDisk) { TimeTakenString += string.Format("DiskMoveTime={0:F1} ", WriteToDiskTimer.Elapsed.TotalSeconds); } CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} AddReport: ReportID={1,8} {2}Path={3}", ProcessorIndex, ReportID, TimeTakenString, NewContext.CrashDirectory)); UpdateProcessedReports(); WebAddCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingSucceededEvent); double Ratio = (double)WebAddCounter.TotalEvents / (double)ProcessedReportCount * 100; double AddedPerDay = (double)WebAddCounter.TotalEvents / Timer.Elapsed.TotalDays; CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format( "AddReport: Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4} AddedPerMinute={5:N1}", (int)Ratio, ProcessedReportCount, WebAddCounter.TotalEvents, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay, WebAddCounter.EventsPerSecond * 60)); return(AddReportResult.Added); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "AddReport: " + DirInfo.Name + "\n\n" + Ex, Ex); } return(AddReportResult.Failed); }
// From CrashUpload.cpp /* * struct FCompressedCrashFile : FNoncopyable * { * int32 CurrentFileIndex; // 4 bytes for file index * FString Filename; // 4 bytes for length + 260 bytes for char data * TArray<uint8> Filedata; // 4 bytes for length + N bytes for data * } */ /// <summary> Enqueues a new crash. </summary> private bool EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return(false); } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bContextDirty = false; WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { if (Context == null) { // Missing crash context FReportData ReportData = new FReportData(MetaData, NewReportPath); ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context); bContextDirty = true; } else if (!Context.HasProcessedData()) { // Missing data - try to get from WER metadata FReportData ReportData = new FReportData(MetaData, NewReportPath); GetErrorMessageFromMetadata(ReportData, Context); bContextDirty = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // Added data from WER, save to the crash context file. if (bContextDirty) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion; Context.ToFile(); } else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName; Context.ToFile(); } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg"; Context.ToFile(); } // Check static reports index for duplicated reports // This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router) // This can be safely discontinued when all crashes come from the same SQS // This DOES NOT scale to multiple CRP instances // ReportIndex can be disabled by leaving the path empty in config if (!CrashReporterProcessServicer.ReportIndex.TryAddNewReport(ReportName)) { // Crash report not accepted by index CrashReporterProcessServicer.WriteEvent(string.Format( "EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } if (ShouldDecimateNextReport()) { CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } lock (NewReportsLock) { NewCrashContexts.Enqueue(Context); } EnqueueCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent); CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); return(true); }
/// <summary> /// Create thread to watch for new crash reports landing. /// </summary> /// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks> void Init() { CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:"); var Settings = Config.Default; if (!string.IsNullOrEmpty(Settings.DataRouterLandingZone)) { if (System.IO.Directory.Exists(Settings.DataRouterLandingZone)) { ReportQueues.Add(new DataRouterReportQueue("DataRouter Crashes", Settings.DataRouterLandingZone, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (all crashes from data router)", Settings.DataRouterLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (all crashes from data router) is not accessible", Settings.DataRouterLandingZone)); } } if (!string.IsNullOrEmpty(Settings.InternalLandingZone)) { if (System.IO.Directory.Exists(Settings.InternalLandingZone)) { ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone, StatusReportingEventNames.ProcessingStartedReceiverEvent, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority (legacy))", Settings.InternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority (legacy)) is not accessible", Settings.InternalLandingZone)); } } if (!string.IsNullOrEmpty(Settings.PS4LandingZone)) { if (System.IO.Directory.Exists(Settings.PS4LandingZone)) { ReportQueues.Add(new ReceiverReportQueue("PS4 Crashes", Settings.PS4LandingZone, StatusReportingEventNames.ProcessingStartedPS4Event, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, PS4 non-retail hardware)", Settings.PS4LandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, PS4 non-retail hardware) is not accessible", Settings.PS4LandingZone)); } } #if !DEBUG if (!string.IsNullOrEmpty(Settings.ExternalLandingZone)) { if (System.IO.Directory.Exists(Settings.ExternalLandingZone)) { ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone, StatusReportingEventNames.ProcessingStartedReceiverEvent, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (legacy)", Settings.ExternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (legacy) is not accessible", Settings.ExternalLandingZone)); } } #endif //!DEBUG // Init queue entries in StatusReporter foreach (var Queue in ReportQueues) { CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath); } var Cancel = CancelSource.Token; WatcherTask = new Task(async() => { DateTime LastQueueSizeReport = DateTime.MinValue; while (!Cancel.IsCancellationRequested) { // Check the landing zones for new reports DateTime StartTime = DateTime.Now; foreach (var Queue in ReportQueues) { int QueueSize = Queue.CheckForNewReports(); CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize); } TimeSpan TimeTaken = DateTime.Now - StartTime; CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds)); await Task.Delay(30000, Cancel); } }); }
/// <summary> /// Look for new report folders and add them to the publicly available thread-safe queue. /// </summary> public override int CheckForNewReports() { try { if (QueueCount >= SkipQueueRefreshMin) { CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: skipped busy queue size {0} in {1}", QueueCount, LandingZone)); return(LastQueueSizeOnDisk); } var NewReportPath = ""; var ReportsInLandingZone = new ReportIdSet(); // Check the landing zones for new reports DirectoryInfo DirInfo = new DirectoryInfo(LandingZone); // Couldn't find a landing zone, skip and try again later. // Crash receiver will recreate them if needed. if (!DirInfo.Exists) { CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone); return(LastQueueSizeOnDisk); } var DirectoriesInLandingZone = DirInfo.GetDirectories().OrderBy(dirinfo => dirinfo.CreationTimeUtc).ToArray(); LastQueueSizeOnDisk = DirectoriesInLandingZone.Length; int EnqueuedCount = 0; CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} reports in {1}", DirectoriesInLandingZone.Length, LandingZone)); // Add any new reports for (int DirIndex = 0; DirIndex < DirectoriesInLandingZone.Length && QueueCount < QueueMax; DirIndex++) { var SubDirInfo = DirectoriesInLandingZone[DirIndex]; try { if (Directory.Exists(SubDirInfo.FullName)) { NewReportPath = SubDirInfo.FullName; ReportsInLandingZone.Add(NewReportPath); if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath)) { EnqueueNewReport(NewReportPath); EnqueuedCount++; } } } catch (System.Exception Ex) { CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex.ToString()); ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks); } } //CrashReporterProcessServicer.WriteEvent( string.Format( "ReportsInLandingZone={0}, ReportsInLandingZoneLastTimeWeChecked={1}", ReportsInLandingZone.Count, ReportsInLandingZoneLastTimeWeChecked.Count ) ); ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone; CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} enqueued to queue size {1} from {2}", EnqueuedCount, QueueCount, LandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: Enqueue rate {0:N1}/minute from {1}", EnqueueCounter.EventsPerSecond * 60, LandingZone)); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex.ToString()); } return(LastQueueSizeOnDisk); }
/// <summary> Enqueues a new crash. </summary> private void EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return; } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bFromWER = false; if (Context == null || !Context.HasProcessedData()) { WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { FReportData ReportData = new FReportData(MetaData, NewReportPath); ConvertMetadataToCrashContext(MetaData, NewReportPath, ref Context); bFromWER = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); } else { Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // Added data from WER, save to the crash context file. if (bFromWER) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion; } else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName; } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg"; } lock (NewReportsLock) { NewCrashContexts.Enqueue(Context); } EnqueueCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingConstants.QueuedEvent); CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); } }
/// <summary> /// Create an Xml payload representing a new crash. /// </summary> /// <param name="DirInfo">The DirectoryInfo of the report folder.</param> /// <param name="NewContext">The generic crash context.</param> /// <param name="bHasVideoFile">Whether the report contains a video file.</param> /// <param name="bHasLog">Whether the report contains a log file.</param> /// <param name="bHasMinidump">Whether the report contains a minidump.</param> /// <returns>A string of Xml payload representing the newly found crash report.</returns> private string CreateCrash(DirectoryInfo DirInfo, FGenericCrashContext NewContext, bool bHasVideoFile, bool bHasLog, bool bHasMinidump) { string XmlPayload = ""; try { FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion); // Create a new crash description for uploading CrashDescription NewCrash = new CrashDescription(); NewCrash.BranchName = EngineVersion.GetCleanedBranch(); NewCrash.GameName = NewContext.PrimaryCrashProperties.GameName; NewCrash.Platform = NewContext.PrimaryCrashProperties.PlatformFullName; NewCrash.EngineMode = NewContext.PrimaryCrashProperties.EngineMode; NewCrash.BuildVersion = EngineVersion.VersionNumber; NewCrash.CommandLine = NewContext.PrimaryCrashProperties.CommandLine; NewCrash.BaseDir = NewContext.PrimaryCrashProperties.BaseDir; NewCrash.Language = NewContext.PrimaryCrashProperties.AppDefaultLocale; // // Create a locate and get the system language. // int SystemLanguageCode = 0; // int.TryParse( ReportData.SystemLanguage, out SystemLanguageCode ); // try // { // if( SystemLanguageCode > 0 ) // { // CultureInfo SystemLanguageCI = new CultureInfo( SystemLanguageCode ); // NewCrash.SystemLanguage = SystemLanguageCI.Name; // } // } // catch( System.Exception ) // { // // Default to en-US // NewCrash.SystemLanguage = "en-US"; // } NewCrash.MachineGuid = NewContext.PrimaryCrashProperties.MachineId; // Valid for all kind of builds, previously only for UE4 releases. NewCrash.UserName = NewContext.PrimaryCrashProperties.UserName; // Only valid for non-UE4 releases. NewCrash.EpicAccountId = NewContext.PrimaryCrashProperties.EpicAccountId; // Only valid for UE4 releases. NewCrash.CallStack = NewContext.PrimaryCrashProperties.GetCallstack(); NewCrash.SourceContext = NewContext.PrimaryCrashProperties.GetSourceContext(); NewCrash.ErrorMessage = NewContext.PrimaryCrashProperties.GetErrorMessage(); NewCrash.UserDescription = NewContext.PrimaryCrashProperties.GetUserDescription(); // Iterate through all files and find a file with the earliest date. DateTime TimeOfCrash = DateTime.UtcNow; foreach (var File in DirInfo.GetFiles()) { if (File.CreationTimeUtc < TimeOfCrash) { TimeOfCrash = File.CreationTimeUtc; } } //NewCrash.TimeofCrash = NewContext.PrimaryCrashProperties.TimeofCrash; NewCrash.TimeofCrash = TimeOfCrash; NewCrash.bHasMiniDump = bHasMinidump; NewCrash.bHasLog = bHasLog; NewCrash.bHasVideo = bHasVideoFile; NewCrash.BuiltFromCL = (int)EngineVersion.Changelist; NewCrash.bAllowToBeContacted = NewContext.PrimaryCrashProperties.bAllowToBeContacted; // Ignore any callstack that is shorter than expected, usually the callstack is invalid. if (NewCrash.CallStack.Length <= CrashReporterConstants.MinCallstackDepth) { CrashReporterProcessServicer.WriteFailure("! BadStack: BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory); } else { XmlPayload = XmlHandler.ToXmlString <CrashDescription>(NewCrash); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("CreateCrash: " + Ex.ToString()); } return(XmlPayload); }
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext) { // Use the latest MinidumpDiagnostics from the main branch. string Win64BinariesDirectory = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine", "Binaries", "Win64"); #if DEBUG // Note: the debug executable must be built locally or synced from Perforce manually string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe"); //string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe"); #else string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe"); #endif // Purge logs every 2048 processed crashes string PurgeLogsDays = ProcessedReports % 2048 == 0 ? "2" : "-1"; FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion); List <string> MinidumpDiagnosticsParams = new List <string> ( new string[] { "\"" + DiagnosticsPath + "\"", "-BranchName=" + EngineVersion.Branch, // Backward compatibility "-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility "-GameName=" + NewContext.PrimaryCrashProperties.GameName, "-EngineVersion=" + NewContext.PrimaryCrashProperties.EngineVersion, "-bUsePDBCache=true", "-Annotate", "-SyncSymbols", "-SyncMicrosoftSymbols", "-unattended", "-Log=" + NewContext.GetAsFilename() + "-backup-.log", "-DepotIndex=" + Properties.Settings.Default.DepotIndex, "-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays, "-LOGTIMESINCESTART" } ); LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null; if (Environment.UserInteractive) { CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD; } else { MinidumpDiagnosticsParams.AddRange ( new[] { "-buildmachine", "-forcelogflush" } ); // Write some debugging message. CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams)); } LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray()); if (ReportParser.WaitForExit(MinidumpDiagnosticsTimeoutSeconds * 1000) == EWaitResult.TimedOut) { CrashReporterProcessServicer.WriteFailure("ProcessDumpFile: Timed out running MinidumpDiagnostics"); } ReadDiagnosticsFile(NewContext); }
/// <summary> /// A function to add a report to the database, and rename the relevant files. /// Thread-safe. /// </summary> /// <param name="DirInfo">The DirectoryInfo of the report folder.</param> /// <param name="NewContext">The generic crash context.</param> /// <param name="LogFileName">The file name of the log file in the report.</param> /// <param name="DumpFileName">The file name of the minidump in the report.</param> /// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param> private bool AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName) { try { Stopwatch AddReportTime = Stopwatch.StartNew(); // Create an XML representation of a crash string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null); if (CrashDetails == "") { CrashReporterProcessServicer.WriteFailure("! NoDetail: Path=" + NewContext.CrashDirectory); return(false); } // Upload the crash to the database, and retrieve the new row id int ReportID = UploadCrash(CrashDetails); if (ReportID <= 0) { CrashReporterProcessServicer.WriteFailure("! NoUpload: Path=" + NewContext.CrashDirectory); return(false); } string IDThenUnderscore = string.Format("{0}_", ReportID); // Use the row id to name and move the files the way the web site requires string DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedReports, IDThenUnderscore); // Move report files to crash reporter file store if (LogFileName != null) { LogFileName = Path.Combine(DirInfo.FullName, LogFileName); FileInfo LogInfo = new FileInfo(LogFileName); if (LogInfo.Exists) { LogInfo.MoveTo(DestinationFolder + "Launch.log"); } } string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName); FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName); if (CrashContextInfo.Exists) { CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName); } // WERMetaDataName = Path.Combine(DirInfo.FullName, WERMetaDataName); // FileInfo MetaInfo = new FileInfo(WERMetaDataName); // if (MetaInfo.Exists) // { // MetaInfo.MoveTo(DestinationFolder + "WERMeta.xml"); // } if (DumpFileName != null) { DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName); FileInfo DumpInfo = new FileInfo(DumpFileName); if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/) { DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp"); } } // DiagnosticsFileName = Path.Combine(DirInfo.FullName, DiagnosticsFileName); // FileInfo DiagnosticsInfo = new FileInfo(DiagnosticsFileName); // if (DiagnosticsInfo.Exists) // { // DiagnosticsInfo.MoveTo(DestinationFolder + CrashReporterConstants.DiagnosticsFileName); // } // Move the video (if it exists) to an alternate store if (VideoFileName != null) { DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedVideos, IDThenUnderscore); VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName); FileInfo VideoInfo = new FileInfo(VideoFileName); if (VideoInfo.Exists) { VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName); } } CrashReporterProcessServicer.WriteEvent("# WebAdded: ReportID =" + string.Format("{0,7}", ReportID) + " Path=" + NewContext.CrashDirectory); UpdateProcessedReports(); WebAddedReports++; double Ratio = (double)WebAddedReports / (double)ProcessedReports * 100; double AddedPerDay = (double)WebAddedReports / Timer.Elapsed.TotalDays; CrashReporterProcessServicer.WriteEvent(string.Format("Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4}", (int)Ratio, ProcessedReports, WebAddedReports, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay)); return(true); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("AddReport: " + DirInfo.Name + "\n\n" + Ex.ToString()); } return(false); }
private static bool DecompressDataRouterContent(byte[] CompressedBufferArray, string InLandingZone) { // Decompress to landing zone byte[] UncompressedBufferArray = new byte[Config.Default.MaxUncompressedS3RecordSize]; int UncompressedSize = NativeMethods.UncompressMemoryZlib(UncompressedBufferArray, CompressedBufferArray); if (UncompressedSize < 0) { string FailString = "! DecompressDataRouterContent() failed in UncompressMemoryZlib() with " + NativeMethods.GetZlibError(UncompressedSize); CrashReporterProcessServicer.WriteFailure(FailString); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent); return(false); } using (BinaryReader BinaryData = new BinaryReader(new MemoryStream(UncompressedBufferArray, 0, UncompressedSize, false))) { char[] MarkerChars = BinaryData.ReadChars(3); if (MarkerChars[0] == 'C' && MarkerChars[1] == 'R' && MarkerChars[2] == '1') { CrashHeader CrashHeader = DataRouterReportQueue.CrashHeader.ParseCrashHeader(BinaryData); // Create safe directory name and then create the directory on disk string CrashFolderName = GetSafeFilename(CrashHeader.DirectoryName); string CrashFolderPath = Path.Combine(InLandingZone, CrashFolderName); // Early check for duplicate processed report lock (ReportIndexLock) { if (CrashReporterProcessServicer.ReportIndex.ContainsReport(CrashFolderName)) { // Crash report not accepted by index CrashReporterProcessServicer.WriteEvent(string.Format( "DataRouterReportQueue: Duplicate report skipped early {0} in a DataRouterReportQueue", CrashFolderPath)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected); return(false); // this isn't an error so don't set error message } } // Create target folder int TryIndex = 1; while (Directory.Exists(CrashFolderPath)) { CrashFolderPath = Path.Combine(InLandingZone, string.Format("{0}_DUPE{1:D3}", CrashFolderName, TryIndex++)); } Directory.CreateDirectory(CrashFolderPath); if (UncompressedSize != CrashHeader.UncompressedSize) { CrashReporterProcessServicer.WriteEvent( string.Format( "DecompressDataRouterContent() warning UncompressedSize mismatch (embedded={0}, actual={1}) Path={2}", CrashHeader.UncompressedSize, UncompressedSize, CrashFolderPath)); } if (!ParseCrashFiles(BinaryData, CrashHeader.FileCount, CrashFolderPath)) { string FailString = "! DecompressDataRouterContent() failed to write files Path=" + CrashFolderPath; CrashReporterProcessServicer.WriteFailure(FailString); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent); return(false); } } else { #if ALLOWOLDCLIENTDATA // Early Data Router upload format was broken. // Should be [CR1][CrashHeader][File][File][File][File]... // Actually [Undefined CrashHeader][File][File][File][File]...[CrashHeader] // Seek to end minus header size BinaryData.BaseStream.Position = UncompressedSize - DataRouterReportQueue.CrashHeader.FixedSize; var CrashHeader = DataRouterReportQueue.CrashHeader.ParseCrashHeader(BinaryData); // Create safe directory name and then create the directory on disk string CrashFolderName = GetSafeFilename(CrashHeader.DirectoryName); string CrashFolderPath = Path.Combine(InLandingZone, CrashFolderName); // Early check for duplicate processed report lock (ReportIndexLock) { if (CrashReporterProcessServicer.ReportIndex.ContainsReport(CrashFolderName)) { // Crash report not accepted by index CrashReporterProcessServicer.WriteEvent(string.Format( "DataRouterReportQueue: Duplicate report skipped early {0} in a DataRouterReportQueue", CrashFolderPath)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected); return(false); // this isn't an error so don't set error message } } // Create target folder int TryIndex = 1; while (Directory.Exists(CrashFolderPath)) { CrashFolderPath = Path.Combine(InLandingZone, string.Format("{0}_DUPE{1:D3}", CrashFolderName, TryIndex++)); } Directory.CreateDirectory(CrashFolderPath); if (UncompressedSize != CrashHeader.UncompressedSize + CrashHeader.FixedSize) { CrashReporterProcessServicer.WriteEvent( string.Format( "DecompressDataRouterContent() warning UncompressedSize mismatch (embedded={0}, actual={1}) Path={2}", CrashHeader.UncompressedSize, UncompressedSize, CrashFolderPath)); } // Seek to start of files (header size in from start) BinaryData.BaseStream.Position = CrashHeader.FixedSize; if (!ParseCrashFiles(BinaryData, CrashHeader.FileCount, CrashFolderPath)) { string FailString = "! DecompressDataRouterContent() failed to write files Path=" + CrashFolderPath; CrashReporterProcessServicer.WriteFailure(FailString); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent); return(false); } #else ErrorMessage = "! DecompressDataRouterContent() failed to read invalid data format. Corrupt or old format data received Path=" + CrashFolderPath; return(false); #endif } } return(true); }
double ReadProcessAddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext) { try { string DumpFileName = null; string LogFileName = null; string DiagnosticsFileName = ""; string VideoFileName = null; string CrashContextName = ""; foreach (FileInfo Info in DirInfo.GetFiles()) { switch (Info.Extension.ToLower()) { case ".avi": case ".mp4": VideoFileName = Info.Name; break; case ".runtime-xml": CrashContextName = Info.Name; break; case ".log": LogFileName = Info.Name; break; case ".txt": if (Info.Name == CrashReporterConstants.DiagnosticsFileName) { DiagnosticsFileName = Info.Name; ReadDiagnosticsFile(NewContext); } break; case ".dmp": case ".mdmp": DumpFileName = Info.Name; // Check to see if this has been processed locally FileInfo DiagnosticsInfo = new FileInfo(Path.Combine(DirInfo.FullName, CrashReporterConstants.DiagnosticsFileName)); if (!DiagnosticsInfo.Exists && !NewContext.HasProcessedData()) { ProcessDumpFile(Info.FullName, NewContext); } // Check to see if a new one has been created DiagnosticsInfo.Refresh(); if (DiagnosticsInfo.Exists) { DiagnosticsFileName = DiagnosticsInfo.Name; } break; default: break; } } // Check if the new context has processed data. if (NewContext.HasProcessedData()) { Stopwatch WaitSW = Stopwatch.StartNew(); // Wait for previous task, should not really happen. if (AddReportTask != null) { AddReportTask.Wait(); } double WaitTime = WaitSW.Elapsed.TotalSeconds; //bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName ); // Save/update crash context to the file. NewContext.ToFile(); AddReportTask = Task.Factory.StartNew(() => { bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName); FinalizeReport(bAdded, DirInfo, NewContext); }); return(WaitTime); } else { CrashReporterProcessServicer.WriteFailure("! Invalid : BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory); FinalizeReport(false, DirInfo, NewContext); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("ProcessReport: " + NewContext.CrashDirectory + "\n\n: " + Ex.ToString()); } return(0.0); }