/// <summary> /// Moves specified report to the directory where are stored invalid reports. /// Thread-safe. /// </summary> public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename) { try { DirectoryInfo DirInfo = new DirectoryInfo(ReportName); // Rename the report directory, so we should be able to quickly find the invalid reports. Directory.CreateDirectory(Properties.Settings.Default.InvalidReportsDirectory); string CleanFilename = String.Concat(ReportNameAsFilename.Split(Path.GetInvalidFileNameChars())); string DestinationDirectory = Path.Combine(Properties.Settings.Default.InvalidReportsDirectory, CleanFilename); Directory.CreateDirectory(DestinationDirectory); // Copy all files from the source directory. We can't use MoveTo due to different disc location. foreach (FileInfo File in DirInfo.GetFiles()) { string DestinationFilepath = Path.Combine(DestinationDirectory, File.Name); File.CopyTo(DestinationFilepath, true); } DirInfo.Delete(true); CrashReporterProcessServicer.WriteEvent(string.Format("Moved to {0}", DestinationDirectory)); UpdateProcessedReports(); } catch (System.Exception Ex) { CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex.ToString()); } }
public bool TryAddNewReport(string ReportKey) { if (!IsEnabled) { return(true); } lock (IndexLock) { if (Index.ContainsKey(ReportKey)) { return(false); } Index.Add(ReportKey, DateTime.UtcNow); } lock (FileLock) { if (LastFlush.Date < DateTime.UtcNow.Date) { CrashReporterProcessServicer.WriteEvent("ReportIndex.TryAddNewReport flushing index to disk on schedule."); WriteToFile(); } } return(true); }
/// <summary> /// A thread to watch for new crash reports landing. /// </summary> /// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks> void Start() { CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:"); var Settings = Config.Default; if (!string.IsNullOrEmpty(Settings.InternalLandingZone)) { if (System.IO.Directory.Exists(Settings.InternalLandingZone)) { ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority)", Settings.InternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority) is not accessible", Settings.InternalLandingZone)); } } #if !DEBUG if (!string.IsNullOrEmpty(Settings.ExternalLandingZone)) { if (System.IO.Directory.Exists(Settings.ExternalLandingZone)) { ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0}", Settings.ExternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} is not accessible", Settings.ExternalLandingZone)); } } #endif //!DEBUG // Init queue entries in StatusReporter foreach (var Queue in ReportQueues) { CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath); } var Cancel = CancelSource.Token; WatcherTask = Task.Factory.StartNew(async() => { DateTime LastQueueSizeReport = DateTime.MinValue; while (!Cancel.IsCancellationRequested) { // Check the landing zones for new reports DateTime StartTime = DateTime.Now; foreach (var Queue in ReportQueues) { int QueueSize = Queue.CheckForNewReports(); CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize); } TimeSpan TimeTaken = DateTime.Now - StartTime; CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds)); await Task.Delay(60000, Cancel); } }); }
/// <summary> /// Shutdown: stop the thread. Will request stop first if RequestStop() hasn't been called. Then blocks until the processor thread exits. /// </summary> public void Dispose() { if (!CancelSource.IsCancellationRequested) { CancelSource.Cancel(); } CrashReporterProcessServicer.WriteEvent("Shutdown: Stopping ReportProcessor thread..."); ProcessorTask.Wait(); ProcessorTask.Dispose(); CrashReporterProcessServicer.WriteEvent("Shutdown: ReportProcessor thread stopped."); CrashReporterProcessServicer.WriteEvent("Shutdown: Stopping ReportProcessor's AddReport threads..."); foreach (Task AddReportTask in AddReportTasks) { if (AddReportTask != null) { AddReportTask.Wait(); AddReportTask.Dispose(); } } CrashReporterProcessServicer.WriteEvent("Shutdown: AddReport threads stopped."); CancelSource.Dispose(); }
public void AlertOnLowDisk(string Filepath, float AlertThresholdPercent) { try { string Drive; if (!StorageSpaceHelper.TryGetDriveLetter(Filepath, out Drive)) { throw new CrashReporterException("Failed to get drive letter for path " + Filepath); } Int64 FreeSpace; float FreePercent; if (StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent)) { if (FreePercent < AlertThresholdPercent) { Alert("AlertOnLowDisk" + Drive, "Low disk space warning on " + Drive + " =>> " + GetDiskSpaceString(FreeSpace, FreePercent), 3 * Config.Default.SlackAlertRepeatMinimumMinutes); } } else { CrashReporterProcessServicer.WriteEvent("Failed to read disk space for " + Drive); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("AlertOnLowDisk failed: " + Ex, Ex); } }
void ProcessDumpFile(string DumpPath, FGenericCrashContext NewContext) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DumpPath); if (TrySimulateSymbolication(NewContext)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Simulated symbolication (skipped MDD) " + NewContext.CrashDirectory); } else if (CrashReporterProcessServicer.Symbolicator.Run(DumpPath, NewContext, ProcessorIndex)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DumpPath); ReadDiagnosticsFile(NewContext); } else { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DumpPath); } if (string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.CallStack)) { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationFailedEvent); } else { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationSucceededEvent); } }
/// <summary> /// Moves specified report to the directory where are stored invalid reports. /// Thread-safe. /// </summary> public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename) { try { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingFailedEvent); DirectoryInfo DirInfo = new DirectoryInfo(ReportName); // Rename the report directory, so we should be able to quickly find the invalid reports. CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.InvalidReportsDirectory, Config.Default.DiskSpaceAlertPercent); Directory.CreateDirectory(Config.Default.InvalidReportsDirectory); string DestinationDirectory = Path.Combine(Config.Default.InvalidReportsDirectory, ReportNameAsFilename); Directory.CreateDirectory(DestinationDirectory); // Copy all files from the source directory. We can't use MoveTo due to different disc location. foreach (FileInfo File in DirInfo.GetFiles()) { string DestinationFilepath = Path.Combine(DestinationDirectory, File.Name); File.CopyTo(DestinationFilepath, true); } DirInfo.Delete(true); CrashReporterProcessServicer.WriteEvent(string.Format("Moved to {0}", DestinationDirectory)); UpdateProcessedReports(); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex); } }
/// <summary> /// Clear out old MDD logs /// </summary> /// <param name="NumDays">Number of days worth of logs to keep</param> public static void CleanOutOldLogs(int NumDays) { DateTime DeleteTime = DateTime.UtcNow - TimeSpan.FromDays(NumDays); try { DirectoryInfo BaseFolder = new DirectoryInfo(CrashReporterProcessServicer.SymbolicatorLogFolder); if (BaseFolder.Exists) { foreach (DirectoryInfo SubFolder in BaseFolder.EnumerateDirectories()) { if (!SubFolder.EnumerateFiles("*", SearchOption.AllDirectories).Any(x => x.LastWriteTimeUtc > DeleteTime)) { try { SubFolder.Delete(true); } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", SubFolder.FullName, Ex.Message)); } } } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs: {0}", Ex.Message)); } }
/// <summary> /// Clear out old MDD logs /// </summary> /// <param name="NumDays">Number of days worth of logs to keep</param> public static void CleanOutOldLogs(int NumDays) { DateTime DeleteTime = DateTime.UtcNow - TimeSpan.FromDays(NumDays); try { DirectoryInfo BaseFolder = new DirectoryInfo(CrashReporterProcessServicer.SymbolicatorLogFolder); if (BaseFolder.Exists) { foreach (DirectoryInfo SubFolder in BaseFolder.EnumerateDirectories()) { if (!SubFolder.EnumerateFiles("*", SearchOption.AllDirectories).Any(x => x.LastWriteTimeUtc > DeleteTime)) { try { SubFolder.Delete(true); } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", SubFolder.FullName, Ex.Message)); } } } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs: {0}", Ex.Message)); } try { DirectoryInfo BaseFolder = new DirectoryInfo(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Config.Default.MDDExecutablePath), "..", "..", "Programs", "MinidumpDiagnostics", "Saved", "Logs"))); if (BaseFolder.Exists) { foreach (FileInfo File in BaseFolder.EnumerateFiles("*", SearchOption.AllDirectories)) { try { if (File.LastWriteTimeUtc < DeleteTime) { File.Delete(); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", File.FullName, Ex.Message)); } } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs from program folder: {0}", Ex.Message)); } }
/// <summary> /// Main processing thread. /// </summary> /// <remarks>All exceptions are caught and written to the event log.</remarks> private void Init() { ProcessorTask = new Task(() => { // parse out hte list of blacklisted games. string[] GameNamesToBlacklist = (Config.Default.GameNamesToBlacklist == null ? "" : Config.Default.GameNamesToBlacklist).ToLowerInvariant().Split(','); while (!CancelSource.IsCancellationRequested) { try { bool bIdle = true; foreach (var Queue in Watcher.ReportQueues) { FGenericCrashContext NewContext = null; if (Queue.TryDequeueReport(out NewContext)) { bool bIsBlacklistedGame = GameNamesToBlacklist.Contains(NewContext.PrimaryCrashProperties.GameName.ToLowerInvariant()); // if it's blacklisted, skip it, else process it. if (bIsBlacklistedGame) { CrashReporterProcessServicer.WriteEvent(string.Format("Discarding crash from blacklisted GameName '{0}'", NewContext.PrimaryCrashProperties.GameName)); // delete the report from disk since we don't care about it. FinalizeReport(AddReportResult.Added, new DirectoryInfo(NewContext.CrashDirectory), NewContext); } else { ProcessReport(NewContext); } // The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest bIdle = false; break; } } if (bIdle) { // Don't use the CPU if we don't need. Thread.Sleep(1000); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessNewReports: " + Ex, Ex); } TickStatic(Watcher); } }); }
/// <summary> /// Delete a report directory. /// Thread-safe. /// </summary> /// <param name="DirInfo">The directory to delete</param> public static void CleanReport(DirectoryInfo DirInfo) { const int MaxRetries = 3; bool bWriteException = true; for (int Retry = 0; Retry < MaxRetries; ++Retry) { try { if (!DirInfo.Exists) { break; } foreach (FileInfo Info in DirInfo.GetFiles()) { Info.IsReadOnly = false; Info.Attributes = FileAttributes.Normal; } DirInfo.Delete(true /* delete contents */); // Random failures to delete with no exception seen regularly - retry DirInfo.Refresh(); if (DirInfo.Exists) { CrashReporterProcessServicer.WriteEvent("CleanReport: Failed to delete folder without an Exception " + DirInfo); Thread.Sleep(500); continue; } break; } catch (Exception Ex) { if (bWriteException) { CrashReporterProcessServicer.WriteException("CleanReport: " + Ex, Ex); bWriteException = false; } } System.Threading.Thread.Sleep(100); } DirInfo.Refresh(); if (DirInfo.Exists) { CrashReporterProcessServicer.WriteEvent(string.Format("CleanReport: Failed to delete folder {0} after {1} retries", DirInfo, MaxRetries)); } }
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DiagnosticsPath); if (CrashReporterProcessServicer.Symbolicator.Run(DiagnosticsPath, NewContext, ProcessorIndex)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DiagnosticsPath); ReadDiagnosticsFile(NewContext); } else { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DiagnosticsPath); } }
/// <summary> /// A thread to watch for new crash reports landing. /// </summary> /// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks> void Start() { CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:"); var Settings = Properties.Settings.Default; if (!string.IsNullOrEmpty(Settings.InternalLandingZone)) { if (System.IO.Directory.Exists(Settings.InternalLandingZone)) { ReportQueues.Add(new ReportQueue(Settings.InternalLandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority)", Settings.InternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority) is not accessible", Settings.InternalLandingZone)); } } #if !DEBUG if (!string.IsNullOrEmpty(Settings.ExternalLandingZone)) { if (System.IO.Directory.Exists(Settings.ExternalLandingZone)) { ReportQueues.Add(new ReportQueue(Settings.ExternalLandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0}", Settings.ExternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} is not accessible", Settings.ExternalLandingZone)); } } #endif //!DEBUG var Cancel = CancelSource.Token; WatcherTask = Task.Factory.StartNew(async() => { while (!Cancel.IsCancellationRequested) { // Check the landing zones for new reports foreach (var Queue in ReportQueues) { Queue.CheckForNewReports(); } await Task.Delay(60000, Cancel); } }); }
/// <summary> Tries to dequeue a report from the list. </summary> public bool TryDequeueReport(out FGenericCrashContext Context) { lock (NewReportsLock) { if (NewCrashContexts.Count > 0) { Context = NewCrashContexts.Dequeue(); DequeueCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(QueueProcessingStartedEventName); CrashReporterProcessServicer.WriteEvent(string.Format("- Dequeued: {0:N1}/minute BuiltFromCL={1,7} Path={2}", DequeueCounter.EventsPerSecond * 60, Context.PrimaryCrashProperties.EngineVersion, Context.CrashDirectory)); return(true); } } Context = null; return(false); }
/// <summary> /// Shutdown: stop the thread /// </summary> public void Dispose() { // Cancel the task and wait for it to quit CancelSource.Cancel(); CrashReporterProcessServicer.WriteEvent("Shutdown: Stopping ReportWatcher thread..."); WatcherTask.Wait(); WatcherTask.Dispose(); CrashReporterProcessServicer.WriteEvent("Shutdown: ReportWatcher thread stopped."); CrashReporterProcessServicer.WriteEvent("Shutdown: Disposing ReportQueues..."); foreach (var Queue in ReportQueues) { Queue.Dispose(); } CrashReporterProcessServicer.WriteEvent("Shutdown: ReportQueues disposed."); CancelSource.Dispose(); }
/// <summary> /// A function to process a newly landed crash report. /// </summary> /// <param name="NewContext">An instance of the generic crash context</param> private void ProcessReport(FGenericCrashContext NewContext) { Stopwatch ProcessReportSW = Stopwatch.StartNew(); // Just to verify that the report is still there. DirectoryInfo DirInfo = new DirectoryInfo(NewContext.CrashDirectory); if (!DirInfo.Exists) { // Something very odd happened. CrashReporterProcessServicer.WriteEvent("ProcessReport: Directory not found: " + NewContext.CrashDirectory); return; } double WaitTime = ReadProcessAddReport(DirInfo, NewContext); // Make sure any log messages have been written to disk CrashReporterProcessServicer.WriteEvent(string.Format("ProcessReportTime={0} WaitTime={1}", ProcessReportSW.Elapsed.TotalSeconds.ToString("0.0"), WaitTime.ToString("0.00"))); }
private static Config LoadConfigPrivate() { Config LoadedConfig = new Config(); string ExePath = Assembly.GetEntryAssembly().Location; string ExeFolder = Path.GetDirectoryName(ExePath); string ConfigPath = Path.Combine(ExeFolder, "CrashReportProcess.config"); if (File.Exists(ConfigPath)) { if (!string.IsNullOrEmpty(ConfigPath)) { using (XmlReader Reader = XmlReader.Create(ConfigPath)) { CrashReporterProcessServicer.WriteEvent("Loading config from " + ConfigPath); XmlSerializer Xml = new XmlSerializer(typeof(Config)); LoadedConfig = Xml.Deserialize(Reader) as Config; } } } #if DEBUG // Debug mode redirects to local folder in DebugTestingFolder LoadedConfig.ProcessedReports = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedReports"); LoadedConfig.ProcessedVideos = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedVideos"); LoadedConfig.DepotRoot = Path.Combine(LoadedConfig.DebugTestingFolder, "DepotRoot"); LoadedConfig.InternalLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "InternalLandingZone"); LoadedConfig.ExternalLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "ExternalLandingZone"); LoadedConfig.DataRouterLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "DataRouterLandingZone"); LoadedConfig.InvalidReportsDirectory = Path.Combine(LoadedConfig.DebugTestingFolder, "InvalidReportsDirectory"); LoadedConfig.VersionString += " debugbuild"; LoadedConfig.AWSCredentialsFilepath = Path.Combine(LoadedConfig.DebugTestingFolder, "AWS", "credentials.ini"); LoadedConfig.ProcessedReportsIndexPath = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedReports.ini"); #if SLACKTESTING LoadedConfig.SlackUsername = "******"; //LoadedConfig.SlackChannel = "OPTIONALTESTINGCHANNELHERE"; #else LoadedConfig.SlackWebhookUrl = string.Empty; // no Slack in dbeug #endif #endif return(LoadedConfig); }
/// <summary> Tries to dequeue a report from the list. </summary> public bool TryDequeueReport(out FGenericCrashContext Context) { lock ( NewReportsLock ) { int LastIndex = NewCrashContexts.Count - 1; if (LastIndex >= 0) { Context = NewCrashContexts[LastIndex]; NewCrashContexts.RemoveAt(LastIndex); CrashReporterProcessServicer.WriteEvent("- Dequeued: BuiltFromCL=" + string.Format("{0,7}", Context.PrimaryCrashProperties.EngineVersion) + " Path=" + Context.CrashDirectory); return(true); } else { Context = null; return(false); } } }
/// <summary> /// Moves specified report to the directory where are stored invalid reports. /// Thread-safe. /// </summary> public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename) { try { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingFailedEvent); DirectoryInfo DirInfo = new DirectoryInfo(ReportName); if (Config.Default.InvalidReportsToAWS) { string LeafReportName = Path.GetFileName(ReportName); DateTime S3KeyTime = DateTime.UtcNow; // Copy all files from the source directory. We can't use MoveTo due to different disc location. int FilesMoved = 0; foreach (FileInfo InvalidFile in DirInfo.GetFiles()) { try { string S3IDPrefix = string.Format("/{0}_{1}_{2}/{3}/{4}/", S3KeyTime.Year, S3KeyTime.Month, S3KeyTime.Day, S3KeyTime.Hour, LeafReportName); UploadFileToS3(InvalidFile, Config.Default.AWSS3InvalidKeyPrefix + S3IDPrefix + InvalidFile.Name, false); FilesMoved++; } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent("MoveReportToInvalid: Failed to write report file " + LeafReportName + ": " + InvalidFile.Name); CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex); } } CrashReporterProcessServicer.WriteEvent(string.Format("MoveReportToInvalid moved {0} file(s) from {1} to S3", FilesMoved, LeafReportName)); } DirInfo.Delete(true); UpdateProcessedReports(); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex); } }
public virtual void Dispose() { // Attempt to remove the queued crashes from the ReportIndex lock (NewReportsLock) { CrashReporterProcessServicer.WriteEvent(string.Format("{0} shutting down", QueueName)); CrashReporterProcessServicer.WriteEvent(string.Format("{0} dequeuing {1} crashes for next time", QueueName, NewCrashContexts.Count)); while (NewCrashContexts.Count > 0) { try { string ReportName = Path.GetFileName(NewCrashContexts.Peek().CrashDirectory); CrashReporterProcessServicer.ReportIndex.TryRemoveReport(ReportName); } finally { NewCrashContexts.Dequeue(); } } } }
private static Config LoadConfigPrivate() { Config LoadedConfig = new Config(); string ExePath = Assembly.GetEntryAssembly().Location; string ExeFolder = Path.GetDirectoryName(ExePath); string ConfigPath = Path.Combine(ExeFolder, "CrashReportProcess.config"); if (File.Exists(ConfigPath)) { if (!string.IsNullOrEmpty(ConfigPath)) { using (XmlReader Reader = XmlReader.Create(ConfigPath)) { CrashReporterProcessServicer.WriteEvent("Loading config from " + ConfigPath); XmlSerializer Xml = new XmlSerializer(typeof(Config)); LoadedConfig = Xml.Deserialize(Reader) as Config; } } } #if DEBUG // Debug mode redirects to local folder in DebugTestingFolder LoadedConfig.ProcessedReports = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedReports"); LoadedConfig.ProcessedVideos = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedVideos"); LoadedConfig.DepotRoot = Path.Combine(LoadedConfig.DebugTestingFolder, "DepotRoot"); if (!string.IsNullOrWhiteSpace(LoadedConfig.InternalLandingZone)) { LoadedConfig.InternalLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "InternalLandingZone"); } if (!string.IsNullOrWhiteSpace(LoadedConfig.ExternalLandingZone)) { LoadedConfig.ExternalLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "ExternalLandingZone"); } if (!string.IsNullOrWhiteSpace(LoadedConfig.DataRouterLandingZone)) { LoadedConfig.DataRouterLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "DataRouterLandingZone"); } if (!string.IsNullOrWhiteSpace(LoadedConfig.PS4LandingZone)) { LoadedConfig.PS4LandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "PS4LandingZone"); } if (!string.IsNullOrWhiteSpace(LoadedConfig.MDDExecutablePath)) { LoadedConfig.MDDExecutablePath = Path.Combine(LoadedConfig.DebugTestingFolder, "MinidumpDiagnostics", "Engine", "Binaries", "Win64", "MinidumpDiagnostics.exe"); } LoadedConfig.VersionString += " debugbuild"; LoadedConfig.AWSCredentialsFilepath = Path.Combine(LoadedConfig.DebugTestingFolder, "AWS", "credentials.ini"); if (!string.IsNullOrWhiteSpace(LoadedConfig.ProcessedReportsIndexPath)) { LoadedConfig.ProcessedReportsIndexPath = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedReports.ini"); } LoadedConfig.CrashReportWebSite = string.Empty; LoadedConfig.AWSS3OutputKeyPrefix = LoadedConfig.AWSS3OutputKeyPrefix.Replace("prod", "test"); LoadedConfig.AWSS3InvalidKeyPrefix = LoadedConfig.AWSS3InvalidKeyPrefix.Replace("prod", "test"); LoadedConfig.MinDesiredMemoryQueueSize = 5; LoadedConfig.MaxMemoryQueueSize = 15; #if SLACKTESTING LoadedConfig.SlackUsername = "******"; //LoadedConfig.SlackChannel = "OPTIONALTESTINGCHANNELHERE"; #else LoadedConfig.SlackWebhookUrl = string.Empty; // no Slack in dbeug #endif #endif return(LoadedConfig); }
/// <summary> Enqueues a new crash. </summary> void EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return; } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bFromWER = false; if (Context == null || !Context.HasProcessedData()) { WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { FReportData ReportData = new FReportData(MetaData, NewReportPath); Context = ConvertMetadataToCrashContext(MetaData, NewReportPath); bFromWER = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); } else { if (GenericContext != null && GenericContext.PrimaryCrashProperties.ErrorMessage.Length > 0) { // Get error message from the crash context and fix value in the metadata. Context.PrimaryCrashProperties.ErrorMessage = GenericContext.PrimaryCrashProperties.ErrorMessage; } Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // If based on WER, save to the file. if (bFromWER && GenericContext == null) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteFailure("% NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.MoveReportToInvalid(NewReportPath, Context.GetAsFilename()); return; } if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteFailure("% UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.CleanReport(NewReportPath); return; } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteFailure("! BROKEN0 : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); ReportProcessor.CleanReport(NewReportPath); return; } lock (NewReportsLock) { NewCrashContexts.Add(Context); } CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); } }
/// <summary> /// Builds MDD command line args, waits for a MDD task slot, runs MDD and blocks for the result. Writes diag text into DiagnosticsPath folder if successful. /// </summary> /// <param name="DiagnosticsPath">Path of the minidump file</param> /// <param name="Context">The crash context</param> /// <param name="ProcessorIndex">Processor thread index for logging purposes</param> /// <returns>True, if successful</returns> public bool Run(string DiagnosticsPath, FGenericCrashContext Context, int ProcessorIndex) { if (!File.Exists(Config.Default.MDDExecutablePath)) { CrashReporterProcessServicer.WriteEvent("Symbolicator.Run() file not found " + Config.Default.MDDExecutablePath); return(false); } // Use MinidumpDiagnostics from MDDExecutablePath. CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.MDDExecutablePath, Config.Default.DiskSpaceAlertPercent); // Don't purge logs // TODO: make this clear to logs once a day or something (without letting MDD check on every run!) string PurgeLogsDays = "-1"; FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); // Pass Windows variants (Win32/64) to MinidumpDiagnostics string PlatformVariant = Context.PrimaryCrashProperties.PlatformName; if (PlatformVariant != null && Context.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS")) { if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win32") || Context.PrimaryCrashProperties.PlatformFullName.Contains("32b")) { PlatformVariant = "Win32"; } else if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win64") || Context.PrimaryCrashProperties.PlatformFullName.Contains("64b")) { PlatformVariant = "Win64"; } } // Build the absolute log file path for MinidumpDiagnostics string BaseFolder = CrashReporterProcessServicer.SymbolicatorLogFolder; DateTime WriteTime = DateTime.UtcNow; string DateFolder = WriteTime.ToString("yyyy_MM_dd"); string HourFolder = WriteTime.ToString("HH"); string Folder = Path.Combine(BaseFolder, DateFolder, HourFolder); string AbsLogPath = Path.Combine(Folder, Context.GetAsFilename() + ".log"); Directory.CreateDirectory(Folder); List <string> MinidumpDiagnosticsParams = new List <string> ( new string[] { "\"" + DiagnosticsPath + "\"", "-BranchName=" + EngineVersion.Branch, // Backward compatibility "-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility "-GameName=" + Context.PrimaryCrashProperties.GameName, "-EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion, "-BuildVersion=" + (string.IsNullOrWhiteSpace(Context.PrimaryCrashProperties.BuildVersion) ? string.Format("{0}-CL-{1}", EngineVersion.Branch, EngineVersion.Changelist).Replace('/', '+') : Context.PrimaryCrashProperties.BuildVersion), "-PlatformName=" + Context.PrimaryCrashProperties.PlatformName, "-PlatformVariantName=" + PlatformVariant, "-bUsePDBCache=true", "-PDBCacheDepotRoot=" + Config.Default.DepotRoot, "-PDBCachePath=" + Config.Default.MDDPDBCachePath, "-PDBCacheSizeGB=" + Config.Default.MDDPDBCacheSizeGB, "-PDBCacheMinFreeSpaceGB=" + Config.Default.MDDPDBCacheMinFreeSpaceGB, "-PDBCacheFileDeleteDays=" + Config.Default.MDDPDBCacheFileDeleteDays, "-MutexPDBCache", "-PDBCacheLock=CrashReportProcessPDBCacheLock", "-NoTrimCallstack", "-SyncSymbols", "-NoP4Symbols", "-ForceUsePDBCache", "-MutexSourceSync", "-SourceSyncLock=CrashReportProcessSourceSyncLock", "-SyncMicrosoftSymbols", "-unattended", "-AbsLog=" + AbsLogPath, "-DepotIndex=" + Config.Default.DepotIndex, "-P4User="******"-P4Client=" + Config.Default.P4Client, "-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays + ",[LogFiles]:MaxLogFilesOnDisk=-1", "-LOGTIMESINCESTART" } ); LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null; if (Environment.UserInteractive) { CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD; } else { MinidumpDiagnosticsParams.AddRange ( new[] { "-buildmachine" } ); // Write some debugging message. CrashReporterProcessServicer.WriteMDD("MinidumpDiagnostics Params: " + String.Join(" ", MinidumpDiagnosticsParams)); } Task <bool> NewSymbolicatorTask = Task.FromResult(false); Stopwatch WaitSW = Stopwatch.StartNew(); double WaitForLockTime = 0.0; lock (Tasks) { int TaskIdx = Task.WaitAny(Tasks); Tasks[TaskIdx] = NewSymbolicatorTask = Task <bool> .Factory.StartNew(() => { LaunchProcess ReportParser = new LaunchProcess(Config.Default.MDDExecutablePath, Path.GetDirectoryName(Config.Default.MDDExecutablePath), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray()); return(ReportParser.WaitForExit(Config.Default.MDDTimeoutMinutes * 1000 * 60) == EWaitResult.Ok); }); WaitForLockTime = WaitSW.Elapsed.TotalSeconds; } NewSymbolicatorTask.Wait(); double TotalMDDTime = WaitSW.Elapsed.TotalSeconds; double MDDRunTime = TotalMDDTime - WaitForLockTime; CrashReporterProcessServicer.StatusReporter.AddToMeanCounter(StatusReportingPerfMeanNames.MinidumpDiagnostics, (int)(MDDRunTime * 1000)); CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Symbolicator.Run: Thread blocked for {0:N1}s then MDD ran for {1:N1}s", WaitForLockTime, MDDRunTime)); return(NewSymbolicatorTask.Result); }
/// <summary> /// Look for new report folders and add them to the publicly available thread-safe queue. /// </summary> public int CheckForNewReports() { try { if (QueueCount >= Config.Default.MinDesiredMemoryQueueSize) { CrashReporterProcessServicer.WriteEvent(string.Format( "CheckForNewReports: {0} skipped busy queue size {1} in {2}", QueueName, QueueCount, LandingZone)); } else { var NewReportPath = ""; var ReportsInLandingZone = new ReportIdSet(); CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(LandingZone, Config.Default.DiskSpaceAlertPercent); DirectoryInfo[] DirectoriesInLandingZone; if (GetCrashesFromLandingZone(out DirectoriesInLandingZone)) { LastQueueSizeOnDisk = DirectoriesInLandingZone.Length; int EnqueuedCount = 0; CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} reports in disk landing zone {1}", DirectoriesInLandingZone.Length, LandingZone)); // Add any new reports for (int DirIndex = 0; DirIndex < DirectoriesInLandingZone.Length && QueueCount < Config.Default.MaxMemoryQueueSize; DirIndex++) { var SubDirInfo = DirectoriesInLandingZone[DirIndex]; try { if (Directory.Exists(SubDirInfo.FullName)) { NewReportPath = SubDirInfo.FullName; ReportsInLandingZone.Add(NewReportPath); if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath)) { if (EnqueueNewReport(NewReportPath)) { EnqueuedCount++; } } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex, Ex); ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks); } } ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone; CrashReporterProcessServicer.WriteEvent(string.Format( "CheckForNewReports: {0} enqueued to queue size {1} from {2}", EnqueuedCount, QueueCount, LandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: Enqueue rate {0:N1}/minute from {1}", EnqueueCounter.EventsPerSecond * 60, LandingZone)); } else { LastQueueSizeOnDisk = 0; } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex, Ex); } return(GetTotalWaitingCount()); }
// From CrashUpload.cpp /* * struct FCompressedCrashFile : FNoncopyable * { * int32 CurrentFileIndex; // 4 bytes for file index * FString Filename; // 4 bytes for length + 260 bytes for char data * TArray<uint8> Filedata; // 4 bytes for length + N bytes for data * } */ /// <summary> Enqueues a new crash. </summary> private bool EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return(false); } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bContextDirty = false; WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { if (Context == null) { // Missing crash context FReportData ReportData = new FReportData(MetaData, NewReportPath); ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context); bContextDirty = true; } else if (!Context.HasProcessedData()) { // Missing data - try to get from WER metadata FReportData ReportData = new FReportData(MetaData, NewReportPath); GetErrorMessageFromMetadata(ReportData, Context); bContextDirty = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // Added data from WER, save to the crash context file. if (bContextDirty) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion; Context.ToFile(); } else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName; Context.ToFile(); } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg"; Context.ToFile(); } // Check static reports index for duplicated reports // This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router) // This can be safely discontinued when all crashes come from the same SQS // This DOES NOT scale to multiple CRP instances // ReportIndex can be disabled by leaving the path empty in config if (!CrashReporterProcessServicer.ReportIndex.TryAddNewReport(ReportName)) { // Crash report not accepted by index CrashReporterProcessServicer.WriteEvent(string.Format( "EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } if (ShouldDecimateNextReport()) { CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); return(false); } lock (NewReportsLock) { NewCrashContexts.Enqueue(Context); } EnqueueCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent); CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); return(true); }
/// <summary> /// Create thread to watch for new crash reports landing. /// </summary> /// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks> void Init() { CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:"); var Settings = Config.Default; if (!string.IsNullOrEmpty(Settings.DataRouterLandingZone)) { if (System.IO.Directory.Exists(Settings.DataRouterLandingZone)) { ReportQueues.Add(new DataRouterReportQueue("DataRouter Crashes", Settings.DataRouterLandingZone, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (all crashes from data router)", Settings.DataRouterLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (all crashes from data router) is not accessible", Settings.DataRouterLandingZone)); } } if (!string.IsNullOrEmpty(Settings.InternalLandingZone)) { if (System.IO.Directory.Exists(Settings.InternalLandingZone)) { ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone, StatusReportingEventNames.ProcessingStartedReceiverEvent, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority (legacy))", Settings.InternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority (legacy)) is not accessible", Settings.InternalLandingZone)); } } if (!string.IsNullOrEmpty(Settings.PS4LandingZone)) { if (System.IO.Directory.Exists(Settings.PS4LandingZone)) { ReportQueues.Add(new ReceiverReportQueue("PS4 Crashes", Settings.PS4LandingZone, StatusReportingEventNames.ProcessingStartedPS4Event, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, PS4 non-retail hardware)", Settings.PS4LandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, PS4 non-retail hardware) is not accessible", Settings.PS4LandingZone)); } } #if !DEBUG if (!string.IsNullOrEmpty(Settings.ExternalLandingZone)) { if (System.IO.Directory.Exists(Settings.ExternalLandingZone)) { ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone, StatusReportingEventNames.ProcessingStartedReceiverEvent, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (legacy)", Settings.ExternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (legacy) is not accessible", Settings.ExternalLandingZone)); } } #endif //!DEBUG // Init queue entries in StatusReporter foreach (var Queue in ReportQueues) { CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath); } var Cancel = CancelSource.Token; WatcherTask = new Task(async() => { DateTime LastQueueSizeReport = DateTime.MinValue; while (!Cancel.IsCancellationRequested) { // Check the landing zones for new reports DateTime StartTime = DateTime.Now; foreach (var Queue in ReportQueues) { int QueueSize = Queue.CheckForNewReports(); CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize); } TimeSpan TimeTaken = DateTime.Now - StartTime; CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds)); await Task.Delay(30000, Cancel); } }); }
/// <summary> /// Look for new report folders and add them to the publicly available thread-safe queue. /// </summary> public override int CheckForNewReports() { try { if (QueueCount >= SkipQueueRefreshMin) { CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: skipped busy queue size {0} in {1}", QueueCount, LandingZone)); return(LastQueueSizeOnDisk); } var NewReportPath = ""; var ReportsInLandingZone = new ReportIdSet(); // Check the landing zones for new reports DirectoryInfo DirInfo = new DirectoryInfo(LandingZone); // Couldn't find a landing zone, skip and try again later. // Crash receiver will recreate them if needed. if (!DirInfo.Exists) { CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone); return(LastQueueSizeOnDisk); } var DirectoriesInLandingZone = DirInfo.GetDirectories().OrderBy(dirinfo => dirinfo.CreationTimeUtc).ToArray(); LastQueueSizeOnDisk = DirectoriesInLandingZone.Length; int EnqueuedCount = 0; CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} reports in {1}", DirectoriesInLandingZone.Length, LandingZone)); // Add any new reports for (int DirIndex = 0; DirIndex < DirectoriesInLandingZone.Length && QueueCount < QueueMax; DirIndex++) { var SubDirInfo = DirectoriesInLandingZone[DirIndex]; try { if (Directory.Exists(SubDirInfo.FullName)) { NewReportPath = SubDirInfo.FullName; ReportsInLandingZone.Add(NewReportPath); if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath)) { EnqueueNewReport(NewReportPath); EnqueuedCount++; } } } catch (System.Exception Ex) { CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex.ToString()); ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks); } } //CrashReporterProcessServicer.WriteEvent( string.Format( "ReportsInLandingZone={0}, ReportsInLandingZoneLastTimeWeChecked={1}", ReportsInLandingZone.Count, ReportsInLandingZoneLastTimeWeChecked.Count ) ); ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone; CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} enqueued to queue size {1} from {2}", EnqueuedCount, QueueCount, LandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: Enqueue rate {0:N1}/minute from {1}", EnqueueCounter.EventsPerSecond * 60, LandingZone)); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex.ToString()); } return(LastQueueSizeOnDisk); }
/// <summary> Enqueues a new crash. </summary> private void EnqueueNewReport(string NewReportPath) { string ReportName = Path.GetFileName(NewReportPath); string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash"); string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml"); bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath); if (bIsCompressed) { FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath); bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation); if (!bResult) { ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName); return; } else { // Rename metadata file to not interfere with the WERReportMetadata. File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml")); } } // Unified crash reporting FGenericCrashContext GenericContext = FindCrashContext(NewReportPath); FGenericCrashContext Context = GenericContext; bool bFromWER = false; if (Context == null || !Context.HasProcessedData()) { WERReportMetadata MetaData = FindMetadata(NewReportPath); if (MetaData != null) { FReportData ReportData = new FReportData(MetaData, NewReportPath); ConvertMetadataToCrashContext(MetaData, NewReportPath, ref Context); bFromWER = true; } } if (Context == null) { CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath); ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath)); } else { Context.CrashDirectory = NewReportPath; Context.PrimaryCrashProperties.SetPlatformFullName(); // Added data from WER, save to the crash context file. if (bFromWER) { Context.ToFile(); } FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion); uint BuiltFromCL = EngineVersion.Changelist; string BranchName = EngineVersion.Branch; if (string.IsNullOrEmpty(BranchName)) { CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion; } else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase)) { CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName; } // Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken. bool bHasDiagnostics = FindDiagnostics(NewReportPath); if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData())) { // For now ignore all locally made crashes. CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg"; } lock (NewReportsLock) { NewCrashContexts.Enqueue(Context); } EnqueueCounter.AddEvent(); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingConstants.QueuedEvent); CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath); } }
private static bool DecompressDataRouterContent(byte[] CompressedBufferArray, string InLandingZone) { // Decompress to landing zone byte[] UncompressedBufferArray = new byte[Config.Default.MaxUncompressedS3RecordSize]; int UncompressedSize = NativeMethods.UncompressMemoryZlib(UncompressedBufferArray, CompressedBufferArray); if (UncompressedSize < 0) { string FailString = "! DecompressDataRouterContent() failed in UncompressMemoryZlib() with " + NativeMethods.GetZlibError(UncompressedSize); CrashReporterProcessServicer.WriteFailure(FailString); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent); return(false); } using (BinaryReader BinaryData = new BinaryReader(new MemoryStream(UncompressedBufferArray, 0, UncompressedSize, false))) { char[] MarkerChars = BinaryData.ReadChars(3); if (MarkerChars[0] == 'C' && MarkerChars[1] == 'R' && MarkerChars[2] == '1') { CrashHeader CrashHeader = DataRouterReportQueue.CrashHeader.ParseCrashHeader(BinaryData); // Create safe directory name and then create the directory on disk string CrashFolderName = GetSafeFilename(CrashHeader.DirectoryName); string CrashFolderPath = Path.Combine(InLandingZone, CrashFolderName); // Early check for duplicate processed report lock (ReportIndexLock) { if (CrashReporterProcessServicer.ReportIndex.ContainsReport(CrashFolderName)) { // Crash report not accepted by index CrashReporterProcessServicer.WriteEvent(string.Format( "DataRouterReportQueue: Duplicate report skipped early {0} in a DataRouterReportQueue", CrashFolderPath)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected); return(false); // this isn't an error so don't set error message } } // Create target folder int TryIndex = 1; while (Directory.Exists(CrashFolderPath)) { CrashFolderPath = Path.Combine(InLandingZone, string.Format("{0}_DUPE{1:D3}", CrashFolderName, TryIndex++)); } Directory.CreateDirectory(CrashFolderPath); if (UncompressedSize != CrashHeader.UncompressedSize) { CrashReporterProcessServicer.WriteEvent( string.Format( "DecompressDataRouterContent() warning UncompressedSize mismatch (embedded={0}, actual={1}) Path={2}", CrashHeader.UncompressedSize, UncompressedSize, CrashFolderPath)); } if (!ParseCrashFiles(BinaryData, CrashHeader.FileCount, CrashFolderPath)) { string FailString = "! DecompressDataRouterContent() failed to write files Path=" + CrashFolderPath; CrashReporterProcessServicer.WriteFailure(FailString); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent); return(false); } } else { #if ALLOWOLDCLIENTDATA // Early Data Router upload format was broken. // Should be [CR1][CrashHeader][File][File][File][File]... // Actually [Undefined CrashHeader][File][File][File][File]...[CrashHeader] // Seek to end minus header size BinaryData.BaseStream.Position = UncompressedSize - DataRouterReportQueue.CrashHeader.FixedSize; var CrashHeader = DataRouterReportQueue.CrashHeader.ParseCrashHeader(BinaryData); // Create safe directory name and then create the directory on disk string CrashFolderName = GetSafeFilename(CrashHeader.DirectoryName); string CrashFolderPath = Path.Combine(InLandingZone, CrashFolderName); // Early check for duplicate processed report lock (ReportIndexLock) { if (CrashReporterProcessServicer.ReportIndex.ContainsReport(CrashFolderName)) { // Crash report not accepted by index CrashReporterProcessServicer.WriteEvent(string.Format( "DataRouterReportQueue: Duplicate report skipped early {0} in a DataRouterReportQueue", CrashFolderPath)); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected); return(false); // this isn't an error so don't set error message } } // Create target folder int TryIndex = 1; while (Directory.Exists(CrashFolderPath)) { CrashFolderPath = Path.Combine(InLandingZone, string.Format("{0}_DUPE{1:D3}", CrashFolderName, TryIndex++)); } Directory.CreateDirectory(CrashFolderPath); if (UncompressedSize != CrashHeader.UncompressedSize + CrashHeader.FixedSize) { CrashReporterProcessServicer.WriteEvent( string.Format( "DecompressDataRouterContent() warning UncompressedSize mismatch (embedded={0}, actual={1}) Path={2}", CrashHeader.UncompressedSize, UncompressedSize, CrashFolderPath)); } // Seek to start of files (header size in from start) BinaryData.BaseStream.Position = CrashHeader.FixedSize; if (!ParseCrashFiles(BinaryData, CrashHeader.FileCount, CrashFolderPath)) { string FailString = "! DecompressDataRouterContent() failed to write files Path=" + CrashFolderPath; CrashReporterProcessServicer.WriteFailure(FailString); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent); return(false); } #else ErrorMessage = "! DecompressDataRouterContent() failed to read invalid data format. Corrupt or old format data received Path=" + CrashFolderPath; return(false); #endif } } return(true); }
/// <summary> /// A function to add a report to the database, and rename the relevant files. /// Thread-safe. /// </summary> /// <param name="DirInfo">The DirectoryInfo of the report folder.</param> /// <param name="NewContext">The generic crash context.</param> /// <param name="LogFileName">The file name of the log file in the report.</param> /// <param name="DumpFileName">The file name of the minidump in the report.</param> /// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param> private bool AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName) { try { Stopwatch AddReportTime = Stopwatch.StartNew(); // Create an XML representation of a crash string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null); if (CrashDetails == "") { CrashReporterProcessServicer.WriteFailure("! NoDetail: Path=" + NewContext.CrashDirectory); return(false); } // Upload the crash to the database, and retrieve the new row id int ReportID = UploadCrash(CrashDetails); if (ReportID <= 0) { CrashReporterProcessServicer.WriteFailure("! NoUpload: Path=" + NewContext.CrashDirectory); return(false); } string IDThenUnderscore = string.Format("{0}_", ReportID); // Use the row id to name and move the files the way the web site requires string DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedReports, IDThenUnderscore); // Move report files to crash reporter file store if (LogFileName != null) { LogFileName = Path.Combine(DirInfo.FullName, LogFileName); FileInfo LogInfo = new FileInfo(LogFileName); if (LogInfo.Exists) { LogInfo.MoveTo(DestinationFolder + "Launch.log"); } } string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName); FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName); if (CrashContextInfo.Exists) { CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName); } // WERMetaDataName = Path.Combine(DirInfo.FullName, WERMetaDataName); // FileInfo MetaInfo = new FileInfo(WERMetaDataName); // if (MetaInfo.Exists) // { // MetaInfo.MoveTo(DestinationFolder + "WERMeta.xml"); // } if (DumpFileName != null) { DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName); FileInfo DumpInfo = new FileInfo(DumpFileName); if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/) { DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp"); } } // DiagnosticsFileName = Path.Combine(DirInfo.FullName, DiagnosticsFileName); // FileInfo DiagnosticsInfo = new FileInfo(DiagnosticsFileName); // if (DiagnosticsInfo.Exists) // { // DiagnosticsInfo.MoveTo(DestinationFolder + CrashReporterConstants.DiagnosticsFileName); // } // Move the video (if it exists) to an alternate store if (VideoFileName != null) { DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedVideos, IDThenUnderscore); VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName); FileInfo VideoInfo = new FileInfo(VideoFileName); if (VideoInfo.Exists) { VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName); } } CrashReporterProcessServicer.WriteEvent("# WebAdded: ReportID =" + string.Format("{0,7}", ReportID) + " Path=" + NewContext.CrashDirectory); UpdateProcessedReports(); WebAddedReports++; double Ratio = (double)WebAddedReports / (double)ProcessedReports * 100; double AddedPerDay = (double)WebAddedReports / Timer.Elapsed.TotalDays; CrashReporterProcessServicer.WriteEvent(string.Format("Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4}", (int)Ratio, ProcessedReports, WebAddedReports, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay)); return(true); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("AddReport: " + DirInfo.Name + "\n\n" + Ex.ToString()); } return(false); }