private bool DequeueRecordSQS(out string OutRecordString) { OutRecordString = string.Empty; try { #if DEBUG bool bNoDeleteFromSQS = true; #else bool bNoDeleteFromSQS = false; #endif Message DequeuedMessage = CrashReporterProcessServicer.DataRouterAWS.SQSDequeueMessage(Config.Default.AWSSQSQueueInputUrl, bNoDeleteFromSQS); if (DequeuedMessage != null) { OutRecordString = DequeuedMessage.Body; return(true); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("DequeueRecordSQS: " + Ex, Ex); } return(false); }
private bool DequeueRecordSQS(out string OutRecordString) { OutRecordString = string.Empty; try { var ReceiveRequest = new ReceiveMessageRequest { QueueUrl = Config.Default.AWSSQSQueueUrl, MaxNumberOfMessages = 1 }; var ReceiveResponse = SqsClient.ReceiveMessage(ReceiveRequest); if (ReceiveResponse.Messages.Count == 1) { var Message = ReceiveResponse.Messages[0]; if (Message != null && TryDeleteRecordSQS(Message)) { OutRecordString = Message.Body; return(true); } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("DequeueRecordSQS: " + Ex.ToString()); } return(false); }
/// <summary> /// A thread to watch for new crash reports landing. /// </summary> /// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks> void Start() { CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:"); var Settings = Config.Default; if (!string.IsNullOrEmpty(Settings.InternalLandingZone)) { if (System.IO.Directory.Exists(Settings.InternalLandingZone)) { ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority)", Settings.InternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority) is not accessible", Settings.InternalLandingZone)); } } #if !DEBUG if (!string.IsNullOrEmpty(Settings.ExternalLandingZone)) { if (System.IO.Directory.Exists(Settings.ExternalLandingZone)) { ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone)); CrashReporterProcessServicer.WriteEvent(string.Format("\t{0}", Settings.ExternalLandingZone)); } else { CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} is not accessible", Settings.ExternalLandingZone)); } } #endif //!DEBUG // Init queue entries in StatusReporter foreach (var Queue in ReportQueues) { CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath); } var Cancel = CancelSource.Token; WatcherTask = Task.Factory.StartNew(async() => { DateTime LastQueueSizeReport = DateTime.MinValue; while (!Cancel.IsCancellationRequested) { // Check the landing zones for new reports DateTime StartTime = DateTime.Now; foreach (var Queue in ReportQueues) { int QueueSize = Queue.CheckForNewReports(); CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize); } TimeSpan TimeTaken = DateTime.Now - StartTime; CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds)); await Task.Delay(60000, Cancel); } }); }
/// <summary> /// Clear out old MDD logs /// </summary> /// <param name="NumDays">Number of days worth of logs to keep</param> public static void CleanOutOldLogs(int NumDays) { DateTime DeleteTime = DateTime.UtcNow - TimeSpan.FromDays(NumDays); try { DirectoryInfo BaseFolder = new DirectoryInfo(CrashReporterProcessServicer.SymbolicatorLogFolder); if (BaseFolder.Exists) { foreach (DirectoryInfo SubFolder in BaseFolder.EnumerateDirectories()) { if (!SubFolder.EnumerateFiles("*", SearchOption.AllDirectories).Any(x => x.LastWriteTimeUtc > DeleteTime)) { try { SubFolder.Delete(true); } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", SubFolder.FullName, Ex.Message)); } } } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs: {0}", Ex.Message)); } }
private void TryGetNewS3Crashes(int CrashCount) { int NewCrashCount = 0; while (NewCrashCount < CrashCount) { string SQSRecord = "<unset>"; try { if (!DequeueRecordSQS(out SQSRecord)) { // Queue empty break; } var RecordPair = SQSRecord.Split(','); if (RecordPair.Length != 2) { CrashReporterProcessServicer.WriteFailure("TryGetNewS3Crashes: bad SQS message was " + SQSRecord); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent); continue; } string S3BucketName = RecordPair[0]; string S3Key = RecordPair[1]; string ReadableRequestString = "Bucket=" + S3BucketName + " Key=" + S3Key; var ObjectRequest = new GetObjectRequest { BucketName = S3BucketName, Key = S3Key }; using (Stream ProtocolBufferStream = new MemoryStream()) { using (GetObjectResponse ObjectResponse = S3Client.GetObject(ObjectRequest)) { using (Stream ResponseStream = ObjectResponse.ResponseStream) { if (!TryDecompResponseStream(ResponseStream, ProtocolBufferStream)) { CrashReporterProcessServicer.WriteFailure("! GZip fail in DecompResponseStream(): " + ReadableRequestString); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent); continue; } } } NewCrashCount += UnpackRecordsFromDelimitedProtocolBuffers(ProtocolBufferStream, LandingZone, ReadableRequestString); } } catch (Exception ex) { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent); CrashReporterProcessServicer.WriteException("TryGetNewS3Crashes: failure during processing SQS record " + SQSRecord + "\n" + ex); } } }
private static int UnpackRecordsFromDelimitedProtocolBuffers(Stream ProtocolBufferStream, string InLandingZone, string ReadableRequestString) { var UnpackedRecordCount = 0; DataRouterConsumer Consumer = new DataRouterConsumer(); // Expect one or more pairs of varint size + protocol buffer in the stream while (ProtocolBufferStream.Position < ProtocolBufferStream.Length) { DataRouterConsumer.ProtocolBufferRecord Message; if (!Consumer.TryParse(ProtocolBufferStream, out Message)) { string FailString = "! Protocol buffer parse fail in UnpackRecordsFromDelimitedProtocolBuffers(): " + ReadableRequestString; FailString += '\n' + Consumer.LastError; CrashReporterProcessServicer.WriteFailure(FailString); CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent); break; } if (DecompressDataRouterContent(Message.Payload, InLandingZone)) { UnpackedRecordCount++; } } return(UnpackedRecordCount); }
public static void WriteToFile() { try { if (File.Exists(Filepath)) { File.Delete(Filepath + ".backup"); File.Move(Filepath, Filepath + ".backup"); } using (var Writer = File.CreateText(Filepath)) { var Today = DateTime.UtcNow.Date; foreach (var Item in Index) { if (Today - Item.Value.Date <= Retention) { Writer.WriteLine(ItemToString(Item)); } } } LastFlush = DateTime.UtcNow; } catch (Exception ex) { CrashReporterProcessServicer.WriteException(string.Format("Failed to write ReportIndex to {0}. Exception was: {1}", Filepath, ex)); CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.WriteToFile", string.Format("Failed to write ReportIndex to {0}", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes); } }
void ProcessDumpFile(string DumpPath, FGenericCrashContext NewContext) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DumpPath); if (TrySimulateSymbolication(NewContext)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Simulated symbolication (skipped MDD) " + NewContext.CrashDirectory); } else if (CrashReporterProcessServicer.Symbolicator.Run(DumpPath, NewContext, ProcessorIndex)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DumpPath); ReadDiagnosticsFile(NewContext); } else { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DumpPath); } if (string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.CallStack)) { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationFailedEvent); } else { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationSucceededEvent); } }
/// <summary> /// Moves specified report to the directory where are stored invalid reports. /// Thread-safe. /// </summary> public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename) { try { DirectoryInfo DirInfo = new DirectoryInfo(ReportName); // Rename the report directory, so we should be able to quickly find the invalid reports. Directory.CreateDirectory(Properties.Settings.Default.InvalidReportsDirectory); string CleanFilename = String.Concat(ReportNameAsFilename.Split(Path.GetInvalidFileNameChars())); string DestinationDirectory = Path.Combine(Properties.Settings.Default.InvalidReportsDirectory, CleanFilename); Directory.CreateDirectory(DestinationDirectory); // Copy all files from the source directory. We can't use MoveTo due to different disc location. foreach (FileInfo File in DirInfo.GetFiles()) { string DestinationFilepath = Path.Combine(DestinationDirectory, File.Name); File.CopyTo(DestinationFilepath, true); } DirInfo.Delete(true); CrashReporterProcessServicer.WriteEvent(string.Format("Moved to {0}", DestinationDirectory)); UpdateProcessedReports(); } catch (System.Exception Ex) { CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex.ToString()); } }
public bool TryAddNewReport(string ReportKey) { if (!IsEnabled) { return(true); } lock (IndexLock) { if (Index.ContainsKey(ReportKey)) { return(false); } Index.Add(ReportKey, DateTime.UtcNow); } lock (FileLock) { if (LastFlush.Date < DateTime.UtcNow.Date) { CrashReporterProcessServicer.WriteEvent("ReportIndex.TryAddNewReport flushing index to disk on schedule."); WriteToFile(); } } return(true); }
public StatusReportLoop(TimeSpan InLoopPeriod, Func <TimeSpan, string> LoopFunction) { InitialPause = TimeSpan.FromSeconds(30); LoopPeriod = InLoopPeriod; LoopTask = Task.Factory.StartNew(() => { var Cancel = CancelSource.Token; // Small pause to allow the app to get up and running before we run the first report Thread.Sleep(InitialPause); DateTime PeriodStartTime = CreationTimeUtc; lock (TaskMonitor) { while (!Cancel.IsCancellationRequested) { DateTime PeriodEndTime = DateTime.UtcNow; string ReportMessage = LoopFunction.Invoke(PeriodEndTime - PeriodStartTime); PeriodStartTime = PeriodEndTime; if (!string.IsNullOrWhiteSpace(ReportMessage)) { CrashReporterProcessServicer.WriteSlack(ReportMessage); } Monitor.Wait(TaskMonitor, LoopPeriod); } } }); }
protected static bool WriteIncomingFile(BinaryReader BinaryData, int FileIndex, string DestinationFolderPath) { try { Int32 CurrentFileIndex = BinaryData.ReadInt32(); if (CurrentFileIndex != FileIndex) { CrashReporterProcessServicer.WriteFailure("! WriteIncomingFile index mismatch: Required=" + FileIndex + " Read=" + CurrentFileIndex); return(false); } string Filename = FBinaryReaderHelper.ReadFixedSizeString(BinaryData); string SafeFilename = GetSafeFilename(Filename); Int32 FiledataLength = BinaryData.ReadInt32(); byte[] Filedata = BinaryData.ReadBytes(FiledataLength); Directory.CreateDirectory(DestinationFolderPath); File.WriteAllBytes(Path.Combine(DestinationFolderPath, SafeFilename), Filedata); return(true); } catch (Exception Ex) { throw new CrashReporterException(string.Format("! WriteIncomingFile failed writing FileIndex={0} FolderPath={1}", FileIndex, DestinationFolderPath), Ex); } }
/// <summary> /// Moves specified report to the directory where are stored invalid reports. /// Thread-safe. /// </summary> public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename) { try { CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingFailedEvent); DirectoryInfo DirInfo = new DirectoryInfo(ReportName); // Rename the report directory, so we should be able to quickly find the invalid reports. CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.InvalidReportsDirectory, Config.Default.DiskSpaceAlertPercent); Directory.CreateDirectory(Config.Default.InvalidReportsDirectory); string DestinationDirectory = Path.Combine(Config.Default.InvalidReportsDirectory, ReportNameAsFilename); Directory.CreateDirectory(DestinationDirectory); // Copy all files from the source directory. We can't use MoveTo due to different disc location. foreach (FileInfo File in DirInfo.GetFiles()) { string DestinationFilepath = Path.Combine(DestinationDirectory, File.Name); File.CopyTo(DestinationFilepath, true); } DirInfo.Delete(true); CrashReporterProcessServicer.WriteEvent(string.Format("Moved to {0}", DestinationDirectory)); UpdateProcessedReports(); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex); } }
/// <summary> Looks for the WER metadata xml file, if found, will return a new instance of the WERReportMetadata. </summary> private WERReportMetadata FindMetadata(string NewReportPath) { WERReportMetadata MetaData = null; // Just to verify that the report is still there. DirectoryInfo DirInfo = new DirectoryInfo(NewReportPath); if (!DirInfo.Exists) { CrashReporterProcessServicer.WriteFailure("FindMetadata: Directory not found " + NewReportPath); } else { // Check to see if we wish to suppress processing of this report. foreach (var Info in DirInfo.GetFiles("*.xml")) { var MetaDataToCheck = XmlHandler.ReadXml <WERReportMetadata>(Info.FullName); if (CheckMetaData(MetaDataToCheck)) { MetaData = MetaDataToCheck; break; } } } return(MetaData); }
public void AlertOnLowDisk(string Filepath, float AlertThresholdPercent) { try { string Drive; if (!StorageSpaceHelper.TryGetDriveLetter(Filepath, out Drive)) { throw new CrashReporterException("Failed to get drive letter for path " + Filepath); } Int64 FreeSpace; float FreePercent; if (StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent)) { if (FreePercent < AlertThresholdPercent) { Alert("AlertOnLowDisk" + Drive, "Low disk space warning on " + Drive + " =>> " + GetDiskSpaceString(FreeSpace, FreePercent), 3 * Config.Default.SlackAlertRepeatMinimumMinutes); } } else { CrashReporterProcessServicer.WriteEvent("Failed to read disk space for " + Drive); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("AlertOnLowDisk failed: " + Ex, Ex); } }
/// <summary> /// Delete a report directory. /// Thread-safe. /// </summary> /// <param name="DirInfo">The directory to delete</param> public static void CleanReport(DirectoryInfo DirInfo) { bool bWriteException = true; for (int Retry = 0; Retry < 3; ++Retry) { try { foreach (FileInfo Info in DirInfo.GetFiles()) { Info.IsReadOnly = false; } DirInfo.Delete(true /* delete contents */); break; } catch (Exception Ex) { if (bWriteException) { CrashReporterProcessServicer.WriteException("CleanReport: " + Ex.ToString()); bWriteException = false; } } System.Threading.Thread.Sleep(100); } }
/// <summary> /// Shutdown: stop the thread. Will request stop first if RequestStop() hasn't been called. Then blocks until the processor thread exits. /// </summary> public void Dispose() { if (!CancelSource.IsCancellationRequested) { CancelSource.Cancel(); } CrashReporterProcessServicer.WriteEvent("Shutdown: Stopping ReportProcessor thread..."); ProcessorTask.Wait(); ProcessorTask.Dispose(); CrashReporterProcessServicer.WriteEvent("Shutdown: ReportProcessor thread stopped."); CrashReporterProcessServicer.WriteEvent("Shutdown: Stopping ReportProcessor's AddReport threads..."); foreach (Task AddReportTask in AddReportTasks) { if (AddReportTask != null) { AddReportTask.Wait(); AddReportTask.Dispose(); } } CrashReporterProcessServicer.WriteEvent("Shutdown: AddReport threads stopped."); CancelSource.Dispose(); }
public void Alert(string AlertKey, string AlertText, int RepeatMinimumMinutes) { // Do "no repeat" check here of recent alerts and block duplicates based on timer (use AlertKey) if (CheckRecentAlerts(AlertKey, RepeatMinimumMinutes)) { // Report an alert condition to Slack immediately CrashReporterProcessServicer.WriteSlack("@channel ALERT: " + AlertText); } }
/// <summary> /// Static init for one-time init jobs /// </summary> static ReportProcessor() { #if !DEBUG if (!SyncRequiredFiles()) { CrashReporterProcessServicer.WriteFailure("ReportProcessor: failed to sync files from Perforce"); } #endif }
private static void UploadFileToS3(FileInfo FileInfo, string DestFilepath, bool bCompressed, string CompressedSuffix = null) { try { if (bCompressed) { string LocalCompressedFilepath = FileInfo.FullName; if (CompressedSuffix != null) { LocalCompressedFilepath += CompressedSuffix; } else { LocalCompressedFilepath += ".gz"; } byte[] UncompressedBytes = File.ReadAllBytes(FileInfo.FullName); int ReturnCode = NativeMethods.CompressFileGZIP(LocalCompressedFilepath, UncompressedBytes); if (ReturnCode < 0) { throw new CrashReporterException(string.Format("Failed to compress {0} to gzip file with error {1}", FileInfo.FullName, NativeMethods.GetZlibError(ReturnCode))); } string CompressedDestFilepath = DestFilepath; if (CompressedSuffix != null) { CompressedDestFilepath += CompressedSuffix; } PutObjectResponse Response = CrashReporterProcessServicer.OutputAWS.PutS3ObjectFromFile(Config.Default.AWSS3OutputBucket, CompressedDestFilepath, LocalCompressedFilepath); if (Response == null || Response.HttpStatusCode != HttpStatusCode.OK) { throw new CrashReporterException(string.Format("Failed to upload compressed gzip {0} to {1}", LocalCompressedFilepath, CompressedDestFilepath)); } } else { PutObjectResponse Response = CrashReporterProcessServicer.OutputAWS.PutS3ObjectFromFile(Config.Default.AWSS3OutputBucket, DestFilepath, FileInfo.FullName); if (Response == null || Response.HttpStatusCode != HttpStatusCode.OK) { throw new CrashReporterException(string.Format("Failed to upload {0} to {1}", FileInfo.FullName, DestFilepath)); } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("UploadFileToS3: " + Ex, Ex); } }
/// <summary> /// Clear out old MDD logs /// </summary> /// <param name="NumDays">Number of days worth of logs to keep</param> public static void CleanOutOldLogs(int NumDays) { DateTime DeleteTime = DateTime.UtcNow - TimeSpan.FromDays(NumDays); try { DirectoryInfo BaseFolder = new DirectoryInfo(CrashReporterProcessServicer.SymbolicatorLogFolder); if (BaseFolder.Exists) { foreach (DirectoryInfo SubFolder in BaseFolder.EnumerateDirectories()) { if (!SubFolder.EnumerateFiles("*", SearchOption.AllDirectories).Any(x => x.LastWriteTimeUtc > DeleteTime)) { try { SubFolder.Delete(true); } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", SubFolder.FullName, Ex.Message)); } } } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs: {0}", Ex.Message)); } try { DirectoryInfo BaseFolder = new DirectoryInfo(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Config.Default.MDDExecutablePath), "..", "..", "Programs", "MinidumpDiagnostics", "Saved", "Logs"))); if (BaseFolder.Exists) { foreach (FileInfo File in BaseFolder.EnumerateFiles("*", SearchOption.AllDirectories)) { try { if (File.LastWriteTimeUtc < DeleteTime) { File.Delete(); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", File.FullName, Ex.Message)); } } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs from program folder: {0}", Ex.Message)); } }
public void Dispose() { if (ReporterTasks != null) { foreach (var ReporterTask in ReporterTasks) { ReporterTask.Dispose(); } } CrashReporterProcessServicer.WriteSlack("CRP stopped"); }
/// <summary> /// Static init for one-time init jobs /// </summary> static ReportProcessor() { #if !DEBUG if (Config.Default.bSyncMinidumpDiagnostics) { if (!SyncMinidumpDiagnostics()) { CrashReporterProcessServicer.WriteFailure("ReportProcessor: failed to sync files from Perforce"); } } #endif }
public void ReadFromFile() { if (!IsEnabled) { return; } try { Index = new Dictionary <string, DateTime>(); if (!File.Exists(Filepath)) { if (File.Exists(Filepath + ".backup")) { CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read ReportIndex from {0}. Attempting to read from {1}", Filepath, Filepath + ".backup")); CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.ReadFromFile.UsingBackup", string.Format("Failed to read ReportIndex from {0}. Using backup.", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes); File.Move(Filepath + ".backup", Filepath); } else { CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read ReportIndex from {0}. Generating new one.", Filepath)); File.Create(Filepath).Close(); } } using (var Reader = File.OpenText(Filepath)) { string ItemStringRaw; while ((ItemStringRaw = Reader.ReadLine()) != null) { string ItemString = ItemStringRaw.Trim(); if (!string.IsNullOrWhiteSpace(ItemString)) { KeyValuePair <string, DateTime> NewItem; if (!TryParseItemString(ItemString, out NewItem)) { CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read line from ReportIndex: {0}.", ItemString)); continue; } Index.Add(NewItem.Key, NewItem.Value); } } } LastFlush = DateTime.UtcNow; } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("Failed to read ReportIndex from {0}. Exception was: {1}", Filepath, Ex), Ex); CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.ReadFromFile", string.Format("Failed to read ReportIndex from {0}", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes); } }
/// <summary> /// Returns the count of items in the SQS /// </summary> private int GetSQSCount() { try { return(CrashReporterProcessServicer.DataRouterAWS.GetSQSQueueCount(Config.Default.AWSSQSQueueInputUrl)); } catch (Exception Ex) { CrashReporterProcessServicer.WriteException("GetSQSCount: " + Ex, Ex); } return(0); }
/// <summary> /// Call the web service function with an Xml payload to add a crash to the database. /// </summary> /// <param name="Payload">Xml representation of a new crash to upload.</param> /// <returns>The database id of the newly added row.</returns> private int UploadCrash(string Payload) { int NewID = -1; try { // Simple suppression by blanking out the URL for local testing if (Config.Default.CrashReportWebSite.Length > 0) { bool bDebug = false; string RequestString; if (!bDebug) { RequestString = "http://" + Config.Default.CrashReportWebSite + ":80/Crashes/AddCrash/-1"; } else { RequestString = "http://localhost:80/Crashes/AddCrash/-1"; } string ErrorMessage = string.Empty; for (int Retry = 0; Retry < 3; ++Retry) { string ResponseString = SimpleWebRequest.GetWebServiceResponse(RequestString, Payload); if (ResponseString.Length > 0) { // Convert response into a string CrashReporterResult Result = XmlHandler.FromXmlString <CrashReporterResult>(ResponseString); if (Result.ID > 0) { NewID = Result.ID; break; } ErrorMessage = Result.Message; } Thread.Sleep(200); } if (NewID == -1) { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "UploadCrash: " + ErrorMessage); } } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "UploadCrash: " + Ex.ToString()); } return(NewID); }
/// <summary> /// Decompresses a compressed crash report. /// </summary> unsafe private bool DecompressReport(string CompressedReportPath, FCompressedCrashInformation Meta) { string ReportPath = Path.GetDirectoryName(CompressedReportPath); using (FileStream FileStream = File.OpenRead(CompressedReportPath)) { Int32 UncompressedSize = Meta.GetUncompressedSize(); Int32 CompressedSize = Meta.GetCompressedSize(); if (FileStream.Length != CompressedSize) { return(false); } byte[] UncompressedBufferArray = new byte[UncompressedSize]; byte[] CompressedBufferArray = new byte[CompressedSize]; FileStream.Read(CompressedBufferArray, 0, CompressedSize); fixed(byte *UncompressedBufferPtr = UncompressedBufferArray, CompressedBufferPtr = CompressedBufferArray) { bool bResult = UE4UncompressMemoryZLIB((IntPtr)UncompressedBufferPtr, UncompressedSize, (IntPtr)CompressedBufferPtr, CompressedSize); if (!bResult) { CrashReporterProcessServicer.WriteFailure("! ZLibFail: Path=" + ReportPath); return(false); } } MemoryStream MemoryStream = new MemoryStream(UncompressedBufferArray, false); BinaryReader BinaryData = new BinaryReader(MemoryStream); for (int FileIndex = 0; FileIndex < Meta.GetNumberOfFiles(); FileIndex++) { Int32 CurrentFileIndex = BinaryData.ReadInt32(); if (CurrentFileIndex != FileIndex) { CrashReporterProcessServicer.WriteFailure("! ReadFail: Required=" + FileIndex + " Read=" + CurrentFileIndex); return(false); } string Filename = FBinaryReaderHelper.ReadFixedSizeString(BinaryData); Int32 FiledataLength = BinaryData.ReadInt32(); byte[] Filedata = BinaryData.ReadBytes(FiledataLength); File.WriteAllBytes(Path.Combine(ReportPath, Filename), Filedata); } } return(true); }
/// <summary> /// Main processing thread. /// </summary> /// <remarks>All exceptions are caught and written to the event log.</remarks> private void Init() { ProcessorTask = new Task(() => { // parse out hte list of blacklisted games. string[] GameNamesToBlacklist = (Config.Default.GameNamesToBlacklist == null ? "" : Config.Default.GameNamesToBlacklist).ToLowerInvariant().Split(','); while (!CancelSource.IsCancellationRequested) { try { bool bIdle = true; foreach (var Queue in Watcher.ReportQueues) { FGenericCrashContext NewContext = null; if (Queue.TryDequeueReport(out NewContext)) { bool bIsBlacklistedGame = GameNamesToBlacklist.Contains(NewContext.PrimaryCrashProperties.GameName.ToLowerInvariant()); // if it's blacklisted, skip it, else process it. if (bIsBlacklistedGame) { CrashReporterProcessServicer.WriteEvent(string.Format("Discarding crash from blacklisted GameName '{0}'", NewContext.PrimaryCrashProperties.GameName)); // delete the report from disk since we don't care about it. FinalizeReport(AddReportResult.Added, new DirectoryInfo(NewContext.CrashDirectory), NewContext); } else { ProcessReport(NewContext); } // The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest bIdle = false; break; } } if (bIdle) { // Don't use the CPU if we don't need. Thread.Sleep(1000); } } catch (Exception Ex) { CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessNewReports: " + Ex, Ex); } TickStatic(Watcher); } }); }
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DiagnosticsPath); if (CrashReporterProcessServicer.Symbolicator.Run(DiagnosticsPath, NewContext, ProcessorIndex)) { CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DiagnosticsPath); ReadDiagnosticsFile(NewContext); } else { CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DiagnosticsPath); } }
/// <summary> /// Delete a report directory. /// Thread-safe. /// </summary> /// <param name="DirInfo">The directory to delete</param> public static void CleanReport(DirectoryInfo DirInfo) { const int MaxRetries = 3; bool bWriteException = true; for (int Retry = 0; Retry < MaxRetries; ++Retry) { try { if (!DirInfo.Exists) { break; } foreach (FileInfo Info in DirInfo.GetFiles()) { Info.IsReadOnly = false; Info.Attributes = FileAttributes.Normal; } DirInfo.Delete(true /* delete contents */); // Random failures to delete with no exception seen regularly - retry DirInfo.Refresh(); if (DirInfo.Exists) { CrashReporterProcessServicer.WriteEvent("CleanReport: Failed to delete folder without an Exception " + DirInfo); Thread.Sleep(500); continue; } break; } catch (Exception Ex) { if (bWriteException) { CrashReporterProcessServicer.WriteException("CleanReport: " + Ex, Ex); bWriteException = false; } } System.Threading.Thread.Sleep(100); } DirInfo.Refresh(); if (DirInfo.Exists) { CrashReporterProcessServicer.WriteEvent(string.Format("CleanReport: Failed to delete folder {0} after {1} retries", DirInfo, MaxRetries)); } }
/// <summary> /// The main entry point for the application. /// </summary> static void Main() { CrashReporterProcessServicer CrashReporterProcess = new CrashReporterProcessServicer(); #if !DEBUG if( !Environment.UserInteractive ) { // Launch the service as normal if we aren't in the debugger ServiceBase.Run( CrashReporterProcess ); } else #endif { // Call OnStart, wait for a console key press, call OnStop CrashReporterProcess.DebugRun(); } }