private bool DequeueRecordSQS(out string OutRecordString)
{
OutRecordString = string.Empty;
try
{
#if DEBUG
bool bNoDeleteFromSQS = true;
#else
bool bNoDeleteFromSQS = false;
#endif
Message DequeuedMessage = CrashReporterProcessServicer.DataRouterAWS.SQSDequeueMessage(Config.Default.AWSSQSQueueInputUrl, bNoDeleteFromSQS);
if (DequeuedMessage != null)
{
OutRecordString = DequeuedMessage.Body;
return(true);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("DequeueRecordSQS: " + Ex, Ex);
}
return(false);
}
private bool DequeueRecordSQS(out string OutRecordString)
{
OutRecordString = string.Empty;
try
{
var ReceiveRequest = new ReceiveMessageRequest
{
QueueUrl = Config.Default.AWSSQSQueueUrl,
MaxNumberOfMessages = 1
};
var ReceiveResponse = SqsClient.ReceiveMessage(ReceiveRequest);
if (ReceiveResponse.Messages.Count == 1)
{
var Message = ReceiveResponse.Messages[0];
if (Message != null && TryDeleteRecordSQS(Message))
{
OutRecordString = Message.Body;
return(true);
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("DequeueRecordSQS: " + Ex.ToString());
}
return(false);
}
/// <summary>
/// A thread to watch for new crash reports landing.
/// </summary>
/// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks>
void Start()
{
CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:");
var Settings = Config.Default;
if (!string.IsNullOrEmpty(Settings.InternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.InternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority)", Settings.InternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority) is not accessible", Settings.InternalLandingZone));
}
}
#if !DEBUG
if (!string.IsNullOrEmpty(Settings.ExternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.ExternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0}", Settings.ExternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} is not accessible", Settings.ExternalLandingZone));
}
}
#endif //!DEBUG
// Init queue entries in StatusReporter
foreach (var Queue in ReportQueues)
{
CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath);
}
var Cancel = CancelSource.Token;
WatcherTask = Task.Factory.StartNew(async() =>
{
DateTime LastQueueSizeReport = DateTime.MinValue;
while (!Cancel.IsCancellationRequested)
{
// Check the landing zones for new reports
DateTime StartTime = DateTime.Now;
foreach (var Queue in ReportQueues)
{
int QueueSize = Queue.CheckForNewReports();
CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize);
}
TimeSpan TimeTaken = DateTime.Now - StartTime;
CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds));
await Task.Delay(60000, Cancel);
}
});
}
/// <summary>
/// Clear out old MDD logs
/// </summary>
/// <param name="NumDays">Number of days worth of logs to keep</param>
public static void CleanOutOldLogs(int NumDays)
{
DateTime DeleteTime = DateTime.UtcNow - TimeSpan.FromDays(NumDays);
try
{
DirectoryInfo BaseFolder = new DirectoryInfo(CrashReporterProcessServicer.SymbolicatorLogFolder);
if (BaseFolder.Exists)
{
foreach (DirectoryInfo SubFolder in BaseFolder.EnumerateDirectories())
{
if (!SubFolder.EnumerateFiles("*", SearchOption.AllDirectories).Any(x => x.LastWriteTimeUtc > DeleteTime))
{
try
{
SubFolder.Delete(true);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", SubFolder.FullName, Ex.Message));
}
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs: {0}", Ex.Message));
}
}
private void TryGetNewS3Crashes(int CrashCount)
{
int NewCrashCount = 0;
while (NewCrashCount < CrashCount)
{
string SQSRecord = "<unset>";
try
{
if (!DequeueRecordSQS(out SQSRecord))
{
// Queue empty
break;
}
var RecordPair = SQSRecord.Split(',');
if (RecordPair.Length != 2)
{
CrashReporterProcessServicer.WriteFailure("TryGetNewS3Crashes: bad SQS message was " + SQSRecord);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
continue;
}
string S3BucketName = RecordPair[0];
string S3Key = RecordPair[1];
string ReadableRequestString = "Bucket=" + S3BucketName + " Key=" + S3Key;
var ObjectRequest = new GetObjectRequest
{
BucketName = S3BucketName,
Key = S3Key
};
using (Stream ProtocolBufferStream = new MemoryStream())
{
using (GetObjectResponse ObjectResponse = S3Client.GetObject(ObjectRequest))
{
using (Stream ResponseStream = ObjectResponse.ResponseStream)
{
if (!TryDecompResponseStream(ResponseStream, ProtocolBufferStream))
{
CrashReporterProcessServicer.WriteFailure("! GZip fail in DecompResponseStream(): " + ReadableRequestString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
continue;
}
}
}
NewCrashCount += UnpackRecordsFromDelimitedProtocolBuffers(ProtocolBufferStream, LandingZone, ReadableRequestString);
}
}
catch (Exception ex)
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
CrashReporterProcessServicer.WriteException("TryGetNewS3Crashes: failure during processing SQS record " + SQSRecord +
"\n" + ex);
}
}
}
private static int UnpackRecordsFromDelimitedProtocolBuffers(Stream ProtocolBufferStream, string InLandingZone, string ReadableRequestString)
{
var UnpackedRecordCount = 0;
DataRouterConsumer Consumer = new DataRouterConsumer();
// Expect one or more pairs of varint size + protocol buffer in the stream
while (ProtocolBufferStream.Position < ProtocolBufferStream.Length)
{
DataRouterConsumer.ProtocolBufferRecord Message;
if (!Consumer.TryParse(ProtocolBufferStream, out Message))
{
string FailString = "! Protocol buffer parse fail in UnpackRecordsFromDelimitedProtocolBuffers(): " + ReadableRequestString;
FailString += '\n' + Consumer.LastError;
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
break;
}
if (DecompressDataRouterContent(Message.Payload, InLandingZone))
{
UnpackedRecordCount++;
}
}
return(UnpackedRecordCount);
}
public static void WriteToFile()
{
try
{
if (File.Exists(Filepath))
{
File.Delete(Filepath + ".backup");
File.Move(Filepath, Filepath + ".backup");
}
using (var Writer = File.CreateText(Filepath))
{
var Today = DateTime.UtcNow.Date;
foreach (var Item in Index)
{
if (Today - Item.Value.Date <= Retention)
{
Writer.WriteLine(ItemToString(Item));
}
}
}
LastFlush = DateTime.UtcNow;
}
catch (Exception ex)
{
CrashReporterProcessServicer.WriteException(string.Format("Failed to write ReportIndex to {0}. Exception was: {1}", Filepath, ex));
CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.WriteToFile", string.Format("Failed to write ReportIndex to {0}", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes);
}
}
void ProcessDumpFile(string DumpPath, FGenericCrashContext NewContext)
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DumpPath);
if (TrySimulateSymbolication(NewContext))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Simulated symbolication (skipped MDD) " + NewContext.CrashDirectory);
}
else if (CrashReporterProcessServicer.Symbolicator.Run(DumpPath, NewContext, ProcessorIndex))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DumpPath);
ReadDiagnosticsFile(NewContext);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DumpPath);
}
if (string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.CallStack))
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationFailedEvent);
}
else
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationSucceededEvent);
}
}
/// <summary>
/// Moves specified report to the directory where are stored invalid reports.
/// Thread-safe.
/// </summary>
public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename)
{
try
{
DirectoryInfo DirInfo = new DirectoryInfo(ReportName);
// Rename the report directory, so we should be able to quickly find the invalid reports.
Directory.CreateDirectory(Properties.Settings.Default.InvalidReportsDirectory);
string CleanFilename = String.Concat(ReportNameAsFilename.Split(Path.GetInvalidFileNameChars()));
string DestinationDirectory = Path.Combine(Properties.Settings.Default.InvalidReportsDirectory, CleanFilename);
Directory.CreateDirectory(DestinationDirectory);
// Copy all files from the source directory. We can't use MoveTo due to different disc location.
foreach (FileInfo File in DirInfo.GetFiles())
{
string DestinationFilepath = Path.Combine(DestinationDirectory, File.Name);
File.CopyTo(DestinationFilepath, true);
}
DirInfo.Delete(true);
CrashReporterProcessServicer.WriteEvent(string.Format("Moved to {0}", DestinationDirectory));
UpdateProcessedReports();
}
catch (System.Exception Ex)
{
CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex.ToString());
}
}
public bool TryAddNewReport(string ReportKey)
{
if (!IsEnabled)
{
return(true);
}
lock (IndexLock)
{
if (Index.ContainsKey(ReportKey))
{
return(false);
}
Index.Add(ReportKey, DateTime.UtcNow);
}
lock (FileLock)
{
if (LastFlush.Date < DateTime.UtcNow.Date)
{
CrashReporterProcessServicer.WriteEvent("ReportIndex.TryAddNewReport flushing index to disk on schedule.");
WriteToFile();
}
}
return(true);
}
public StatusReportLoop(TimeSpan InLoopPeriod, Func <TimeSpan, string> LoopFunction)
{
InitialPause = TimeSpan.FromSeconds(30);
LoopPeriod = InLoopPeriod;
LoopTask = Task.Factory.StartNew(() =>
{
var Cancel = CancelSource.Token;
// Small pause to allow the app to get up and running before we run the first report
Thread.Sleep(InitialPause);
DateTime PeriodStartTime = CreationTimeUtc;
lock (TaskMonitor)
{
while (!Cancel.IsCancellationRequested)
{
DateTime PeriodEndTime = DateTime.UtcNow;
string ReportMessage = LoopFunction.Invoke(PeriodEndTime - PeriodStartTime);
PeriodStartTime = PeriodEndTime;
if (!string.IsNullOrWhiteSpace(ReportMessage))
{
CrashReporterProcessServicer.WriteSlack(ReportMessage);
}
Monitor.Wait(TaskMonitor, LoopPeriod);
}
}
});
}
protected static bool WriteIncomingFile(BinaryReader BinaryData, int FileIndex, string DestinationFolderPath)
{
try
{
Int32 CurrentFileIndex = BinaryData.ReadInt32();
if (CurrentFileIndex != FileIndex)
{
CrashReporterProcessServicer.WriteFailure("! WriteIncomingFile index mismatch: Required=" + FileIndex + " Read=" + CurrentFileIndex);
return(false);
}
string Filename = FBinaryReaderHelper.ReadFixedSizeString(BinaryData);
string SafeFilename = GetSafeFilename(Filename);
Int32 FiledataLength = BinaryData.ReadInt32();
byte[] Filedata = BinaryData.ReadBytes(FiledataLength);
Directory.CreateDirectory(DestinationFolderPath);
File.WriteAllBytes(Path.Combine(DestinationFolderPath, SafeFilename), Filedata);
return(true);
}
catch (Exception Ex)
{
throw new CrashReporterException(string.Format("! WriteIncomingFile failed writing FileIndex={0} FolderPath={1}", FileIndex, DestinationFolderPath), Ex);
}
}
/// <summary>
/// Moves specified report to the directory where are stored invalid reports.
/// Thread-safe.
/// </summary>
public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename)
{
try
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingFailedEvent);
DirectoryInfo DirInfo = new DirectoryInfo(ReportName);
// Rename the report directory, so we should be able to quickly find the invalid reports.
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.InvalidReportsDirectory, Config.Default.DiskSpaceAlertPercent);
Directory.CreateDirectory(Config.Default.InvalidReportsDirectory);
string DestinationDirectory = Path.Combine(Config.Default.InvalidReportsDirectory, ReportNameAsFilename);
Directory.CreateDirectory(DestinationDirectory);
// Copy all files from the source directory. We can't use MoveTo due to different disc location.
foreach (FileInfo File in DirInfo.GetFiles())
{
string DestinationFilepath = Path.Combine(DestinationDirectory, File.Name);
File.CopyTo(DestinationFilepath, true);
}
DirInfo.Delete(true);
CrashReporterProcessServicer.WriteEvent(string.Format("Moved to {0}", DestinationDirectory));
UpdateProcessedReports();
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex);
}
}
/// <summary> Looks for the WER metadata xml file, if found, will return a new instance of the WERReportMetadata. </summary>
private WERReportMetadata FindMetadata(string NewReportPath)
{
WERReportMetadata MetaData = null;
// Just to verify that the report is still there.
DirectoryInfo DirInfo = new DirectoryInfo(NewReportPath);
if (!DirInfo.Exists)
{
CrashReporterProcessServicer.WriteFailure("FindMetadata: Directory not found " + NewReportPath);
}
else
{
// Check to see if we wish to suppress processing of this report.
foreach (var Info in DirInfo.GetFiles("*.xml"))
{
var MetaDataToCheck = XmlHandler.ReadXml <WERReportMetadata>(Info.FullName);
if (CheckMetaData(MetaDataToCheck))
{
MetaData = MetaDataToCheck;
break;
}
}
}
return(MetaData);
}
public void AlertOnLowDisk(string Filepath, float AlertThresholdPercent)
{
try
{
string Drive;
if (!StorageSpaceHelper.TryGetDriveLetter(Filepath, out Drive))
{
throw new CrashReporterException("Failed to get drive letter for path " + Filepath);
}
Int64 FreeSpace;
float FreePercent;
if (StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent))
{
if (FreePercent < AlertThresholdPercent)
{
Alert("AlertOnLowDisk" + Drive, "Low disk space warning on " + Drive + " =>> " + GetDiskSpaceString(FreeSpace, FreePercent), 3 * Config.Default.SlackAlertRepeatMinimumMinutes);
}
}
else
{
CrashReporterProcessServicer.WriteEvent("Failed to read disk space for " + Drive);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("AlertOnLowDisk failed: " + Ex, Ex);
}
}
/// <summary>
/// Delete a report directory.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The directory to delete</param>
public static void CleanReport(DirectoryInfo DirInfo)
{
bool bWriteException = true;
for (int Retry = 0; Retry < 3; ++Retry)
{
try
{
foreach (FileInfo Info in DirInfo.GetFiles())
{
Info.IsReadOnly = false;
}
DirInfo.Delete(true /* delete contents */);
break;
}
catch (Exception Ex)
{
if (bWriteException)
{
CrashReporterProcessServicer.WriteException("CleanReport: " + Ex.ToString());
bWriteException = false;
}
}
System.Threading.Thread.Sleep(100);
}
}
/// <summary>
/// Shutdown: stop the thread. Will request stop first if RequestStop() hasn't been called. Then blocks until the processor thread exits.
/// </summary>
public void Dispose()
{
if (!CancelSource.IsCancellationRequested)
{
CancelSource.Cancel();
}
CrashReporterProcessServicer.WriteEvent("Shutdown: Stopping ReportProcessor thread...");
ProcessorTask.Wait();
ProcessorTask.Dispose();
CrashReporterProcessServicer.WriteEvent("Shutdown: ReportProcessor thread stopped.");
CrashReporterProcessServicer.WriteEvent("Shutdown: Stopping ReportProcessor's AddReport threads...");
foreach (Task AddReportTask in AddReportTasks)
{
if (AddReportTask != null)
{
AddReportTask.Wait();
AddReportTask.Dispose();
}
}
CrashReporterProcessServicer.WriteEvent("Shutdown: AddReport threads stopped.");
CancelSource.Dispose();
}
public void Alert(string AlertKey, string AlertText, int RepeatMinimumMinutes)
{
// Do "no repeat" check here of recent alerts and block duplicates based on timer (use AlertKey)
if (CheckRecentAlerts(AlertKey, RepeatMinimumMinutes))
{
// Report an alert condition to Slack immediately
CrashReporterProcessServicer.WriteSlack("@channel ALERT: " + AlertText);
}
}
/// <summary>
/// Static init for one-time init jobs
/// </summary>
static ReportProcessor()
{
#if !DEBUG
if (!SyncRequiredFiles())
{
CrashReporterProcessServicer.WriteFailure("ReportProcessor: failed to sync files from Perforce");
}
#endif
}
private static void UploadFileToS3(FileInfo FileInfo, string DestFilepath, bool bCompressed, string CompressedSuffix = null)
{
try
{
if (bCompressed)
{
string LocalCompressedFilepath = FileInfo.FullName;
if (CompressedSuffix != null)
{
LocalCompressedFilepath += CompressedSuffix;
}
else
{
LocalCompressedFilepath += ".gz";
}
byte[] UncompressedBytes = File.ReadAllBytes(FileInfo.FullName);
int ReturnCode = NativeMethods.CompressFileGZIP(LocalCompressedFilepath, UncompressedBytes);
if (ReturnCode < 0)
{
throw new CrashReporterException(string.Format("Failed to compress {0} to gzip file with error {1}", FileInfo.FullName, NativeMethods.GetZlibError(ReturnCode)));
}
string CompressedDestFilepath = DestFilepath;
if (CompressedSuffix != null)
{
CompressedDestFilepath += CompressedSuffix;
}
PutObjectResponse Response = CrashReporterProcessServicer.OutputAWS.PutS3ObjectFromFile(Config.Default.AWSS3OutputBucket,
CompressedDestFilepath,
LocalCompressedFilepath);
if (Response == null || Response.HttpStatusCode != HttpStatusCode.OK)
{
throw new CrashReporterException(string.Format("Failed to upload compressed gzip {0} to {1}", LocalCompressedFilepath, CompressedDestFilepath));
}
}
else
{
PutObjectResponse Response = CrashReporterProcessServicer.OutputAWS.PutS3ObjectFromFile(Config.Default.AWSS3OutputBucket,
DestFilepath,
FileInfo.FullName);
if (Response == null || Response.HttpStatusCode != HttpStatusCode.OK)
{
throw new CrashReporterException(string.Format("Failed to upload {0} to {1}", FileInfo.FullName, DestFilepath));
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("UploadFileToS3: " + Ex, Ex);
}
}
/// <summary>
/// Clear out old MDD logs
/// </summary>
/// <param name="NumDays">Number of days worth of logs to keep</param>
public static void CleanOutOldLogs(int NumDays)
{
DateTime DeleteTime = DateTime.UtcNow - TimeSpan.FromDays(NumDays);
try
{
DirectoryInfo BaseFolder = new DirectoryInfo(CrashReporterProcessServicer.SymbolicatorLogFolder);
if (BaseFolder.Exists)
{
foreach (DirectoryInfo SubFolder in BaseFolder.EnumerateDirectories())
{
if (!SubFolder.EnumerateFiles("*", SearchOption.AllDirectories).Any(x => x.LastWriteTimeUtc > DeleteTime))
{
try
{
SubFolder.Delete(true);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", SubFolder.FullName, Ex.Message));
}
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs: {0}", Ex.Message));
}
try
{
DirectoryInfo BaseFolder = new DirectoryInfo(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Config.Default.MDDExecutablePath), "..", "..", "Programs", "MinidumpDiagnostics", "Saved", "Logs")));
if (BaseFolder.Exists)
{
foreach (FileInfo File in BaseFolder.EnumerateFiles("*", SearchOption.AllDirectories))
{
try
{
if (File.LastWriteTimeUtc < DeleteTime)
{
File.Delete();
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", File.FullName, Ex.Message));
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs from program folder: {0}", Ex.Message));
}
}
public void Dispose()
{
if (ReporterTasks != null)
{
foreach (var ReporterTask in ReporterTasks)
{
ReporterTask.Dispose();
}
}
CrashReporterProcessServicer.WriteSlack("CRP stopped");
}
/// <summary>
/// Static init for one-time init jobs
/// </summary>
static ReportProcessor()
{
#if !DEBUG
if (Config.Default.bSyncMinidumpDiagnostics)
{
if (!SyncMinidumpDiagnostics())
{
CrashReporterProcessServicer.WriteFailure("ReportProcessor: failed to sync files from Perforce");
}
}
#endif
}
public void ReadFromFile()
{
if (!IsEnabled)
{
return;
}
try
{
Index = new Dictionary <string, DateTime>();
if (!File.Exists(Filepath))
{
if (File.Exists(Filepath + ".backup"))
{
CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read ReportIndex from {0}. Attempting to read from {1}", Filepath, Filepath + ".backup"));
CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.ReadFromFile.UsingBackup", string.Format("Failed to read ReportIndex from {0}. Using backup.", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes);
File.Move(Filepath + ".backup", Filepath);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read ReportIndex from {0}. Generating new one.", Filepath));
File.Create(Filepath).Close();
}
}
using (var Reader = File.OpenText(Filepath))
{
string ItemStringRaw;
while ((ItemStringRaw = Reader.ReadLine()) != null)
{
string ItemString = ItemStringRaw.Trim();
if (!string.IsNullOrWhiteSpace(ItemString))
{
KeyValuePair <string, DateTime> NewItem;
if (!TryParseItemString(ItemString, out NewItem))
{
CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read line from ReportIndex: {0}.", ItemString));
continue;
}
Index.Add(NewItem.Key, NewItem.Value);
}
}
}
LastFlush = DateTime.UtcNow;
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("Failed to read ReportIndex from {0}. Exception was: {1}", Filepath, Ex), Ex);
CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.ReadFromFile", string.Format("Failed to read ReportIndex from {0}", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes);
}
}
/// <summary>
/// Returns the count of items in the SQS
/// </summary>
private int GetSQSCount()
{
try
{
return(CrashReporterProcessServicer.DataRouterAWS.GetSQSQueueCount(Config.Default.AWSSQSQueueInputUrl));
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("GetSQSCount: " + Ex, Ex);
}
return(0);
}
/// <summary>
/// Call the web service function with an Xml payload to add a crash to the database.
/// </summary>
/// <param name="Payload">Xml representation of a new crash to upload.</param>
/// <returns>The database id of the newly added row.</returns>
private int UploadCrash(string Payload)
{
int NewID = -1;
try
{
// Simple suppression by blanking out the URL for local testing
if (Config.Default.CrashReportWebSite.Length > 0)
{
bool bDebug = false;
string RequestString;
if (!bDebug)
{
RequestString = "http://" + Config.Default.CrashReportWebSite + ":80/Crashes/AddCrash/-1";
}
else
{
RequestString = "http://localhost:80/Crashes/AddCrash/-1";
}
string ErrorMessage = string.Empty;
for (int Retry = 0; Retry < 3; ++Retry)
{
string ResponseString = SimpleWebRequest.GetWebServiceResponse(RequestString, Payload);
if (ResponseString.Length > 0)
{
// Convert response into a string
CrashReporterResult Result = XmlHandler.FromXmlString <CrashReporterResult>(ResponseString);
if (Result.ID > 0)
{
NewID = Result.ID;
break;
}
ErrorMessage = Result.Message;
}
Thread.Sleep(200);
}
if (NewID == -1)
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "UploadCrash: " + ErrorMessage);
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "UploadCrash: " + Ex.ToString());
}
return(NewID);
}
/// <summary>
/// Decompresses a compressed crash report.
/// </summary>
unsafe private bool DecompressReport(string CompressedReportPath, FCompressedCrashInformation Meta)
{
string ReportPath = Path.GetDirectoryName(CompressedReportPath);
using (FileStream FileStream = File.OpenRead(CompressedReportPath))
{
Int32 UncompressedSize = Meta.GetUncompressedSize();
Int32 CompressedSize = Meta.GetCompressedSize();
if (FileStream.Length != CompressedSize)
{
return(false);
}
byte[] UncompressedBufferArray = new byte[UncompressedSize];
byte[] CompressedBufferArray = new byte[CompressedSize];
FileStream.Read(CompressedBufferArray, 0, CompressedSize);
fixed(byte *UncompressedBufferPtr = UncompressedBufferArray, CompressedBufferPtr = CompressedBufferArray)
{
bool bResult = UE4UncompressMemoryZLIB((IntPtr)UncompressedBufferPtr, UncompressedSize, (IntPtr)CompressedBufferPtr, CompressedSize);
if (!bResult)
{
CrashReporterProcessServicer.WriteFailure("! ZLibFail: Path=" + ReportPath);
return(false);
}
}
MemoryStream MemoryStream = new MemoryStream(UncompressedBufferArray, false);
BinaryReader BinaryData = new BinaryReader(MemoryStream);
for (int FileIndex = 0; FileIndex < Meta.GetNumberOfFiles(); FileIndex++)
{
Int32 CurrentFileIndex = BinaryData.ReadInt32();
if (CurrentFileIndex != FileIndex)
{
CrashReporterProcessServicer.WriteFailure("! ReadFail: Required=" + FileIndex + " Read=" + CurrentFileIndex);
return(false);
}
string Filename = FBinaryReaderHelper.ReadFixedSizeString(BinaryData);
Int32 FiledataLength = BinaryData.ReadInt32();
byte[] Filedata = BinaryData.ReadBytes(FiledataLength);
File.WriteAllBytes(Path.Combine(ReportPath, Filename), Filedata);
}
}
return(true);
}
/// <summary>
/// Main processing thread.
/// </summary>
/// <remarks>All exceptions are caught and written to the event log.</remarks>
private void Init()
{
ProcessorTask = new Task(() =>
{
// parse out hte list of blacklisted games.
string[] GameNamesToBlacklist = (Config.Default.GameNamesToBlacklist == null ? "" : Config.Default.GameNamesToBlacklist).ToLowerInvariant().Split(',');
while (!CancelSource.IsCancellationRequested)
{
try
{
bool bIdle = true;
foreach (var Queue in Watcher.ReportQueues)
{
FGenericCrashContext NewContext = null;
if (Queue.TryDequeueReport(out NewContext))
{
bool bIsBlacklistedGame = GameNamesToBlacklist.Contains(NewContext.PrimaryCrashProperties.GameName.ToLowerInvariant());
// if it's blacklisted, skip it, else process it.
if (bIsBlacklistedGame)
{
CrashReporterProcessServicer.WriteEvent(string.Format("Discarding crash from blacklisted GameName '{0}'", NewContext.PrimaryCrashProperties.GameName));
// delete the report from disk since we don't care about it.
FinalizeReport(AddReportResult.Added, new DirectoryInfo(NewContext.CrashDirectory), NewContext);
}
else
{
ProcessReport(NewContext);
}
// The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest
bIdle = false;
break;
}
}
if (bIdle)
{
// Don't use the CPU if we don't need.
Thread.Sleep(1000);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessNewReports: " + Ex, Ex);
}
TickStatic(Watcher);
}
});
}
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext)
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DiagnosticsPath);
if (CrashReporterProcessServicer.Symbolicator.Run(DiagnosticsPath, NewContext, ProcessorIndex))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DiagnosticsPath);
ReadDiagnosticsFile(NewContext);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DiagnosticsPath);
}
}
/// <summary>
/// Delete a report directory.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The directory to delete</param>
public static void CleanReport(DirectoryInfo DirInfo)
{
const int MaxRetries = 3;
bool bWriteException = true;
for (int Retry = 0; Retry < MaxRetries; ++Retry)
{
try
{
if (!DirInfo.Exists)
{
break;
}
foreach (FileInfo Info in DirInfo.GetFiles())
{
Info.IsReadOnly = false;
Info.Attributes = FileAttributes.Normal;
}
DirInfo.Delete(true /* delete contents */);
// Random failures to delete with no exception seen regularly - retry
DirInfo.Refresh();
if (DirInfo.Exists)
{
CrashReporterProcessServicer.WriteEvent("CleanReport: Failed to delete folder without an Exception " + DirInfo);
Thread.Sleep(500);
continue;
}
break;
}
catch (Exception Ex)
{
if (bWriteException)
{
CrashReporterProcessServicer.WriteException("CleanReport: " + Ex, Ex);
bWriteException = false;
}
}
System.Threading.Thread.Sleep(100);
}
DirInfo.Refresh();
if (DirInfo.Exists)
{
CrashReporterProcessServicer.WriteEvent(string.Format("CleanReport: Failed to delete folder {0} after {1} retries", DirInfo, MaxRetries));
}
}
/// <summary>
/// The main entry point for the application.
/// </summary>
static void Main()
{
CrashReporterProcessServicer CrashReporterProcess = new CrashReporterProcessServicer();
#if !DEBUG
if( !Environment.UserInteractive )
{
// Launch the service as normal if we aren't in the debugger
ServiceBase.Run( CrashReporterProcess );
}
else
#endif
{
// Call OnStart, wait for a console key press, call OnStop
CrashReporterProcess.DebugRun();
}
}