void ProcessDumpFile(string DumpPath, FGenericCrashContext NewContext)
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DumpPath);
if (TrySimulateSymbolication(NewContext))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Simulated symbolication (skipped MDD) " + NewContext.CrashDirectory);
}
else if (CrashReporterProcessServicer.Symbolicator.Run(DumpPath, NewContext, ProcessorIndex))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DumpPath);
ReadDiagnosticsFile(NewContext);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DumpPath);
}
if (string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.CallStack))
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationFailedEvent);
}
else
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationSucceededEvent);
}
}
/// <summary>
/// A thread to watch for new crash reports landing.
/// </summary>
/// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks>
void Start()
{
CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:");
var Settings = Config.Default;
if (!string.IsNullOrEmpty(Settings.InternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.InternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority)", Settings.InternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority) is not accessible", Settings.InternalLandingZone));
}
}
#if !DEBUG
if (!string.IsNullOrEmpty(Settings.ExternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.ExternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0}", Settings.ExternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} is not accessible", Settings.ExternalLandingZone));
}
}
#endif //!DEBUG
// Init queue entries in StatusReporter
foreach (var Queue in ReportQueues)
{
CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath);
}
var Cancel = CancelSource.Token;
WatcherTask = Task.Factory.StartNew(async() =>
{
DateTime LastQueueSizeReport = DateTime.MinValue;
while (!Cancel.IsCancellationRequested)
{
// Check the landing zones for new reports
DateTime StartTime = DateTime.Now;
foreach (var Queue in ReportQueues)
{
int QueueSize = Queue.CheckForNewReports();
CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize);
}
TimeSpan TimeTaken = DateTime.Now - StartTime;
CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds));
await Task.Delay(60000, Cancel);
}
});
}
private static int UnpackRecordsFromDelimitedProtocolBuffers(Stream ProtocolBufferStream, string InLandingZone, string ReadableRequestString)
{
var UnpackedRecordCount = 0;
DataRouterConsumer Consumer = new DataRouterConsumer();
// Expect one or more pairs of varint size + protocol buffer in the stream
while (ProtocolBufferStream.Position < ProtocolBufferStream.Length)
{
DataRouterConsumer.ProtocolBufferRecord Message;
if (!Consumer.TryParse(ProtocolBufferStream, out Message))
{
string FailString = "! Protocol buffer parse fail in UnpackRecordsFromDelimitedProtocolBuffers(): " + ReadableRequestString;
FailString += '\n' + Consumer.LastError;
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
break;
}
if (DecompressDataRouterContent(Message.Payload, InLandingZone))
{
UnpackedRecordCount++;
}
}
return(UnpackedRecordCount);
}
protected static bool WriteIncomingFile(BinaryReader BinaryData, int FileIndex, string DestinationFolderPath)
{
try
{
Int32 CurrentFileIndex = BinaryData.ReadInt32();
if (CurrentFileIndex != FileIndex)
{
CrashReporterProcessServicer.WriteFailure("! WriteIncomingFile index mismatch: Required=" + FileIndex + " Read=" + CurrentFileIndex);
return(false);
}
string Filename = FBinaryReaderHelper.ReadFixedSizeString(BinaryData);
string SafeFilename = GetSafeFilename(Filename);
Int32 FiledataLength = BinaryData.ReadInt32();
byte[] Filedata = BinaryData.ReadBytes(FiledataLength);
Directory.CreateDirectory(DestinationFolderPath);
File.WriteAllBytes(Path.Combine(DestinationFolderPath, SafeFilename), Filedata);
return(true);
}
catch (Exception Ex)
{
throw new CrashReporterException(string.Format("! WriteIncomingFile failed writing FileIndex={0} FolderPath={1}", FileIndex, DestinationFolderPath), Ex);
}
}
/// <summary> Looks for the WER metadata xml file, if found, will return a new instance of the WERReportMetadata. </summary>
private WERReportMetadata FindMetadata(string NewReportPath)
{
WERReportMetadata MetaData = null;
// Just to verify that the report is still there.
DirectoryInfo DirInfo = new DirectoryInfo(NewReportPath);
if (!DirInfo.Exists)
{
CrashReporterProcessServicer.WriteFailure("FindMetadata: Directory not found " + NewReportPath);
}
else
{
// Check to see if we wish to suppress processing of this report.
foreach (var Info in DirInfo.GetFiles("*.xml"))
{
var MetaDataToCheck = XmlHandler.ReadXml <WERReportMetadata>(Info.FullName);
if (CheckMetaData(MetaDataToCheck))
{
MetaData = MetaDataToCheck;
break;
}
}
}
return(MetaData);
}
private void TryGetNewS3Crashes(int CrashCount)
{
int NewCrashCount = 0;
while (NewCrashCount < CrashCount)
{
string SQSRecord = "<unset>";
try
{
if (!DequeueRecordSQS(out SQSRecord))
{
// Queue empty
break;
}
var RecordPair = SQSRecord.Split(',');
if (RecordPair.Length != 2)
{
CrashReporterProcessServicer.WriteFailure("TryGetNewS3Crashes: bad SQS message was " + SQSRecord);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
continue;
}
string S3BucketName = RecordPair[0];
string S3Key = RecordPair[1];
string ReadableRequestString = "Bucket=" + S3BucketName + " Key=" + S3Key;
var ObjectRequest = new GetObjectRequest
{
BucketName = S3BucketName,
Key = S3Key
};
using (Stream ProtocolBufferStream = new MemoryStream())
{
using (GetObjectResponse ObjectResponse = S3Client.GetObject(ObjectRequest))
{
using (Stream ResponseStream = ObjectResponse.ResponseStream)
{
if (!TryDecompResponseStream(ResponseStream, ProtocolBufferStream))
{
CrashReporterProcessServicer.WriteFailure("! GZip fail in DecompResponseStream(): " + ReadableRequestString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
continue;
}
}
}
NewCrashCount += UnpackRecordsFromDelimitedProtocolBuffers(ProtocolBufferStream, LandingZone, ReadableRequestString);
}
}
catch (Exception ex)
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
CrashReporterProcessServicer.WriteException("TryGetNewS3Crashes: failure during processing SQS record " + SQSRecord +
"\n" + ex);
}
}
}
/// <summary>
/// Static init for one-time init jobs
/// </summary>
static ReportProcessor()
{
#if !DEBUG
if (!SyncRequiredFiles())
{
CrashReporterProcessServicer.WriteFailure("ReportProcessor: failed to sync files from Perforce");
}
#endif
}
public void ReadFromFile()
{
if (!IsEnabled)
{
return;
}
try
{
Index = new Dictionary <string, DateTime>();
if (!File.Exists(Filepath))
{
if (File.Exists(Filepath + ".backup"))
{
CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read ReportIndex from {0}. Attempting to read from {1}", Filepath, Filepath + ".backup"));
CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.ReadFromFile.UsingBackup", string.Format("Failed to read ReportIndex from {0}. Using backup.", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes);
File.Move(Filepath + ".backup", Filepath);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read ReportIndex from {0}. Generating new one.", Filepath));
File.Create(Filepath).Close();
}
}
using (var Reader = File.OpenText(Filepath))
{
string ItemStringRaw;
while ((ItemStringRaw = Reader.ReadLine()) != null)
{
string ItemString = ItemStringRaw.Trim();
if (!string.IsNullOrWhiteSpace(ItemString))
{
KeyValuePair <string, DateTime> NewItem;
if (!TryParseItemString(ItemString, out NewItem))
{
CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read line from ReportIndex: {0}.", ItemString));
continue;
}
Index.Add(NewItem.Key, NewItem.Value);
}
}
}
LastFlush = DateTime.UtcNow;
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("Failed to read ReportIndex from {0}. Exception was: {1}", Filepath, Ex), Ex);
CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.ReadFromFile", string.Format("Failed to read ReportIndex from {0}", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes);
}
}
/// <summary>
/// Static init for one-time init jobs
/// </summary>
static ReportProcessor()
{
#if !DEBUG
if (Config.Default.bSyncMinidumpDiagnostics)
{
if (!SyncMinidumpDiagnostics())
{
CrashReporterProcessServicer.WriteFailure("ReportProcessor: failed to sync files from Perforce");
}
}
#endif
}
/// <summary>
/// Decompresses a compressed crash report.
/// </summary>
unsafe private bool DecompressReport(string CompressedReportPath, FCompressedCrashInformation Meta)
{
string ReportPath = Path.GetDirectoryName(CompressedReportPath);
using (FileStream FileStream = File.OpenRead(CompressedReportPath))
{
Int32 UncompressedSize = Meta.GetUncompressedSize();
Int32 CompressedSize = Meta.GetCompressedSize();
if (FileStream.Length != CompressedSize)
{
return(false);
}
byte[] UncompressedBufferArray = new byte[UncompressedSize];
byte[] CompressedBufferArray = new byte[CompressedSize];
FileStream.Read(CompressedBufferArray, 0, CompressedSize);
fixed(byte *UncompressedBufferPtr = UncompressedBufferArray, CompressedBufferPtr = CompressedBufferArray)
{
bool bResult = UE4UncompressMemoryZLIB((IntPtr)UncompressedBufferPtr, UncompressedSize, (IntPtr)CompressedBufferPtr, CompressedSize);
if (!bResult)
{
CrashReporterProcessServicer.WriteFailure("! ZLibFail: Path=" + ReportPath);
return(false);
}
}
MemoryStream MemoryStream = new MemoryStream(UncompressedBufferArray, false);
BinaryReader BinaryData = new BinaryReader(MemoryStream);
for (int FileIndex = 0; FileIndex < Meta.GetNumberOfFiles(); FileIndex++)
{
Int32 CurrentFileIndex = BinaryData.ReadInt32();
if (CurrentFileIndex != FileIndex)
{
CrashReporterProcessServicer.WriteFailure("! ReadFail: Required=" + FileIndex + " Read=" + CurrentFileIndex);
return(false);
}
string Filename = FBinaryReaderHelper.ReadFixedSizeString(BinaryData);
Int32 FiledataLength = BinaryData.ReadInt32();
byte[] Filedata = BinaryData.ReadBytes(FiledataLength);
File.WriteAllBytes(Path.Combine(ReportPath, Filename), Filedata);
}
}
return(true);
}
/// <summary>
/// Call the web service function with an Xml payload to add a crash to the database.
/// </summary>
/// <param name="Payload">Xml representation of a new crash to upload.</param>
/// <returns>The database id of the newly added row.</returns>
private int UploadCrash(string Payload)
{
int NewID = -1;
try
{
// Simple suppression by blanking out the URL for local testing
if (Config.Default.CrashReportWebSite.Length > 0)
{
bool bDebug = false;
string RequestString;
if (!bDebug)
{
RequestString = "http://" + Config.Default.CrashReportWebSite + ":80/Crashes/AddCrash/-1";
}
else
{
RequestString = "http://localhost:80/Crashes/AddCrash/-1";
}
string ErrorMessage = string.Empty;
for (int Retry = 0; Retry < 3; ++Retry)
{
string ResponseString = SimpleWebRequest.GetWebServiceResponse(RequestString, Payload);
if (ResponseString.Length > 0)
{
// Convert response into a string
CrashReporterResult Result = XmlHandler.FromXmlString <CrashReporterResult>(ResponseString);
if (Result.ID > 0)
{
NewID = Result.ID;
break;
}
ErrorMessage = Result.Message;
}
Thread.Sleep(200);
}
if (NewID == -1)
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "UploadCrash: " + ErrorMessage);
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "UploadCrash: " + Ex.ToString());
}
return(NewID);
}
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext)
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DiagnosticsPath);
if (CrashReporterProcessServicer.Symbolicator.Run(DiagnosticsPath, NewContext, ProcessorIndex))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DiagnosticsPath);
ReadDiagnosticsFile(NewContext);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DiagnosticsPath);
}
}
/// <summary>
/// A thread to watch for new crash reports landing.
/// </summary>
/// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks>
void Start()
{
CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:");
var Settings = Properties.Settings.Default;
if (!string.IsNullOrEmpty(Settings.InternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.InternalLandingZone))
{
ReportQueues.Add(new ReportQueue(Settings.InternalLandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority)", Settings.InternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority) is not accessible", Settings.InternalLandingZone));
}
}
#if !DEBUG
if (!string.IsNullOrEmpty(Settings.ExternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.ExternalLandingZone))
{
ReportQueues.Add(new ReportQueue(Settings.ExternalLandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0}", Settings.ExternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} is not accessible", Settings.ExternalLandingZone));
}
}
#endif //!DEBUG
var Cancel = CancelSource.Token;
WatcherTask = Task.Factory.StartNew(async() =>
{
while (!Cancel.IsCancellationRequested)
{
// Check the landing zones for new reports
foreach (var Queue in ReportQueues)
{
Queue.CheckForNewReports();
}
await Task.Delay(60000, Cancel);
}
});
}
/// <summary>
/// Look for new report folders and add them to the publicly available thread-safe queue
/// </summary>
public void CheckForNewReports()
{
try
{
var NewReportPath = "";
var ReportsInLandingZone = new ReportIdSet();
// Check the landing zones for new reports
DirectoryInfo DirInfo = new DirectoryInfo(LandingZone);
// Couldn't find a landing zone, skip and try again later.
// Crash receiver will recreate them if needed.
if (!DirInfo.Exists)
{
CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone);
return;
}
// Add any new reports
foreach (var SubDirInfo in DirInfo.GetDirectories())
{
try
{
if (Directory.Exists(SubDirInfo.FullName))
{
NewReportPath = SubDirInfo.FullName;
ReportsInLandingZone.Add(NewReportPath);
if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath))
{
EnqueueNewReport(NewReportPath);
}
}
}
catch (System.Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex.ToString());
ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks);
}
}
//CrashReporterProcessServicer.WriteEvent( string.Format( "ReportsInLandingZone={0}, ReportsInLandingZoneLastTimeWeChecked={1}", ReportsInLandingZone.Count, ReportsInLandingZoneLastTimeWeChecked.Count ) );
ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone;
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex.ToString());
}
}
/// <summary>
/// Decompresses a compressed crash report.
/// </summary>
unsafe static protected bool DecompressReport(string CompressedReportPath, FCompressedCrashInformation Meta)
{
string ReportPath = Path.GetDirectoryName(CompressedReportPath);
using (FileStream FileStream = File.OpenRead(CompressedReportPath))
{
Int32 UncompressedSize = Meta.GetUncompressedSize();
Int32 CompressedSize = Meta.GetCompressedSize();
if (FileStream.Length != CompressedSize)
{
return(false);
}
byte[] UncompressedBufferArray = new byte[UncompressedSize];
byte[] CompressedBufferArray = new byte[CompressedSize];
FileStream.Read(CompressedBufferArray, 0, CompressedSize);
fixed(byte *UncompressedBufferPtr = UncompressedBufferArray, CompressedBufferPtr = CompressedBufferArray)
{
Int32 UncompressResult = NativeMethods.UE4UncompressMemoryZLIB((IntPtr)UncompressedBufferPtr, UncompressedSize, (IntPtr)CompressedBufferPtr, CompressedSize);
if (UncompressResult < 0)
{
CrashReporterProcessServicer.WriteFailure("! DecompressReport() failed in UE4UncompressMemoryZLIB() with " + NativeMethods.GetUncompressError(UncompressResult));
CrashReporterProcessServicer.WriteFailure("! ZLibFail in DecompressReport(): Path=" + ReportPath);
return(false);
}
}
using (BinaryReader BinaryData = new BinaryReader(new MemoryStream(UncompressedBufferArray, false)))
{
for (int FileIndex = 0; FileIndex < Meta.GetNumberOfFiles(); FileIndex++)
{
if (!WriteIncomingFile(BinaryData, FileIndex, ReportPath))
{
CrashReporterProcessServicer.WriteFailure("! DecompressReport() failed to write file index " + FileIndex + " Path=" + ReportPath);
return(false);
}
}
}
}
return(true);
}
protected virtual bool GetCrashesFromLandingZone(out DirectoryInfo[] OutDirectories)
{
// Check the landing zones for new reports
DirectoryInfo DirInfo = new DirectoryInfo(LandingZone);
// Couldn't find a landing zone, skip and try again later.
// Crash receiver will recreate them if needed.
if (!DirInfo.Exists)
{
CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone);
OutDirectories = new DirectoryInfo[0];
return(false);
}
OutDirectories = DirInfo.GetDirectories().OrderBy(dirinfo => dirinfo.CreationTimeUtc).ToArray();
return(true);
}
/// <summary>
/// Call the web service function with an Xml payload to add a crash to the database.
/// </summary>
/// <param name="Payload">Xml representation of a new crash to upload.</param>
/// <returns>The database id of the newly added row.</returns>
private int UploadCrash(string Payload)
{
int NewID = -1;
try
{
// Simple suppression by blanking out the URL for local testing
if (Properties.Settings.Default.CrashReportWebSite.Length > 0)
{
bool bDebug = false;
string RequestString;
if (!bDebug)
{
RequestString = "http://" + Properties.Settings.Default.CrashReportWebSite + ":80/Crashes/AddCrash/-1";
}
else
{
RequestString = "http://localhost:80/Crashes/AddCrash/-1";
}
string ResponseString = SimpleWebRequest.GetWebServiceResponse(RequestString, Payload);
if (ResponseString.Length > 0)
{
// Convert response into a string
CrashReporterResult Result = XmlHandler.FromXmlString <CrashReporterResult>(ResponseString);
if (Result.ID > 0)
{
NewID = Result.ID;
}
else
{
CrashReporterProcessServicer.WriteFailure("UploadCrash: " + Result.Message);
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("UploadCrash: " + Ex.ToString());
}
return(NewID);
}
/// <summary>
/// Sync the MinidumpDiagnostics binary and the engine config files to #head.
/// </summary>
/// <returns>true if all the files synced without issue, false otherwise.</returns>
/// <remarks>As MinidumpDiagnostics is synced to #head, it requires the engine config files that match to run properly.</remarks>
private static bool SyncMinidumpDiagnostics()
{
// Use the latest MinidumpDiagnostics from the main branch.
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.DepotRoot, Config.Default.DiskSpaceAlertPercent);
string UserString = "-u " + Config.Default.P4User;
string ClientString = "-c " + Config.Default.P4Client;
string SyncBinariesString = Path.Combine(Config.Default.DepotRoot, Config.Default.SyncBinariesFromDepot);
using (var MDDSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, UserString, ClientString, "sync", SyncBinariesString))
{
if (MDDSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure("Failed to sync MinidumpDiagnostics " + SyncBinariesString);
return(false);
}
}
string SyncConfigString = Path.Combine(Config.Default.DepotRoot, Config.Default.SyncConfigFromDepot);
using (var ConfigSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, UserString, ClientString, "sync", SyncConfigString))
{
if (ConfigSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure("Failed to sync config files " + SyncConfigString);
return(false);
}
}
string SyncThirdPartyString = Path.Combine(Config.Default.DepotRoot, Config.Default.SyncThirdPartyFromDepot); // Required by Perforce
using (var MiscSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, UserString, ClientString, "sync", SyncThirdPartyString))
{
if (MiscSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure("Failed to sync OpenSSL files " + SyncThirdPartyString);
return(false);
}
}
return(true);
}
/// <summary>
/// Sync the MinidumpDiagnostics binary and the engine config files to #head.
/// </summary>
/// <param name="MinidumpDiagnosticsName">The name of the binary to sync to process the minidump.</param>
/// <returns>true if all the files synced without issue, false otherwise.</returns>
/// <remarks>As MinidumpDiagnostics is synced to #head, it requires the engine config files that match to run properly.</remarks>
private bool SyncRequiredFiles(string MinidumpDiagnosticsName)
{
using (var MDDSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, "sync", MinidumpDiagnosticsName))
{
if (MDDSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure("Failed to sync MinidumpDiagnostics " + MinidumpDiagnosticsName);
return(false);
}
}
string ConfigFiles = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine/Config/...");
using (var ConfigSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, "sync", ConfigFiles))
{
if (ConfigSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure("Failed to sync config files " + ConfigFiles);
return(false);
}
}
string SSLFiles = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine/Binaries/ThirdParty/OpenSSL/..."); // Required by Perforce
using (var MiscSyncProc = new LaunchProcess(PerforceExePath, null, CrashReporterProcessServicer.WriteP4, "sync", SSLFiles))
{
if (MiscSyncProc.WaitForExit(SyncTimeoutSeconds * 1000) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure("Failed to sync OpenSSL files " + SSLFiles);
return(false);
}
}
return(true);
}
/// <summary> Enqueues a new crash. </summary>
void EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return;
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bFromWER = false;
if (Context == null || !Context.HasProcessedData())
{
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
FReportData ReportData = new FReportData(MetaData, NewReportPath);
Context = ConvertMetadataToCrashContext(MetaData, NewReportPath);
bFromWER = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
}
else
{
if (GenericContext != null && GenericContext.PrimaryCrashProperties.ErrorMessage.Length > 0)
{
// Get error message from the crash context and fix value in the metadata.
Context.PrimaryCrashProperties.ErrorMessage = GenericContext.PrimaryCrashProperties.ErrorMessage;
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// If based on WER, save to the file.
if (bFromWER && GenericContext == null)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteFailure("% NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.MoveReportToInvalid(NewReportPath, Context.GetAsFilename());
return;
}
if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteFailure("% UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.CleanReport(NewReportPath);
return;
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteFailure("! BROKEN0 : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.CleanReport(NewReportPath);
return;
}
lock (NewReportsLock)
{
NewCrashContexts.Add(Context);
}
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
}
}
/// <summary>
/// A function to add a report to the database, and rename the relevant files.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="LogFileName">The file name of the log file in the report.</param>
/// <param name="DumpFileName">The file name of the minidump in the report.</param>
/// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param>
private AddReportResult AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName)
{
try
{
Stopwatch AddReportTime = Stopwatch.StartNew();
// Create an XML representation of a crash
string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null);
if (CrashDetails == "")
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "! CreateCrash no payload: Path=" + NewContext.CrashDirectory);
return(AddReportResult.Failed);
}
// Upload the crash to the database, and retrieve the new row id
int ReportID = UploadCrash(CrashDetails);
if (ReportID <= 0)
{
if (CancelSource.IsCancellationRequested)
{
// Cancelled upload
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "CancelUpload: Path=" + NewContext.CrashDirectory);
return(AddReportResult.Cancelled);
}
else
{
// Upload failed
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "! NoUpload: Path=" + NewContext.CrashDirectory);
return(AddReportResult.Failed);
}
}
bool bToS3 = Config.Default.CrashFilesToAWS && CrashReporterProcessServicer.OutputAWS.IsS3Valid;
bool bToDisk = Config.Default.CrashFilesToDisk;
// Use the row id to name and move the files the way the web site requires
string IDThenUnderscore = string.Format("{0}_", ReportID);
int ReportIDSegment = (ReportID / 10000) * 10000;
string S3IDPrefix = string.Format("/{0}/{1}/{1}_", ReportIDSegment, ReportID);
string DestinationFolder = Path.Combine(Config.Default.ProcessedReports, IDThenUnderscore);
Stopwatch WriteToS3Timer = new Stopwatch();
Stopwatch WriteToDiskTimer = new Stopwatch();
if (bToDisk)
{
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(DestinationFolder, Config.Default.DiskSpaceAlertPercent);
}
// Save log file
if (LogFileName != null)
{
LogFileName = Path.Combine(DirInfo.FullName, LogFileName);
FileInfo LogInfo = new FileInfo(LogFileName);
if (LogInfo.Exists)
{
if (bToS3)
{
WriteToS3Timer.Start();
UploadFileToS3(LogInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + "Launch.log", Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix);
WriteToS3Timer.Stop();
}
if (bToDisk)
{
WriteToDiskTimer.Start();
LogInfo.MoveTo(DestinationFolder + "Launch.log");
WriteToDiskTimer.Stop();
}
}
}
// Save crash context file
string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName);
FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName);
if (CrashContextInfo.Exists)
{
if (bToS3)
{
WriteToS3Timer.Start();
UploadFileToS3(CrashContextInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + FGenericCrashContext.CrashContextRuntimeXMLName, false);
WriteToS3Timer.Stop();
}
if (bToDisk)
{
WriteToDiskTimer.Start();
CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName);
WriteToDiskTimer.Stop();
}
}
if (DumpFileName != null)
{
DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName);
FileInfo DumpInfo = new FileInfo(DumpFileName);
if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/)
{
if (bToS3)
{
WriteToS3Timer.Start();
UploadFileToS3(DumpInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + "MiniDump.dmp", Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix);
WriteToS3Timer.Stop();
}
if (bToDisk)
{
WriteToDiskTimer.Start();
DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp");
WriteToDiskTimer.Stop();
}
}
}
// Move the video (if it exists) to an alternate store
if (VideoFileName != null)
{
DestinationFolder = Path.Combine(Config.Default.ProcessedVideos, IDThenUnderscore);
VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName);
FileInfo VideoInfo = new FileInfo(VideoFileName);
if (VideoInfo.Exists)
{
if (bToS3)
{
WriteToS3Timer.Start();
UploadFileToS3(VideoInfo, Config.Default.AWSS3OutputKeyPrefix + S3IDPrefix + CrashReporterConstants.VideoFileName, Config.Default.CompressCrashFilesOnAWS, Config.Default.AWSS3CompressedSuffix);
WriteToS3Timer.Stop();
}
if (bToDisk)
{
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(DestinationFolder, Config.Default.DiskSpaceAlertPercent);
WriteToDiskTimer.Start();
VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName);
WriteToDiskTimer.Stop();
}
}
}
string TimeTakenString = string.Empty;
if (bToS3)
{
TimeTakenString = string.Format("S3UploadTime={0:F1} ", WriteToS3Timer.Elapsed.TotalSeconds);
}
if (bToDisk)
{
TimeTakenString += string.Format("DiskMoveTime={0:F1} ", WriteToDiskTimer.Elapsed.TotalSeconds);
}
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} AddReport: ReportID={1,8} {2}Path={3}", ProcessorIndex, ReportID, TimeTakenString, NewContext.CrashDirectory));
UpdateProcessedReports();
WebAddCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingSucceededEvent);
double Ratio = (double)WebAddCounter.TotalEvents / (double)ProcessedReportCount * 100;
double AddedPerDay = (double)WebAddCounter.TotalEvents / Timer.Elapsed.TotalDays;
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) +
string.Format(
"AddReport: Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4} AddedPerMinute={5:N1}", (int)Ratio,
ProcessedReportCount, WebAddCounter.TotalEvents, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay, WebAddCounter.EventsPerSecond * 60));
return(AddReportResult.Added);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "AddReport: " + DirInfo.Name + "\n\n" + Ex, Ex);
}
return(AddReportResult.Failed);
}
// From CrashUpload.cpp
/*
* struct FCompressedCrashFile : FNoncopyable
* {
* int32 CurrentFileIndex; // 4 bytes for file index
* FString Filename; // 4 bytes for length + 260 bytes for char data
* TArray<uint8> Filedata; // 4 bytes for length + N bytes for data
* }
*/
/// <summary> Enqueues a new crash. </summary>
private bool EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return(false);
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bContextDirty = false;
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
if (Context == null)
{
// Missing crash context
FReportData ReportData = new FReportData(MetaData, NewReportPath);
ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context);
bContextDirty = true;
}
else if (!Context.HasProcessedData())
{
// Missing data - try to get from WER metadata
FReportData ReportData = new FReportData(MetaData, NewReportPath);
GetErrorMessageFromMetadata(ReportData, Context);
bContextDirty = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// Added data from WER, save to the crash context file.
if (bContextDirty)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
Context.ToFile();
}
else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
Context.ToFile();
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
Context.ToFile();
}
// Check static reports index for duplicated reports
// This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router)
// This can be safely discontinued when all crashes come from the same SQS
// This DOES NOT scale to multiple CRP instances
// ReportIndex can be disabled by leaving the path empty in config
if (!CrashReporterProcessServicer.ReportIndex.TryAddNewReport(ReportName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
if (ShouldDecimateNextReport())
{
CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
lock (NewReportsLock)
{
NewCrashContexts.Enqueue(Context);
}
EnqueueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent);
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
return(true);
}
/// <summary>
/// Create thread to watch for new crash reports landing.
/// </summary>
/// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks>
void Init()
{
CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:");
var Settings = Config.Default;
if (!string.IsNullOrEmpty(Settings.DataRouterLandingZone))
{
if (System.IO.Directory.Exists(Settings.DataRouterLandingZone))
{
ReportQueues.Add(new DataRouterReportQueue("DataRouter Crashes", Settings.DataRouterLandingZone, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (all crashes from data router)", Settings.DataRouterLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (all crashes from data router) is not accessible", Settings.DataRouterLandingZone));
}
}
if (!string.IsNullOrEmpty(Settings.InternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.InternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone, StatusReportingEventNames.ProcessingStartedReceiverEvent, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority (legacy))", Settings.InternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority (legacy)) is not accessible", Settings.InternalLandingZone));
}
}
if (!string.IsNullOrEmpty(Settings.PS4LandingZone))
{
if (System.IO.Directory.Exists(Settings.PS4LandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("PS4 Crashes", Settings.PS4LandingZone, StatusReportingEventNames.ProcessingStartedPS4Event, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, PS4 non-retail hardware)", Settings.PS4LandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, PS4 non-retail hardware) is not accessible", Settings.PS4LandingZone));
}
}
#if !DEBUG
if (!string.IsNullOrEmpty(Settings.ExternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.ExternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone, StatusReportingEventNames.ProcessingStartedReceiverEvent, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (legacy)", Settings.ExternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (legacy) is not accessible", Settings.ExternalLandingZone));
}
}
#endif //!DEBUG
// Init queue entries in StatusReporter
foreach (var Queue in ReportQueues)
{
CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath);
}
var Cancel = CancelSource.Token;
WatcherTask = new Task(async() =>
{
DateTime LastQueueSizeReport = DateTime.MinValue;
while (!Cancel.IsCancellationRequested)
{
// Check the landing zones for new reports
DateTime StartTime = DateTime.Now;
foreach (var Queue in ReportQueues)
{
int QueueSize = Queue.CheckForNewReports();
CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize);
}
TimeSpan TimeTaken = DateTime.Now - StartTime;
CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds));
await Task.Delay(30000, Cancel);
}
});
}
/// <summary>
/// Look for new report folders and add them to the publicly available thread-safe queue.
/// </summary>
public override int CheckForNewReports()
{
try
{
if (QueueCount >= SkipQueueRefreshMin)
{
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: skipped busy queue size {0} in {1}", QueueCount, LandingZone));
return(LastQueueSizeOnDisk);
}
var NewReportPath = "";
var ReportsInLandingZone = new ReportIdSet();
// Check the landing zones for new reports
DirectoryInfo DirInfo = new DirectoryInfo(LandingZone);
// Couldn't find a landing zone, skip and try again later.
// Crash receiver will recreate them if needed.
if (!DirInfo.Exists)
{
CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone);
return(LastQueueSizeOnDisk);
}
var DirectoriesInLandingZone = DirInfo.GetDirectories().OrderBy(dirinfo => dirinfo.CreationTimeUtc).ToArray();
LastQueueSizeOnDisk = DirectoriesInLandingZone.Length;
int EnqueuedCount = 0;
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} reports in {1}", DirectoriesInLandingZone.Length, LandingZone));
// Add any new reports
for (int DirIndex = 0; DirIndex < DirectoriesInLandingZone.Length && QueueCount < QueueMax; DirIndex++)
{
var SubDirInfo = DirectoriesInLandingZone[DirIndex];
try
{
if (Directory.Exists(SubDirInfo.FullName))
{
NewReportPath = SubDirInfo.FullName;
ReportsInLandingZone.Add(NewReportPath);
if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath))
{
EnqueueNewReport(NewReportPath);
EnqueuedCount++;
}
}
}
catch (System.Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex.ToString());
ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks);
}
}
//CrashReporterProcessServicer.WriteEvent( string.Format( "ReportsInLandingZone={0}, ReportsInLandingZoneLastTimeWeChecked={1}", ReportsInLandingZone.Count, ReportsInLandingZoneLastTimeWeChecked.Count ) );
ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone;
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} enqueued to queue size {1} from {2}", EnqueuedCount, QueueCount, LandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: Enqueue rate {0:N1}/minute from {1}", EnqueueCounter.EventsPerSecond * 60, LandingZone));
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex.ToString());
}
return(LastQueueSizeOnDisk);
}
/// <summary> Enqueues a new crash. </summary>
private void EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return;
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bFromWER = false;
if (Context == null || !Context.HasProcessedData())
{
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
FReportData ReportData = new FReportData(MetaData, NewReportPath);
ConvertMetadataToCrashContext(MetaData, NewReportPath, ref Context);
bFromWER = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
}
else
{
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// Added data from WER, save to the crash context file.
if (bFromWER)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
}
else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
}
lock (NewReportsLock)
{
NewCrashContexts.Enqueue(Context);
}
EnqueueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingConstants.QueuedEvent);
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
}
}
/// <summary>
/// Create an Xml payload representing a new crash.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="bHasVideoFile">Whether the report contains a video file.</param>
/// <param name="bHasLog">Whether the report contains a log file.</param>
/// <param name="bHasMinidump">Whether the report contains a minidump.</param>
/// <returns>A string of Xml payload representing the newly found crash report.</returns>
private string CreateCrash(DirectoryInfo DirInfo, FGenericCrashContext NewContext, bool bHasVideoFile, bool bHasLog, bool bHasMinidump)
{
string XmlPayload = "";
try
{
FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion);
// Create a new crash description for uploading
CrashDescription NewCrash = new CrashDescription();
NewCrash.BranchName = EngineVersion.GetCleanedBranch();
NewCrash.GameName = NewContext.PrimaryCrashProperties.GameName;
NewCrash.Platform = NewContext.PrimaryCrashProperties.PlatformFullName;
NewCrash.EngineMode = NewContext.PrimaryCrashProperties.EngineMode;
NewCrash.BuildVersion = EngineVersion.VersionNumber;
NewCrash.CommandLine = NewContext.PrimaryCrashProperties.CommandLine;
NewCrash.BaseDir = NewContext.PrimaryCrashProperties.BaseDir;
NewCrash.Language = NewContext.PrimaryCrashProperties.AppDefaultLocale;
// // Create a locate and get the system language.
// int SystemLanguageCode = 0;
// int.TryParse( ReportData.SystemLanguage, out SystemLanguageCode );
// try
// {
// if( SystemLanguageCode > 0 )
// {
// CultureInfo SystemLanguageCI = new CultureInfo( SystemLanguageCode );
// NewCrash.SystemLanguage = SystemLanguageCI.Name;
// }
// }
// catch( System.Exception )
// {
// // Default to en-US
// NewCrash.SystemLanguage = "en-US";
// }
NewCrash.MachineGuid = NewContext.PrimaryCrashProperties.MachineId; // Valid for all kind of builds, previously only for UE4 releases.
NewCrash.UserName = NewContext.PrimaryCrashProperties.UserName; // Only valid for non-UE4 releases.
NewCrash.EpicAccountId = NewContext.PrimaryCrashProperties.EpicAccountId; // Only valid for UE4 releases.
NewCrash.CallStack = NewContext.PrimaryCrashProperties.GetCallstack();
NewCrash.SourceContext = NewContext.PrimaryCrashProperties.GetSourceContext();
NewCrash.ErrorMessage = NewContext.PrimaryCrashProperties.GetErrorMessage();
NewCrash.UserDescription = NewContext.PrimaryCrashProperties.GetUserDescription();
// Iterate through all files and find a file with the earliest date.
DateTime TimeOfCrash = DateTime.UtcNow;
foreach (var File in DirInfo.GetFiles())
{
if (File.CreationTimeUtc < TimeOfCrash)
{
TimeOfCrash = File.CreationTimeUtc;
}
}
//NewCrash.TimeofCrash = NewContext.PrimaryCrashProperties.TimeofCrash;
NewCrash.TimeofCrash = TimeOfCrash;
NewCrash.bHasMiniDump = bHasMinidump;
NewCrash.bHasLog = bHasLog;
NewCrash.bHasVideo = bHasVideoFile;
NewCrash.BuiltFromCL = (int)EngineVersion.Changelist;
NewCrash.bAllowToBeContacted = NewContext.PrimaryCrashProperties.bAllowToBeContacted;
// Ignore any callstack that is shorter than expected, usually the callstack is invalid.
if (NewCrash.CallStack.Length <= CrashReporterConstants.MinCallstackDepth)
{
CrashReporterProcessServicer.WriteFailure("! BadStack: BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory);
}
else
{
XmlPayload = XmlHandler.ToXmlString <CrashDescription>(NewCrash);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CreateCrash: " + Ex.ToString());
}
return(XmlPayload);
}
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext)
{
// Use the latest MinidumpDiagnostics from the main branch.
string Win64BinariesDirectory = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine", "Binaries", "Win64");
#if DEBUG
// Note: the debug executable must be built locally or synced from Perforce manually
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe");
//string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe");
#else
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe");
#endif
// Purge logs every 2048 processed crashes
string PurgeLogsDays = ProcessedReports % 2048 == 0 ? "2" : "-1";
FEngineVersion EngineVersion = new FEngineVersion(NewContext.PrimaryCrashProperties.EngineVersion);
List <string> MinidumpDiagnosticsParams = new List <string>
(
new string[]
{
"\"" + DiagnosticsPath + "\"",
"-BranchName=" + EngineVersion.Branch, // Backward compatibility
"-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility
"-GameName=" + NewContext.PrimaryCrashProperties.GameName,
"-EngineVersion=" + NewContext.PrimaryCrashProperties.EngineVersion,
"-bUsePDBCache=true",
"-Annotate",
"-SyncSymbols",
"-SyncMicrosoftSymbols",
"-unattended",
"-Log=" + NewContext.GetAsFilename() + "-backup-.log",
"-DepotIndex=" + Properties.Settings.Default.DepotIndex,
"-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays,
"-LOGTIMESINCESTART"
}
);
LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null;
if (Environment.UserInteractive)
{
CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD;
}
else
{
MinidumpDiagnosticsParams.AddRange
(
new[]
{
"-buildmachine",
"-forcelogflush"
}
);
// Write some debugging message.
CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams));
}
LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate, MinidumpDiagnosticsParams.ToArray());
if (ReportParser.WaitForExit(MinidumpDiagnosticsTimeoutSeconds * 1000) == EWaitResult.TimedOut)
{
CrashReporterProcessServicer.WriteFailure("ProcessDumpFile: Timed out running MinidumpDiagnostics");
}
ReadDiagnosticsFile(NewContext);
}
/// <summary>
/// A function to add a report to the database, and rename the relevant files.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="LogFileName">The file name of the log file in the report.</param>
/// <param name="DumpFileName">The file name of the minidump in the report.</param>
/// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param>
private bool AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName)
{
try
{
Stopwatch AddReportTime = Stopwatch.StartNew();
// Create an XML representation of a crash
string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null);
if (CrashDetails == "")
{
CrashReporterProcessServicer.WriteFailure("! NoDetail: Path=" + NewContext.CrashDirectory);
return(false);
}
// Upload the crash to the database, and retrieve the new row id
int ReportID = UploadCrash(CrashDetails);
if (ReportID <= 0)
{
CrashReporterProcessServicer.WriteFailure("! NoUpload: Path=" + NewContext.CrashDirectory);
return(false);
}
string IDThenUnderscore = string.Format("{0}_", ReportID);
// Use the row id to name and move the files the way the web site requires
string DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedReports, IDThenUnderscore);
// Move report files to crash reporter file store
if (LogFileName != null)
{
LogFileName = Path.Combine(DirInfo.FullName, LogFileName);
FileInfo LogInfo = new FileInfo(LogFileName);
if (LogInfo.Exists)
{
LogInfo.MoveTo(DestinationFolder + "Launch.log");
}
}
string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName);
FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName);
if (CrashContextInfo.Exists)
{
CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName);
}
// WERMetaDataName = Path.Combine(DirInfo.FullName, WERMetaDataName);
// FileInfo MetaInfo = new FileInfo(WERMetaDataName);
// if (MetaInfo.Exists)
// {
// MetaInfo.MoveTo(DestinationFolder + "WERMeta.xml");
// }
if (DumpFileName != null)
{
DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName);
FileInfo DumpInfo = new FileInfo(DumpFileName);
if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/)
{
DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp");
}
}
// DiagnosticsFileName = Path.Combine(DirInfo.FullName, DiagnosticsFileName);
// FileInfo DiagnosticsInfo = new FileInfo(DiagnosticsFileName);
// if (DiagnosticsInfo.Exists)
// {
// DiagnosticsInfo.MoveTo(DestinationFolder + CrashReporterConstants.DiagnosticsFileName);
// }
// Move the video (if it exists) to an alternate store
if (VideoFileName != null)
{
DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedVideos, IDThenUnderscore);
VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName);
FileInfo VideoInfo = new FileInfo(VideoFileName);
if (VideoInfo.Exists)
{
VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName);
}
}
CrashReporterProcessServicer.WriteEvent("# WebAdded: ReportID =" + string.Format("{0,7}", ReportID) + " Path=" + NewContext.CrashDirectory);
UpdateProcessedReports();
WebAddedReports++;
double Ratio = (double)WebAddedReports / (double)ProcessedReports * 100;
double AddedPerDay = (double)WebAddedReports / Timer.Elapsed.TotalDays;
CrashReporterProcessServicer.WriteEvent(string.Format("Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4}", (int)Ratio, ProcessedReports, WebAddedReports, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay));
return(true);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("AddReport: " + DirInfo.Name + "\n\n" + Ex.ToString());
}
return(false);
}
private static bool DecompressDataRouterContent(byte[] CompressedBufferArray, string InLandingZone)
{
// Decompress to landing zone
byte[] UncompressedBufferArray = new byte[Config.Default.MaxUncompressedS3RecordSize];
int UncompressedSize = NativeMethods.UncompressMemoryZlib(UncompressedBufferArray, CompressedBufferArray);
if (UncompressedSize < 0)
{
string FailString = "! DecompressDataRouterContent() failed in UncompressMemoryZlib() with " +
NativeMethods.GetZlibError(UncompressedSize);
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent);
return(false);
}
using (BinaryReader BinaryData = new BinaryReader(new MemoryStream(UncompressedBufferArray, 0, UncompressedSize, false)))
{
char[] MarkerChars = BinaryData.ReadChars(3);
if (MarkerChars[0] == 'C' && MarkerChars[1] == 'R' && MarkerChars[2] == '1')
{
CrashHeader CrashHeader = DataRouterReportQueue.CrashHeader.ParseCrashHeader(BinaryData);
// Create safe directory name and then create the directory on disk
string CrashFolderName = GetSafeFilename(CrashHeader.DirectoryName);
string CrashFolderPath = Path.Combine(InLandingZone, CrashFolderName);
// Early check for duplicate processed report
lock (ReportIndexLock)
{
if (CrashReporterProcessServicer.ReportIndex.ContainsReport(CrashFolderName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"DataRouterReportQueue: Duplicate report skipped early {0} in a DataRouterReportQueue", CrashFolderPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
return(false); // this isn't an error so don't set error message
}
}
// Create target folder
int TryIndex = 1;
while (Directory.Exists(CrashFolderPath))
{
CrashFolderPath = Path.Combine(InLandingZone, string.Format("{0}_DUPE{1:D3}", CrashFolderName, TryIndex++));
}
Directory.CreateDirectory(CrashFolderPath);
if (UncompressedSize != CrashHeader.UncompressedSize)
{
CrashReporterProcessServicer.WriteEvent(
string.Format(
"DecompressDataRouterContent() warning UncompressedSize mismatch (embedded={0}, actual={1}) Path={2}",
CrashHeader.UncompressedSize, UncompressedSize, CrashFolderPath));
}
if (!ParseCrashFiles(BinaryData, CrashHeader.FileCount, CrashFolderPath))
{
string FailString = "! DecompressDataRouterContent() failed to write files Path=" + CrashFolderPath;
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent);
return(false);
}
}
else
{
#if ALLOWOLDCLIENTDATA
// Early Data Router upload format was broken.
// Should be [CR1][CrashHeader][File][File][File][File]...
// Actually [Undefined CrashHeader][File][File][File][File]...[CrashHeader]
// Seek to end minus header size
BinaryData.BaseStream.Position = UncompressedSize - DataRouterReportQueue.CrashHeader.FixedSize;
var CrashHeader = DataRouterReportQueue.CrashHeader.ParseCrashHeader(BinaryData);
// Create safe directory name and then create the directory on disk
string CrashFolderName = GetSafeFilename(CrashHeader.DirectoryName);
string CrashFolderPath = Path.Combine(InLandingZone, CrashFolderName);
// Early check for duplicate processed report
lock (ReportIndexLock)
{
if (CrashReporterProcessServicer.ReportIndex.ContainsReport(CrashFolderName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"DataRouterReportQueue: Duplicate report skipped early {0} in a DataRouterReportQueue", CrashFolderPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
return(false); // this isn't an error so don't set error message
}
}
// Create target folder
int TryIndex = 1;
while (Directory.Exists(CrashFolderPath))
{
CrashFolderPath = Path.Combine(InLandingZone, string.Format("{0}_DUPE{1:D3}", CrashFolderName, TryIndex++));
}
Directory.CreateDirectory(CrashFolderPath);
if (UncompressedSize != CrashHeader.UncompressedSize + CrashHeader.FixedSize)
{
CrashReporterProcessServicer.WriteEvent(
string.Format(
"DecompressDataRouterContent() warning UncompressedSize mismatch (embedded={0}, actual={1}) Path={2}",
CrashHeader.UncompressedSize, UncompressedSize, CrashFolderPath));
}
// Seek to start of files (header size in from start)
BinaryData.BaseStream.Position = CrashHeader.FixedSize;
if (!ParseCrashFiles(BinaryData, CrashHeader.FileCount, CrashFolderPath))
{
string FailString = "! DecompressDataRouterContent() failed to write files Path=" + CrashFolderPath;
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent);
return(false);
}
#else
ErrorMessage = "! DecompressDataRouterContent() failed to read invalid data format. Corrupt or old format data received Path=" + CrashFolderPath;
return(false);
#endif
}
}
return(true);
}
double ReadProcessAddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext)
{
try
{
string DumpFileName = null;
string LogFileName = null;
string DiagnosticsFileName = "";
string VideoFileName = null;
string CrashContextName = "";
foreach (FileInfo Info in DirInfo.GetFiles())
{
switch (Info.Extension.ToLower())
{
case ".avi":
case ".mp4":
VideoFileName = Info.Name;
break;
case ".runtime-xml":
CrashContextName = Info.Name;
break;
case ".log":
LogFileName = Info.Name;
break;
case ".txt":
if (Info.Name == CrashReporterConstants.DiagnosticsFileName)
{
DiagnosticsFileName = Info.Name;
ReadDiagnosticsFile(NewContext);
}
break;
case ".dmp":
case ".mdmp":
DumpFileName = Info.Name;
// Check to see if this has been processed locally
FileInfo DiagnosticsInfo = new FileInfo(Path.Combine(DirInfo.FullName, CrashReporterConstants.DiagnosticsFileName));
if (!DiagnosticsInfo.Exists && !NewContext.HasProcessedData())
{
ProcessDumpFile(Info.FullName, NewContext);
}
// Check to see if a new one has been created
DiagnosticsInfo.Refresh();
if (DiagnosticsInfo.Exists)
{
DiagnosticsFileName = DiagnosticsInfo.Name;
}
break;
default:
break;
}
}
// Check if the new context has processed data.
if (NewContext.HasProcessedData())
{
Stopwatch WaitSW = Stopwatch.StartNew();
// Wait for previous task, should not really happen.
if (AddReportTask != null)
{
AddReportTask.Wait();
}
double WaitTime = WaitSW.Elapsed.TotalSeconds;
//bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName );
// Save/update crash context to the file.
NewContext.ToFile();
AddReportTask = Task.Factory.StartNew(() =>
{
bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName);
FinalizeReport(bAdded, DirInfo, NewContext);
});
return(WaitTime);
}
else
{
CrashReporterProcessServicer.WriteFailure("! Invalid : BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory);
FinalizeReport(false, DirInfo, NewContext);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("ProcessReport: " + NewContext.CrashDirectory + "\n\n: " + Ex.ToString());
}
return(0.0);
}