/// <summary>
/// Moves specified report to the directory where are stored invalid reports.
/// Thread-safe.
/// </summary>
public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename)
{
try
{
DirectoryInfo DirInfo = new DirectoryInfo(ReportName);
// Rename the report directory, so we should be able to quickly find the invalid reports.
Directory.CreateDirectory(Properties.Settings.Default.InvalidReportsDirectory);
string CleanFilename = String.Concat(ReportNameAsFilename.Split(Path.GetInvalidFileNameChars()));
string DestinationDirectory = Path.Combine(Properties.Settings.Default.InvalidReportsDirectory, CleanFilename);
Directory.CreateDirectory(DestinationDirectory);
// Copy all files from the source directory. We can't use MoveTo due to different disc location.
foreach (FileInfo File in DirInfo.GetFiles())
{
string DestinationFilepath = Path.Combine(DestinationDirectory, File.Name);
File.CopyTo(DestinationFilepath, true);
}
DirInfo.Delete(true);
CrashReporterProcessServicer.WriteEvent(string.Format("Moved to {0}", DestinationDirectory));
UpdateProcessedReports();
}
catch (System.Exception Ex)
{
CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex.ToString());
}
}
public bool TryAddNewReport(string ReportKey)
{
if (!IsEnabled)
{
return(true);
}
lock (IndexLock)
{
if (Index.ContainsKey(ReportKey))
{
return(false);
}
Index.Add(ReportKey, DateTime.UtcNow);
}
lock (FileLock)
{
if (LastFlush.Date < DateTime.UtcNow.Date)
{
CrashReporterProcessServicer.WriteEvent("ReportIndex.TryAddNewReport flushing index to disk on schedule.");
WriteToFile();
}
}
return(true);
}
/// <summary>
/// A thread to watch for new crash reports landing.
/// </summary>
/// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks>
void Start()
{
CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:");
var Settings = Config.Default;
if (!string.IsNullOrEmpty(Settings.InternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.InternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority)", Settings.InternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority) is not accessible", Settings.InternalLandingZone));
}
}
#if !DEBUG
if (!string.IsNullOrEmpty(Settings.ExternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.ExternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0}", Settings.ExternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} is not accessible", Settings.ExternalLandingZone));
}
}
#endif //!DEBUG
// Init queue entries in StatusReporter
foreach (var Queue in ReportQueues)
{
CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath);
}
var Cancel = CancelSource.Token;
WatcherTask = Task.Factory.StartNew(async() =>
{
DateTime LastQueueSizeReport = DateTime.MinValue;
while (!Cancel.IsCancellationRequested)
{
// Check the landing zones for new reports
DateTime StartTime = DateTime.Now;
foreach (var Queue in ReportQueues)
{
int QueueSize = Queue.CheckForNewReports();
CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize);
}
TimeSpan TimeTaken = DateTime.Now - StartTime;
CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds));
await Task.Delay(60000, Cancel);
}
});
}
/// <summary>
/// Shutdown: stop the thread. Will request stop first if RequestStop() hasn't been called. Then blocks until the processor thread exits.
/// </summary>
public void Dispose()
{
if (!CancelSource.IsCancellationRequested)
{
CancelSource.Cancel();
}
CrashReporterProcessServicer.WriteEvent("Shutdown: Stopping ReportProcessor thread...");
ProcessorTask.Wait();
ProcessorTask.Dispose();
CrashReporterProcessServicer.WriteEvent("Shutdown: ReportProcessor thread stopped.");
CrashReporterProcessServicer.WriteEvent("Shutdown: Stopping ReportProcessor's AddReport threads...");
foreach (Task AddReportTask in AddReportTasks)
{
if (AddReportTask != null)
{
AddReportTask.Wait();
AddReportTask.Dispose();
}
}
CrashReporterProcessServicer.WriteEvent("Shutdown: AddReport threads stopped.");
CancelSource.Dispose();
}
public void AlertOnLowDisk(string Filepath, float AlertThresholdPercent)
{
try
{
string Drive;
if (!StorageSpaceHelper.TryGetDriveLetter(Filepath, out Drive))
{
throw new CrashReporterException("Failed to get drive letter for path " + Filepath);
}
Int64 FreeSpace;
float FreePercent;
if (StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent))
{
if (FreePercent < AlertThresholdPercent)
{
Alert("AlertOnLowDisk" + Drive, "Low disk space warning on " + Drive + " =>> " + GetDiskSpaceString(FreeSpace, FreePercent), 3 * Config.Default.SlackAlertRepeatMinimumMinutes);
}
}
else
{
CrashReporterProcessServicer.WriteEvent("Failed to read disk space for " + Drive);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("AlertOnLowDisk failed: " + Ex, Ex);
}
}
void ProcessDumpFile(string DumpPath, FGenericCrashContext NewContext)
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DumpPath);
if (TrySimulateSymbolication(NewContext))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Simulated symbolication (skipped MDD) " + NewContext.CrashDirectory);
}
else if (CrashReporterProcessServicer.Symbolicator.Run(DumpPath, NewContext, ProcessorIndex))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DumpPath);
ReadDiagnosticsFile(NewContext);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DumpPath);
}
if (string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.CallStack))
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationFailedEvent);
}
else
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.SymbolicationSucceededEvent);
}
}
/// <summary>
/// Moves specified report to the directory where are stored invalid reports.
/// Thread-safe.
/// </summary>
public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename)
{
try
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingFailedEvent);
DirectoryInfo DirInfo = new DirectoryInfo(ReportName);
// Rename the report directory, so we should be able to quickly find the invalid reports.
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.InvalidReportsDirectory, Config.Default.DiskSpaceAlertPercent);
Directory.CreateDirectory(Config.Default.InvalidReportsDirectory);
string DestinationDirectory = Path.Combine(Config.Default.InvalidReportsDirectory, ReportNameAsFilename);
Directory.CreateDirectory(DestinationDirectory);
// Copy all files from the source directory. We can't use MoveTo due to different disc location.
foreach (FileInfo File in DirInfo.GetFiles())
{
string DestinationFilepath = Path.Combine(DestinationDirectory, File.Name);
File.CopyTo(DestinationFilepath, true);
}
DirInfo.Delete(true);
CrashReporterProcessServicer.WriteEvent(string.Format("Moved to {0}", DestinationDirectory));
UpdateProcessedReports();
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex);
}
}
/// <summary>
/// Clear out old MDD logs
/// </summary>
/// <param name="NumDays">Number of days worth of logs to keep</param>
public static void CleanOutOldLogs(int NumDays)
{
DateTime DeleteTime = DateTime.UtcNow - TimeSpan.FromDays(NumDays);
try
{
DirectoryInfo BaseFolder = new DirectoryInfo(CrashReporterProcessServicer.SymbolicatorLogFolder);
if (BaseFolder.Exists)
{
foreach (DirectoryInfo SubFolder in BaseFolder.EnumerateDirectories())
{
if (!SubFolder.EnumerateFiles("*", SearchOption.AllDirectories).Any(x => x.LastWriteTimeUtc > DeleteTime))
{
try
{
SubFolder.Delete(true);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", SubFolder.FullName, Ex.Message));
}
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs: {0}", Ex.Message));
}
}
/// <summary>
/// Clear out old MDD logs
/// </summary>
/// <param name="NumDays">Number of days worth of logs to keep</param>
public static void CleanOutOldLogs(int NumDays)
{
DateTime DeleteTime = DateTime.UtcNow - TimeSpan.FromDays(NumDays);
try
{
DirectoryInfo BaseFolder = new DirectoryInfo(CrashReporterProcessServicer.SymbolicatorLogFolder);
if (BaseFolder.Exists)
{
foreach (DirectoryInfo SubFolder in BaseFolder.EnumerateDirectories())
{
if (!SubFolder.EnumerateFiles("*", SearchOption.AllDirectories).Any(x => x.LastWriteTimeUtc > DeleteTime))
{
try
{
SubFolder.Delete(true);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", SubFolder.FullName, Ex.Message));
}
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs: {0}", Ex.Message));
}
try
{
DirectoryInfo BaseFolder = new DirectoryInfo(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Config.Default.MDDExecutablePath), "..", "..", "Programs", "MinidumpDiagnostics", "Saved", "Logs")));
if (BaseFolder.Exists)
{
foreach (FileInfo File in BaseFolder.EnumerateFiles("*", SearchOption.AllDirectories))
{
try
{
if (File.LastWriteTimeUtc < DeleteTime)
{
File.Delete();
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Unable to delete {0}: {1}", File.FullName, Ex.Message));
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent(String.Format("Symbolicator.CleanOutOldLogs: Failed to delete logs from program folder: {0}", Ex.Message));
}
}
/// <summary>
/// Main processing thread.
/// </summary>
/// <remarks>All exceptions are caught and written to the event log.</remarks>
private void Init()
{
ProcessorTask = new Task(() =>
{
// parse out hte list of blacklisted games.
string[] GameNamesToBlacklist = (Config.Default.GameNamesToBlacklist == null ? "" : Config.Default.GameNamesToBlacklist).ToLowerInvariant().Split(',');
while (!CancelSource.IsCancellationRequested)
{
try
{
bool bIdle = true;
foreach (var Queue in Watcher.ReportQueues)
{
FGenericCrashContext NewContext = null;
if (Queue.TryDequeueReport(out NewContext))
{
bool bIsBlacklistedGame = GameNamesToBlacklist.Contains(NewContext.PrimaryCrashProperties.GameName.ToLowerInvariant());
// if it's blacklisted, skip it, else process it.
if (bIsBlacklistedGame)
{
CrashReporterProcessServicer.WriteEvent(string.Format("Discarding crash from blacklisted GameName '{0}'", NewContext.PrimaryCrashProperties.GameName));
// delete the report from disk since we don't care about it.
FinalizeReport(AddReportResult.Added, new DirectoryInfo(NewContext.CrashDirectory), NewContext);
}
else
{
ProcessReport(NewContext);
}
// The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest
bIdle = false;
break;
}
}
if (bIdle)
{
// Don't use the CPU if we don't need.
Thread.Sleep(1000);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessNewReports: " + Ex, Ex);
}
TickStatic(Watcher);
}
});
}
/// <summary>
/// Delete a report directory.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The directory to delete</param>
public static void CleanReport(DirectoryInfo DirInfo)
{
const int MaxRetries = 3;
bool bWriteException = true;
for (int Retry = 0; Retry < MaxRetries; ++Retry)
{
try
{
if (!DirInfo.Exists)
{
break;
}
foreach (FileInfo Info in DirInfo.GetFiles())
{
Info.IsReadOnly = false;
Info.Attributes = FileAttributes.Normal;
}
DirInfo.Delete(true /* delete contents */);
// Random failures to delete with no exception seen regularly - retry
DirInfo.Refresh();
if (DirInfo.Exists)
{
CrashReporterProcessServicer.WriteEvent("CleanReport: Failed to delete folder without an Exception " + DirInfo);
Thread.Sleep(500);
continue;
}
break;
}
catch (Exception Ex)
{
if (bWriteException)
{
CrashReporterProcessServicer.WriteException("CleanReport: " + Ex, Ex);
bWriteException = false;
}
}
System.Threading.Thread.Sleep(100);
}
DirInfo.Refresh();
if (DirInfo.Exists)
{
CrashReporterProcessServicer.WriteEvent(string.Format("CleanReport: Failed to delete folder {0} after {1} retries", DirInfo, MaxRetries));
}
}
void ProcessDumpFile(string DiagnosticsPath, FGenericCrashContext NewContext)
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DiagnosticsPath);
if (CrashReporterProcessServicer.Symbolicator.Run(DiagnosticsPath, NewContext, ProcessorIndex))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DiagnosticsPath);
ReadDiagnosticsFile(NewContext);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DiagnosticsPath);
}
}
/// <summary>
/// A thread to watch for new crash reports landing.
/// </summary>
/// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks>
void Start()
{
CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:");
var Settings = Properties.Settings.Default;
if (!string.IsNullOrEmpty(Settings.InternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.InternalLandingZone))
{
ReportQueues.Add(new ReportQueue(Settings.InternalLandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority)", Settings.InternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority) is not accessible", Settings.InternalLandingZone));
}
}
#if !DEBUG
if (!string.IsNullOrEmpty(Settings.ExternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.ExternalLandingZone))
{
ReportQueues.Add(new ReportQueue(Settings.ExternalLandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0}", Settings.ExternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} is not accessible", Settings.ExternalLandingZone));
}
}
#endif //!DEBUG
var Cancel = CancelSource.Token;
WatcherTask = Task.Factory.StartNew(async() =>
{
while (!Cancel.IsCancellationRequested)
{
// Check the landing zones for new reports
foreach (var Queue in ReportQueues)
{
Queue.CheckForNewReports();
}
await Task.Delay(60000, Cancel);
}
});
}
/// <summary> Tries to dequeue a report from the list. </summary>
public bool TryDequeueReport(out FGenericCrashContext Context)
{
lock (NewReportsLock)
{
if (NewCrashContexts.Count > 0)
{
Context = NewCrashContexts.Dequeue();
DequeueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(QueueProcessingStartedEventName);
CrashReporterProcessServicer.WriteEvent(string.Format("- Dequeued: {0:N1}/minute BuiltFromCL={1,7} Path={2}", DequeueCounter.EventsPerSecond * 60, Context.PrimaryCrashProperties.EngineVersion, Context.CrashDirectory));
return(true);
}
}
Context = null;
return(false);
}
/// <summary>
/// Shutdown: stop the thread
/// </summary>
public void Dispose()
{
// Cancel the task and wait for it to quit
CancelSource.Cancel();
CrashReporterProcessServicer.WriteEvent("Shutdown: Stopping ReportWatcher thread...");
WatcherTask.Wait();
WatcherTask.Dispose();
CrashReporterProcessServicer.WriteEvent("Shutdown: ReportWatcher thread stopped.");
CrashReporterProcessServicer.WriteEvent("Shutdown: Disposing ReportQueues...");
foreach (var Queue in ReportQueues)
{
Queue.Dispose();
}
CrashReporterProcessServicer.WriteEvent("Shutdown: ReportQueues disposed.");
CancelSource.Dispose();
}
/// <summary>
/// A function to process a newly landed crash report.
/// </summary>
/// <param name="NewContext">An instance of the generic crash context</param>
private void ProcessReport(FGenericCrashContext NewContext)
{
Stopwatch ProcessReportSW = Stopwatch.StartNew();
// Just to verify that the report is still there.
DirectoryInfo DirInfo = new DirectoryInfo(NewContext.CrashDirectory);
if (!DirInfo.Exists)
{
// Something very odd happened.
CrashReporterProcessServicer.WriteEvent("ProcessReport: Directory not found: " + NewContext.CrashDirectory);
return;
}
double WaitTime = ReadProcessAddReport(DirInfo, NewContext);
// Make sure any log messages have been written to disk
CrashReporterProcessServicer.WriteEvent(string.Format("ProcessReportTime={0} WaitTime={1}", ProcessReportSW.Elapsed.TotalSeconds.ToString("0.0"), WaitTime.ToString("0.00")));
}
private static Config LoadConfigPrivate()
{
Config LoadedConfig = new Config();
string ExePath = Assembly.GetEntryAssembly().Location;
string ExeFolder = Path.GetDirectoryName(ExePath);
string ConfigPath = Path.Combine(ExeFolder, "CrashReportProcess.config");
if (File.Exists(ConfigPath))
{
if (!string.IsNullOrEmpty(ConfigPath))
{
using (XmlReader Reader = XmlReader.Create(ConfigPath))
{
CrashReporterProcessServicer.WriteEvent("Loading config from " + ConfigPath);
XmlSerializer Xml = new XmlSerializer(typeof(Config));
LoadedConfig = Xml.Deserialize(Reader) as Config;
}
}
}
#if DEBUG
// Debug mode redirects to local folder in DebugTestingFolder
LoadedConfig.ProcessedReports = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedReports");
LoadedConfig.ProcessedVideos = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedVideos");
LoadedConfig.DepotRoot = Path.Combine(LoadedConfig.DebugTestingFolder, "DepotRoot");
LoadedConfig.InternalLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "InternalLandingZone");
LoadedConfig.ExternalLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "ExternalLandingZone");
LoadedConfig.DataRouterLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "DataRouterLandingZone");
LoadedConfig.InvalidReportsDirectory = Path.Combine(LoadedConfig.DebugTestingFolder, "InvalidReportsDirectory");
LoadedConfig.VersionString += " debugbuild";
LoadedConfig.AWSCredentialsFilepath = Path.Combine(LoadedConfig.DebugTestingFolder, "AWS", "credentials.ini");
LoadedConfig.ProcessedReportsIndexPath = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedReports.ini");
#if SLACKTESTING
LoadedConfig.SlackUsername = "******";
//LoadedConfig.SlackChannel = "OPTIONALTESTINGCHANNELHERE";
#else
LoadedConfig.SlackWebhookUrl = string.Empty; // no Slack in dbeug
#endif
#endif
return(LoadedConfig);
}
/// <summary> Tries to dequeue a report from the list. </summary>
public bool TryDequeueReport(out FGenericCrashContext Context)
{
lock ( NewReportsLock )
{
int LastIndex = NewCrashContexts.Count - 1;
if (LastIndex >= 0)
{
Context = NewCrashContexts[LastIndex];
NewCrashContexts.RemoveAt(LastIndex);
CrashReporterProcessServicer.WriteEvent("- Dequeued: BuiltFromCL=" + string.Format("{0,7}", Context.PrimaryCrashProperties.EngineVersion) + " Path=" + Context.CrashDirectory);
return(true);
}
else
{
Context = null;
return(false);
}
}
}
/// <summary>
/// Moves specified report to the directory where are stored invalid reports.
/// Thread-safe.
/// </summary>
public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename)
{
try
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingFailedEvent);
DirectoryInfo DirInfo = new DirectoryInfo(ReportName);
if (Config.Default.InvalidReportsToAWS)
{
string LeafReportName = Path.GetFileName(ReportName);
DateTime S3KeyTime = DateTime.UtcNow;
// Copy all files from the source directory. We can't use MoveTo due to different disc location.
int FilesMoved = 0;
foreach (FileInfo InvalidFile in DirInfo.GetFiles())
{
try
{
string S3IDPrefix = string.Format("/{0}_{1}_{2}/{3}/{4}/", S3KeyTime.Year, S3KeyTime.Month, S3KeyTime.Day, S3KeyTime.Hour, LeafReportName);
UploadFileToS3(InvalidFile, Config.Default.AWSS3InvalidKeyPrefix + S3IDPrefix + InvalidFile.Name, false);
FilesMoved++;
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent("MoveReportToInvalid: Failed to write report file " + LeafReportName + ": " + InvalidFile.Name);
CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex);
}
}
CrashReporterProcessServicer.WriteEvent(string.Format("MoveReportToInvalid moved {0} file(s) from {1} to S3", FilesMoved, LeafReportName));
}
DirInfo.Delete(true);
UpdateProcessedReports();
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex);
}
}
public virtual void Dispose()
{
// Attempt to remove the queued crashes from the ReportIndex
lock (NewReportsLock)
{
CrashReporterProcessServicer.WriteEvent(string.Format("{0} shutting down", QueueName));
CrashReporterProcessServicer.WriteEvent(string.Format("{0} dequeuing {1} crashes for next time", QueueName, NewCrashContexts.Count));
while (NewCrashContexts.Count > 0)
{
try
{
string ReportName = Path.GetFileName(NewCrashContexts.Peek().CrashDirectory);
CrashReporterProcessServicer.ReportIndex.TryRemoveReport(ReportName);
}
finally
{
NewCrashContexts.Dequeue();
}
}
}
}
private static Config LoadConfigPrivate()
{
Config LoadedConfig = new Config();
string ExePath = Assembly.GetEntryAssembly().Location;
string ExeFolder = Path.GetDirectoryName(ExePath);
string ConfigPath = Path.Combine(ExeFolder, "CrashReportProcess.config");
if (File.Exists(ConfigPath))
{
if (!string.IsNullOrEmpty(ConfigPath))
{
using (XmlReader Reader = XmlReader.Create(ConfigPath))
{
CrashReporterProcessServicer.WriteEvent("Loading config from " + ConfigPath);
XmlSerializer Xml = new XmlSerializer(typeof(Config));
LoadedConfig = Xml.Deserialize(Reader) as Config;
}
}
}
#if DEBUG
// Debug mode redirects to local folder in DebugTestingFolder
LoadedConfig.ProcessedReports = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedReports");
LoadedConfig.ProcessedVideos = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedVideos");
LoadedConfig.DepotRoot = Path.Combine(LoadedConfig.DebugTestingFolder, "DepotRoot");
if (!string.IsNullOrWhiteSpace(LoadedConfig.InternalLandingZone))
{
LoadedConfig.InternalLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "InternalLandingZone");
}
if (!string.IsNullOrWhiteSpace(LoadedConfig.ExternalLandingZone))
{
LoadedConfig.ExternalLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "ExternalLandingZone");
}
if (!string.IsNullOrWhiteSpace(LoadedConfig.DataRouterLandingZone))
{
LoadedConfig.DataRouterLandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "DataRouterLandingZone");
}
if (!string.IsNullOrWhiteSpace(LoadedConfig.PS4LandingZone))
{
LoadedConfig.PS4LandingZone = Path.Combine(LoadedConfig.DebugTestingFolder, "PS4LandingZone");
}
if (!string.IsNullOrWhiteSpace(LoadedConfig.MDDExecutablePath))
{
LoadedConfig.MDDExecutablePath = Path.Combine(LoadedConfig.DebugTestingFolder, "MinidumpDiagnostics", "Engine", "Binaries", "Win64", "MinidumpDiagnostics.exe");
}
LoadedConfig.VersionString += " debugbuild";
LoadedConfig.AWSCredentialsFilepath = Path.Combine(LoadedConfig.DebugTestingFolder, "AWS", "credentials.ini");
if (!string.IsNullOrWhiteSpace(LoadedConfig.ProcessedReportsIndexPath))
{
LoadedConfig.ProcessedReportsIndexPath = Path.Combine(LoadedConfig.DebugTestingFolder, "ProcessedReports.ini");
}
LoadedConfig.CrashReportWebSite = string.Empty;
LoadedConfig.AWSS3OutputKeyPrefix = LoadedConfig.AWSS3OutputKeyPrefix.Replace("prod", "test");
LoadedConfig.AWSS3InvalidKeyPrefix = LoadedConfig.AWSS3InvalidKeyPrefix.Replace("prod", "test");
LoadedConfig.MinDesiredMemoryQueueSize = 5;
LoadedConfig.MaxMemoryQueueSize = 15;
#if SLACKTESTING
LoadedConfig.SlackUsername = "******";
//LoadedConfig.SlackChannel = "OPTIONALTESTINGCHANNELHERE";
#else
LoadedConfig.SlackWebhookUrl = string.Empty; // no Slack in dbeug
#endif
#endif
return(LoadedConfig);
}
/// <summary> Enqueues a new crash. </summary>
void EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return;
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bFromWER = false;
if (Context == null || !Context.HasProcessedData())
{
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
FReportData ReportData = new FReportData(MetaData, NewReportPath);
Context = ConvertMetadataToCrashContext(MetaData, NewReportPath);
bFromWER = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
}
else
{
if (GenericContext != null && GenericContext.PrimaryCrashProperties.ErrorMessage.Length > 0)
{
// Get error message from the crash context and fix value in the metadata.
Context.PrimaryCrashProperties.ErrorMessage = GenericContext.PrimaryCrashProperties.ErrorMessage;
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// If based on WER, save to the file.
if (bFromWER && GenericContext == null)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteFailure("% NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.MoveReportToInvalid(NewReportPath, Context.GetAsFilename());
return;
}
if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteFailure("% UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.CleanReport(NewReportPath);
return;
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteFailure("! BROKEN0 : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
ReportProcessor.CleanReport(NewReportPath);
return;
}
lock (NewReportsLock)
{
NewCrashContexts.Add(Context);
}
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
}
}
/// <summary>
/// Builds MDD command line args, waits for a MDD task slot, runs MDD and blocks for the result. Writes diag text into DiagnosticsPath folder if successful.
/// </summary>
/// <param name="DiagnosticsPath">Path of the minidump file</param>
/// <param name="Context">The crash context</param>
/// <param name="ProcessorIndex">Processor thread index for logging purposes</param>
/// <returns>True, if successful</returns>
public bool Run(string DiagnosticsPath, FGenericCrashContext Context, int ProcessorIndex)
{
if (!File.Exists(Config.Default.MDDExecutablePath))
{
CrashReporterProcessServicer.WriteEvent("Symbolicator.Run() file not found " + Config.Default.MDDExecutablePath);
return(false);
}
// Use MinidumpDiagnostics from MDDExecutablePath.
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.MDDExecutablePath, Config.Default.DiskSpaceAlertPercent);
// Don't purge logs
// TODO: make this clear to logs once a day or something (without letting MDD check on every run!)
string PurgeLogsDays = "-1";
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
// Pass Windows variants (Win32/64) to MinidumpDiagnostics
string PlatformVariant = Context.PrimaryCrashProperties.PlatformName;
if (PlatformVariant != null && Context.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS"))
{
if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win32") ||
Context.PrimaryCrashProperties.PlatformFullName.Contains("32b"))
{
PlatformVariant = "Win32";
}
else if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win64") ||
Context.PrimaryCrashProperties.PlatformFullName.Contains("64b"))
{
PlatformVariant = "Win64";
}
}
// Build the absolute log file path for MinidumpDiagnostics
string BaseFolder = CrashReporterProcessServicer.SymbolicatorLogFolder;
DateTime WriteTime = DateTime.UtcNow;
string DateFolder = WriteTime.ToString("yyyy_MM_dd");
string HourFolder = WriteTime.ToString("HH");
string Folder = Path.Combine(BaseFolder, DateFolder, HourFolder);
string AbsLogPath = Path.Combine(Folder, Context.GetAsFilename() + ".log");
Directory.CreateDirectory(Folder);
List <string> MinidumpDiagnosticsParams = new List <string>
(
new string[]
{
"\"" + DiagnosticsPath + "\"",
"-BranchName=" + EngineVersion.Branch, // Backward compatibility
"-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility
"-GameName=" + Context.PrimaryCrashProperties.GameName,
"-EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion,
"-BuildVersion=" + (string.IsNullOrWhiteSpace(Context.PrimaryCrashProperties.BuildVersion) ?
string.Format("{0}-CL-{1}", EngineVersion.Branch, EngineVersion.Changelist).Replace('/', '+') :
Context.PrimaryCrashProperties.BuildVersion),
"-PlatformName=" + Context.PrimaryCrashProperties.PlatformName,
"-PlatformVariantName=" + PlatformVariant,
"-bUsePDBCache=true",
"-PDBCacheDepotRoot=" + Config.Default.DepotRoot,
"-PDBCachePath=" + Config.Default.MDDPDBCachePath,
"-PDBCacheSizeGB=" + Config.Default.MDDPDBCacheSizeGB,
"-PDBCacheMinFreeSpaceGB=" + Config.Default.MDDPDBCacheMinFreeSpaceGB,
"-PDBCacheFileDeleteDays=" + Config.Default.MDDPDBCacheFileDeleteDays,
"-MutexPDBCache",
"-PDBCacheLock=CrashReportProcessPDBCacheLock",
"-NoTrimCallstack",
"-SyncSymbols",
"-NoP4Symbols",
"-ForceUsePDBCache",
"-MutexSourceSync",
"-SourceSyncLock=CrashReportProcessSourceSyncLock",
"-SyncMicrosoftSymbols",
"-unattended",
"-AbsLog=" + AbsLogPath,
"-DepotIndex=" + Config.Default.DepotIndex,
"-P4User="******"-P4Client=" + Config.Default.P4Client,
"-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays + ",[LogFiles]:MaxLogFilesOnDisk=-1",
"-LOGTIMESINCESTART"
}
);
LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null;
if (Environment.UserInteractive)
{
CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD;
}
else
{
MinidumpDiagnosticsParams.AddRange
(
new[]
{
"-buildmachine"
}
);
// Write some debugging message.
CrashReporterProcessServicer.WriteMDD("MinidumpDiagnostics Params: " + String.Join(" ", MinidumpDiagnosticsParams));
}
Task <bool> NewSymbolicatorTask = Task.FromResult(false);
Stopwatch WaitSW = Stopwatch.StartNew();
double WaitForLockTime = 0.0;
lock (Tasks)
{
int TaskIdx = Task.WaitAny(Tasks);
Tasks[TaskIdx] = NewSymbolicatorTask = Task <bool> .Factory.StartNew(() =>
{
LaunchProcess ReportParser = new LaunchProcess(Config.Default.MDDExecutablePath, Path.GetDirectoryName(Config.Default.MDDExecutablePath), CaptureMessageDelegate,
MinidumpDiagnosticsParams.ToArray());
return(ReportParser.WaitForExit(Config.Default.MDDTimeoutMinutes * 1000 * 60) == EWaitResult.Ok);
});
WaitForLockTime = WaitSW.Elapsed.TotalSeconds;
}
NewSymbolicatorTask.Wait();
double TotalMDDTime = WaitSW.Elapsed.TotalSeconds;
double MDDRunTime = TotalMDDTime - WaitForLockTime;
CrashReporterProcessServicer.StatusReporter.AddToMeanCounter(StatusReportingPerfMeanNames.MinidumpDiagnostics, (int)(MDDRunTime * 1000));
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Symbolicator.Run: Thread blocked for {0:N1}s then MDD ran for {1:N1}s", WaitForLockTime, MDDRunTime));
return(NewSymbolicatorTask.Result);
}
/// <summary>
/// Look for new report folders and add them to the publicly available thread-safe queue.
/// </summary>
public int CheckForNewReports()
{
try
{
if (QueueCount >= Config.Default.MinDesiredMemoryQueueSize)
{
CrashReporterProcessServicer.WriteEvent(string.Format(
"CheckForNewReports: {0} skipped busy queue size {1} in {2}", QueueName, QueueCount, LandingZone));
}
else
{
var NewReportPath = "";
var ReportsInLandingZone = new ReportIdSet();
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(LandingZone, Config.Default.DiskSpaceAlertPercent);
DirectoryInfo[] DirectoriesInLandingZone;
if (GetCrashesFromLandingZone(out DirectoriesInLandingZone))
{
LastQueueSizeOnDisk = DirectoriesInLandingZone.Length;
int EnqueuedCount = 0;
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} reports in disk landing zone {1}",
DirectoriesInLandingZone.Length, LandingZone));
// Add any new reports
for (int DirIndex = 0; DirIndex < DirectoriesInLandingZone.Length && QueueCount < Config.Default.MaxMemoryQueueSize; DirIndex++)
{
var SubDirInfo = DirectoriesInLandingZone[DirIndex];
try
{
if (Directory.Exists(SubDirInfo.FullName))
{
NewReportPath = SubDirInfo.FullName;
ReportsInLandingZone.Add(NewReportPath);
if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath))
{
if (EnqueueNewReport(NewReportPath))
{
EnqueuedCount++;
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex, Ex);
ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks);
}
}
ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone;
CrashReporterProcessServicer.WriteEvent(string.Format(
"CheckForNewReports: {0} enqueued to queue size {1} from {2}", EnqueuedCount, QueueCount, LandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: Enqueue rate {0:N1}/minute from {1}",
EnqueueCounter.EventsPerSecond * 60, LandingZone));
}
else
{
LastQueueSizeOnDisk = 0;
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex, Ex);
}
return(GetTotalWaitingCount());
}
// From CrashUpload.cpp
/*
* struct FCompressedCrashFile : FNoncopyable
* {
* int32 CurrentFileIndex; // 4 bytes for file index
* FString Filename; // 4 bytes for length + 260 bytes for char data
* TArray<uint8> Filedata; // 4 bytes for length + N bytes for data
* }
*/
/// <summary> Enqueues a new crash. </summary>
private bool EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return(false);
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bContextDirty = false;
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
if (Context == null)
{
// Missing crash context
FReportData ReportData = new FReportData(MetaData, NewReportPath);
ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context);
bContextDirty = true;
}
else if (!Context.HasProcessedData())
{
// Missing data - try to get from WER metadata
FReportData ReportData = new FReportData(MetaData, NewReportPath);
GetErrorMessageFromMetadata(ReportData, Context);
bContextDirty = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// Added data from WER, save to the crash context file.
if (bContextDirty)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
Context.ToFile();
}
else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
Context.ToFile();
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
Context.ToFile();
}
// Check static reports index for duplicated reports
// This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router)
// This can be safely discontinued when all crashes come from the same SQS
// This DOES NOT scale to multiple CRP instances
// ReportIndex can be disabled by leaving the path empty in config
if (!CrashReporterProcessServicer.ReportIndex.TryAddNewReport(ReportName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
if (ShouldDecimateNextReport())
{
CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
lock (NewReportsLock)
{
NewCrashContexts.Enqueue(Context);
}
EnqueueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent);
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
return(true);
}
/// <summary>
/// Create thread to watch for new crash reports landing.
/// </summary>
/// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks>
void Init()
{
CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:");
var Settings = Config.Default;
if (!string.IsNullOrEmpty(Settings.DataRouterLandingZone))
{
if (System.IO.Directory.Exists(Settings.DataRouterLandingZone))
{
ReportQueues.Add(new DataRouterReportQueue("DataRouter Crashes", Settings.DataRouterLandingZone, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (all crashes from data router)", Settings.DataRouterLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (all crashes from data router) is not accessible", Settings.DataRouterLandingZone));
}
}
if (!string.IsNullOrEmpty(Settings.InternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.InternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone, StatusReportingEventNames.ProcessingStartedReceiverEvent, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority (legacy))", Settings.InternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority (legacy)) is not accessible", Settings.InternalLandingZone));
}
}
if (!string.IsNullOrEmpty(Settings.PS4LandingZone))
{
if (System.IO.Directory.Exists(Settings.PS4LandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("PS4 Crashes", Settings.PS4LandingZone, StatusReportingEventNames.ProcessingStartedPS4Event, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, PS4 non-retail hardware)", Settings.PS4LandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, PS4 non-retail hardware) is not accessible", Settings.PS4LandingZone));
}
}
#if !DEBUG
if (!string.IsNullOrEmpty(Settings.ExternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.ExternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone, StatusReportingEventNames.ProcessingStartedReceiverEvent, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (legacy)", Settings.ExternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (legacy) is not accessible", Settings.ExternalLandingZone));
}
}
#endif //!DEBUG
// Init queue entries in StatusReporter
foreach (var Queue in ReportQueues)
{
CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath);
}
var Cancel = CancelSource.Token;
WatcherTask = new Task(async() =>
{
DateTime LastQueueSizeReport = DateTime.MinValue;
while (!Cancel.IsCancellationRequested)
{
// Check the landing zones for new reports
DateTime StartTime = DateTime.Now;
foreach (var Queue in ReportQueues)
{
int QueueSize = Queue.CheckForNewReports();
CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize);
}
TimeSpan TimeTaken = DateTime.Now - StartTime;
CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds));
await Task.Delay(30000, Cancel);
}
});
}
/// <summary>
/// Look for new report folders and add them to the publicly available thread-safe queue.
/// </summary>
public override int CheckForNewReports()
{
try
{
if (QueueCount >= SkipQueueRefreshMin)
{
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: skipped busy queue size {0} in {1}", QueueCount, LandingZone));
return(LastQueueSizeOnDisk);
}
var NewReportPath = "";
var ReportsInLandingZone = new ReportIdSet();
// Check the landing zones for new reports
DirectoryInfo DirInfo = new DirectoryInfo(LandingZone);
// Couldn't find a landing zone, skip and try again later.
// Crash receiver will recreate them if needed.
if (!DirInfo.Exists)
{
CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone);
return(LastQueueSizeOnDisk);
}
var DirectoriesInLandingZone = DirInfo.GetDirectories().OrderBy(dirinfo => dirinfo.CreationTimeUtc).ToArray();
LastQueueSizeOnDisk = DirectoriesInLandingZone.Length;
int EnqueuedCount = 0;
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} reports in {1}", DirectoriesInLandingZone.Length, LandingZone));
// Add any new reports
for (int DirIndex = 0; DirIndex < DirectoriesInLandingZone.Length && QueueCount < QueueMax; DirIndex++)
{
var SubDirInfo = DirectoriesInLandingZone[DirIndex];
try
{
if (Directory.Exists(SubDirInfo.FullName))
{
NewReportPath = SubDirInfo.FullName;
ReportsInLandingZone.Add(NewReportPath);
if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath))
{
EnqueueNewReport(NewReportPath);
EnqueuedCount++;
}
}
}
catch (System.Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex.ToString());
ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks);
}
}
//CrashReporterProcessServicer.WriteEvent( string.Format( "ReportsInLandingZone={0}, ReportsInLandingZoneLastTimeWeChecked={1}", ReportsInLandingZone.Count, ReportsInLandingZoneLastTimeWeChecked.Count ) );
ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone;
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} enqueued to queue size {1} from {2}", EnqueuedCount, QueueCount, LandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: Enqueue rate {0:N1}/minute from {1}", EnqueueCounter.EventsPerSecond * 60, LandingZone));
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex.ToString());
}
return(LastQueueSizeOnDisk);
}
/// <summary> Enqueues a new crash. </summary>
private void EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return;
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bFromWER = false;
if (Context == null || !Context.HasProcessedData())
{
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
FReportData ReportData = new FReportData(MetaData, NewReportPath);
ConvertMetadataToCrashContext(MetaData, NewReportPath, ref Context);
bFromWER = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
}
else
{
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// Added data from WER, save to the crash context file.
if (bFromWER)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
}
else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
}
lock (NewReportsLock)
{
NewCrashContexts.Enqueue(Context);
}
EnqueueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingConstants.QueuedEvent);
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
}
}
private static bool DecompressDataRouterContent(byte[] CompressedBufferArray, string InLandingZone)
{
// Decompress to landing zone
byte[] UncompressedBufferArray = new byte[Config.Default.MaxUncompressedS3RecordSize];
int UncompressedSize = NativeMethods.UncompressMemoryZlib(UncompressedBufferArray, CompressedBufferArray);
if (UncompressedSize < 0)
{
string FailString = "! DecompressDataRouterContent() failed in UncompressMemoryZlib() with " +
NativeMethods.GetZlibError(UncompressedSize);
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent);
return(false);
}
using (BinaryReader BinaryData = new BinaryReader(new MemoryStream(UncompressedBufferArray, 0, UncompressedSize, false)))
{
char[] MarkerChars = BinaryData.ReadChars(3);
if (MarkerChars[0] == 'C' && MarkerChars[1] == 'R' && MarkerChars[2] == '1')
{
CrashHeader CrashHeader = DataRouterReportQueue.CrashHeader.ParseCrashHeader(BinaryData);
// Create safe directory name and then create the directory on disk
string CrashFolderName = GetSafeFilename(CrashHeader.DirectoryName);
string CrashFolderPath = Path.Combine(InLandingZone, CrashFolderName);
// Early check for duplicate processed report
lock (ReportIndexLock)
{
if (CrashReporterProcessServicer.ReportIndex.ContainsReport(CrashFolderName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"DataRouterReportQueue: Duplicate report skipped early {0} in a DataRouterReportQueue", CrashFolderPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
return(false); // this isn't an error so don't set error message
}
}
// Create target folder
int TryIndex = 1;
while (Directory.Exists(CrashFolderPath))
{
CrashFolderPath = Path.Combine(InLandingZone, string.Format("{0}_DUPE{1:D3}", CrashFolderName, TryIndex++));
}
Directory.CreateDirectory(CrashFolderPath);
if (UncompressedSize != CrashHeader.UncompressedSize)
{
CrashReporterProcessServicer.WriteEvent(
string.Format(
"DecompressDataRouterContent() warning UncompressedSize mismatch (embedded={0}, actual={1}) Path={2}",
CrashHeader.UncompressedSize, UncompressedSize, CrashFolderPath));
}
if (!ParseCrashFiles(BinaryData, CrashHeader.FileCount, CrashFolderPath))
{
string FailString = "! DecompressDataRouterContent() failed to write files Path=" + CrashFolderPath;
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent);
return(false);
}
}
else
{
#if ALLOWOLDCLIENTDATA
// Early Data Router upload format was broken.
// Should be [CR1][CrashHeader][File][File][File][File]...
// Actually [Undefined CrashHeader][File][File][File][File]...[CrashHeader]
// Seek to end minus header size
BinaryData.BaseStream.Position = UncompressedSize - DataRouterReportQueue.CrashHeader.FixedSize;
var CrashHeader = DataRouterReportQueue.CrashHeader.ParseCrashHeader(BinaryData);
// Create safe directory name and then create the directory on disk
string CrashFolderName = GetSafeFilename(CrashHeader.DirectoryName);
string CrashFolderPath = Path.Combine(InLandingZone, CrashFolderName);
// Early check for duplicate processed report
lock (ReportIndexLock)
{
if (CrashReporterProcessServicer.ReportIndex.ContainsReport(CrashFolderName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"DataRouterReportQueue: Duplicate report skipped early {0} in a DataRouterReportQueue", CrashFolderPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
return(false); // this isn't an error so don't set error message
}
}
// Create target folder
int TryIndex = 1;
while (Directory.Exists(CrashFolderPath))
{
CrashFolderPath = Path.Combine(InLandingZone, string.Format("{0}_DUPE{1:D3}", CrashFolderName, TryIndex++));
}
Directory.CreateDirectory(CrashFolderPath);
if (UncompressedSize != CrashHeader.UncompressedSize + CrashHeader.FixedSize)
{
CrashReporterProcessServicer.WriteEvent(
string.Format(
"DecompressDataRouterContent() warning UncompressedSize mismatch (embedded={0}, actual={1}) Path={2}",
CrashHeader.UncompressedSize, UncompressedSize, CrashFolderPath));
}
// Seek to start of files (header size in from start)
BinaryData.BaseStream.Position = CrashHeader.FixedSize;
if (!ParseCrashFiles(BinaryData, CrashHeader.FileCount, CrashFolderPath))
{
string FailString = "! DecompressDataRouterContent() failed to write files Path=" + CrashFolderPath;
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent);
return(false);
}
#else
ErrorMessage = "! DecompressDataRouterContent() failed to read invalid data format. Corrupt or old format data received Path=" + CrashFolderPath;
return(false);
#endif
}
}
return(true);
}
/// <summary>
/// A function to add a report to the database, and rename the relevant files.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="LogFileName">The file name of the log file in the report.</param>
/// <param name="DumpFileName">The file name of the minidump in the report.</param>
/// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param>
private bool AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName)
{
try
{
Stopwatch AddReportTime = Stopwatch.StartNew();
// Create an XML representation of a crash
string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null);
if (CrashDetails == "")
{
CrashReporterProcessServicer.WriteFailure("! NoDetail: Path=" + NewContext.CrashDirectory);
return(false);
}
// Upload the crash to the database, and retrieve the new row id
int ReportID = UploadCrash(CrashDetails);
if (ReportID <= 0)
{
CrashReporterProcessServicer.WriteFailure("! NoUpload: Path=" + NewContext.CrashDirectory);
return(false);
}
string IDThenUnderscore = string.Format("{0}_", ReportID);
// Use the row id to name and move the files the way the web site requires
string DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedReports, IDThenUnderscore);
// Move report files to crash reporter file store
if (LogFileName != null)
{
LogFileName = Path.Combine(DirInfo.FullName, LogFileName);
FileInfo LogInfo = new FileInfo(LogFileName);
if (LogInfo.Exists)
{
LogInfo.MoveTo(DestinationFolder + "Launch.log");
}
}
string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName);
FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName);
if (CrashContextInfo.Exists)
{
CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName);
}
// WERMetaDataName = Path.Combine(DirInfo.FullName, WERMetaDataName);
// FileInfo MetaInfo = new FileInfo(WERMetaDataName);
// if (MetaInfo.Exists)
// {
// MetaInfo.MoveTo(DestinationFolder + "WERMeta.xml");
// }
if (DumpFileName != null)
{
DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName);
FileInfo DumpInfo = new FileInfo(DumpFileName);
if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/)
{
DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp");
}
}
// DiagnosticsFileName = Path.Combine(DirInfo.FullName, DiagnosticsFileName);
// FileInfo DiagnosticsInfo = new FileInfo(DiagnosticsFileName);
// if (DiagnosticsInfo.Exists)
// {
// DiagnosticsInfo.MoveTo(DestinationFolder + CrashReporterConstants.DiagnosticsFileName);
// }
// Move the video (if it exists) to an alternate store
if (VideoFileName != null)
{
DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedVideos, IDThenUnderscore);
VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName);
FileInfo VideoInfo = new FileInfo(VideoFileName);
if (VideoInfo.Exists)
{
VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName);
}
}
CrashReporterProcessServicer.WriteEvent("# WebAdded: ReportID =" + string.Format("{0,7}", ReportID) + " Path=" + NewContext.CrashDirectory);
UpdateProcessedReports();
WebAddedReports++;
double Ratio = (double)WebAddedReports / (double)ProcessedReports * 100;
double AddedPerDay = (double)WebAddedReports / Timer.Elapsed.TotalDays;
CrashReporterProcessServicer.WriteEvent(string.Format("Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4}", (int)Ratio, ProcessedReports, WebAddedReports, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay));
return(true);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("AddReport: " + DirInfo.Name + "\n\n" + Ex.ToString());
}
return(false);
}