private void OnDispose()
{
bDisposing = true;
// Clean up the directory watcher and crash processor threads
WriteEvent("Shutdown: Stopping ReportProcessors...");
foreach (var Processor in Processors)
{
Processor.RequestStop();
}
WriteEvent("Shutdown: Disposing ReportProcessors...");
foreach (var Processor in Processors)
{
Processor.Dispose();
}
Processors.Clear();
WriteEvent("Shutdown: ReportProcessors stopped and disposed.");
WriteEvent("Shutdown: Disposing ReportWatcher...");
Watcher.Dispose();
Watcher = null;
WriteEvent("Shutdown: ReportWatcher disposed.");
WriteEvent("Shutdown: Writing ReportIndex.");
ReportIndex.WriteToFile();
ReportIndex = null;
WriteEvent("Shutdown: ReportIndex written.");
WriteEvent("Shutdown: Disposing AmazonClients...");
OutputAWS.Dispose();
OutputAWS = null;
DataRouterAWS.Dispose();
DataRouterAWS = null;
WriteEvent("Shutdown: AmazonClients disposed.");
WriteEvent("Shutdown: Disposing StatusReporter...");
StatusReporter.Dispose();
StatusReporter = null;
WriteEvent("Shutdown: StatusReporter disposed.");
WriteEvent("Shutdown: Disposing SlackWriter...");
Slack.Dispose();
Slack = null;
WriteEvent("Shutdown: SlackWriter disposed.");
// Flush the log to disk
WriteEvent("Shutdown: Disposing LogWriter.");
Log.Dispose();
Log = null;
bDisposed = true;
}
/// <summary>
/// Shutdown: stop the thread
/// </summary>
public void Dispose()
{
// Cancel the task and wait for it to quit
CancelSource.Cancel();
WatcherTask.Wait();
WatcherTask.Dispose();
foreach (var Queue in ReportQueues)
{
Queue.Dispose();
}
ReportIndex.WriteToFile();
CancelSource.Dispose();
}
private void OnDispose()
{
bDisposing = true;
// Clean up the directory watcher and crash processor threads
foreach (var Processor in Processors)
{
Processor.RequestStop();
}
foreach (var Processor in Processors)
{
Processor.Dispose();
}
Processors.Clear();
Watcher.Dispose();
Watcher = null;
ReportIndex.WriteToFile();
ReportIndex = null;
OutputAWS.Dispose();
OutputAWS = null;
DataRouterAWS.Dispose();
DataRouterAWS = null;
StatusReporter.Dispose();
StatusReporter = null;
Slack.Dispose();
Slack = null;
// Flush the log to disk
Log.Dispose();
Log = null;
bDisposed = true;
}
public virtual void Dispose()
{
// Attempt to remove the queued crashes from the ReportIndex
lock (NewReportsLock)
{
CrashReporterProcessServicer.WriteEvent(string.Format("{0} shutting down", QueueName));
CrashReporterProcessServicer.WriteEvent(string.Format("{0} dequeuing {1} crashes for next time", QueueName, NewCrashContexts.Count));
while (NewCrashContexts.Count > 0)
{
try
{
string ReportName = Path.GetFileName(NewCrashContexts.Peek().CrashDirectory);
ReportIndex.TryRemoveReport(ReportName);
}
finally
{
NewCrashContexts.Dequeue();
}
}
}
}
/// <summary>
/// Start the service, and stop the service if there were any errors found.
/// </summary>
/// <param name="Arguments">Command line arguments (unused).</param>
protected override void OnStart(string[] Arguments)
{
// Create a log file for any start-up messages
Log = new LogWriter("CrashReportProcess", LogFolder);
Config.LoadConfig();
Slack = new SlackWriter
{
WebhookUrl = Config.Default.SlackWebhookUrl,
Channel = Config.Default.SlackChannel,
Username = Config.Default.SlackUsername,
IconEmoji = Config.Default.SlackEmoji
};
Symbolicator = new Symbolicator();
StatusReporter = new StatusReporting();
ReportIndex = new ReportIndex
{
IsEnabled = !string.IsNullOrWhiteSpace(Config.Default.ProcessedReportsIndexPath),
Filepath = Config.Default.ProcessedReportsIndexPath,
Retention = TimeSpan.FromDays(Config.Default.ReportsIndexRetentionDays)
};
ReportIndex.ReadFromFile();
WriteEvent("Initializing AWS");
string AWSError;
AWSCredentials AWSCredentialsForDataRouter = new StoredProfileAWSCredentials(Config.Default.AWSProfileInputName, Config.Default.AWSCredentialsFilepath);
AmazonSQSConfig SqsConfigForDataRouter = new AmazonSQSConfig
{
ServiceURL = Config.Default.AWSSQSServiceInputURL
};
AmazonS3Config S3ConfigForDataRouter = new AmazonS3Config
{
ServiceURL = Config.Default.AWSS3ServiceInputURL
};
DataRouterAWS = new AmazonClient(AWSCredentialsForDataRouter, SqsConfigForDataRouter, S3ConfigForDataRouter, out AWSError);
if (!DataRouterAWS.IsSQSValid || !DataRouterAWS.IsS3Valid)
{
WriteFailure("AWS failed to initialize profile for DataRouter access. Error:" + AWSError);
StatusReporter.Alert("AWSFailInput", "AWS failed to initialize profile for DataRouter access", 0);
}
AWSCredentials AWSCredentialsForOutput = new StoredProfileAWSCredentials(Config.Default.AWSProfileOutputName, Config.Default.AWSCredentialsFilepath);
AmazonS3Config S3ConfigForOutput = new AmazonS3Config
{
ServiceURL = Config.Default.AWSS3ServiceOutputURL
};
OutputAWS = new AmazonClient(AWSCredentialsForOutput, null, S3ConfigForOutput, out AWSError);
if (!OutputAWS.IsS3Valid)
{
WriteFailure("AWS failed to initialize profile for output S3 bucket access. Error:" + AWSError);
StatusReporter.Alert("AWSFailOutput", "AWS failed to initialize profile for output S3 bucket access", 0);
}
// Add directory watchers
WriteEvent("Creating ReportWatcher");
Watcher = new ReportWatcher();
WriteEvent("Creating ReportProcessors");
for (int ProcessorIndex = 0; ProcessorIndex < Config.Default.ProcessorThreadCount; ProcessorIndex++)
{
var Processor = new ReportProcessor(Watcher, ProcessorIndex);
Processors.Add(Processor);
}
// Init events by enumerating event names
WriteEvent("Initializing Event Counters");
FieldInfo[] EventNameFields = typeof(StatusReportingEventNames).GetFields(BindingFlags.Static | BindingFlags.Public);
StatusReporter.InitCounters(EventNameFields.Select(EventNameField => (string)EventNameField.GetValue(null)));
WriteEvent("Initializing Performance Mean Counters");
FieldInfo[] MeanNameFields = typeof(StatusReportingPerfMeanNames).GetFields(BindingFlags.Static | BindingFlags.Public);
StatusReporter.InitMeanCounters(MeanNameFields.Select(MeanNameField => (string)MeanNameField.GetValue(null)));
WriteEvent("Initializing Folder Monitors");
Dictionary <string, string> FoldersToMonitor = new Dictionary <string, string>();
FoldersToMonitor.Add(Config.Default.ProcessedReports, "Processed Reports");
FoldersToMonitor.Add(Config.Default.ProcessedVideos, "Processed Videos");
FoldersToMonitor.Add(Config.Default.DepotRoot, "P4 Workspace");
FoldersToMonitor.Add(Config.Default.InternalLandingZone, "CRR Landing Zone");
FoldersToMonitor.Add(Config.Default.DataRouterLandingZone, "Data Router Landing Zone");
FoldersToMonitor.Add(Config.Default.PS4LandingZone, "PS4 Landing Zone");
FoldersToMonitor.Add(Assembly.GetExecutingAssembly().Location, "CRP Binaries and Logs");
FoldersToMonitor.Add(Config.Default.MDDPDBCachePath, "MDD PDB Cache");
StatusReporter.InitFolderMonitors(FoldersToMonitor);
WriteEvent("Starting StatusReporter");
StatusReporter.Start();
// Start the threads now
Watcher.Start();
foreach (var Processor in Processors)
{
Processor.Start();
}
DateTime StartupTime = DateTime.UtcNow;
WriteEvent("Successfully started at " + StartupTime);
}
/// <summary>
/// Create thread to watch for new crash reports landing.
/// </summary>
/// <remarks>The NFS storage does not support file system watchers, so this has to be done laboriously.</remarks>
void Init()
{
CrashReporterProcessServicer.WriteEvent("CrashReportProcessor watching directories:");
var Settings = Config.Default;
if (!string.IsNullOrEmpty(Settings.DataRouterLandingZone))
{
if (System.IO.Directory.Exists(Settings.DataRouterLandingZone))
{
ReportQueues.Add(new DataRouterReportQueue("DataRouter Crashes", Settings.DataRouterLandingZone, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (all crashes from data router)", Settings.DataRouterLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (all crashes from data router) is not accessible", Settings.DataRouterLandingZone));
}
}
if (!string.IsNullOrEmpty(Settings.InternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.InternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("Epic Crashes", Settings.InternalLandingZone, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (internal, high priority (legacy))", Settings.InternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (internal, high priority (legacy)) is not accessible", Settings.InternalLandingZone));
}
}
#if !DEBUG
if (!string.IsNullOrEmpty(Settings.ExternalLandingZone))
{
if (System.IO.Directory.Exists(Settings.ExternalLandingZone))
{
ReportQueues.Add(new ReceiverReportQueue("External Crashes", Settings.ExternalLandingZone, Config.Default.QueueLowerLimitForDiscard, Config.Default.QueueUpperLimitForDiscard));
CrashReporterProcessServicer.WriteEvent(string.Format("\t{0} (legacy)", Settings.ExternalLandingZone));
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("\t{0} (legacy) is not accessible", Settings.ExternalLandingZone));
}
}
#endif //!DEBUG
// Init queue entries in StatusReporter
foreach (var Queue in ReportQueues)
{
CrashReporterProcessServicer.StatusReporter.InitQueue(Queue.QueueId, Queue.LandingZonePath);
}
ReportIndex.Filepath = Config.Default.ProcessedReportsIndexPath;
ReportIndex.Retention = TimeSpan.FromDays(Config.Default.ReportsIndexRetentionDays);
ReportIndex.ReadFromFile();
var Cancel = CancelSource.Token;
WatcherTask = new Task(async() =>
{
DateTime LastQueueSizeReport = DateTime.MinValue;
while (!Cancel.IsCancellationRequested)
{
// Check the landing zones for new reports
DateTime StartTime = DateTime.Now;
foreach (var Queue in ReportQueues)
{
int QueueSize = Queue.CheckForNewReports();
CrashReporterProcessServicer.StatusReporter.SetQueueSize(Queue.QueueId, QueueSize);
}
TimeSpan TimeTaken = DateTime.Now - StartTime;
CrashReporterProcessServicer.WriteEvent(string.Format("Checking Landing Zones took {0:F2} seconds", TimeTaken.TotalSeconds));
await Task.Delay(30000, Cancel);
}
});
}
// From CrashUpload.cpp
/*
* struct FCompressedCrashFile : FNoncopyable
* {
* int32 CurrentFileIndex; // 4 bytes for file index
* FString Filename; // 4 bytes for length + 260 bytes for char data
* TArray<uint8> Filedata; // 4 bytes for length + N bytes for data
* }
*/
/// <summary> Enqueues a new crash. </summary>
private bool EnqueueNewReport(string NewReportPath)
{
string ReportName = Path.GetFileName(NewReportPath);
string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
if (bIsCompressed)
{
FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml <FCompressedCrashInformation>(MetadataPath);
bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
if (!bResult)
{
ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
return(false);
}
else
{
// Rename metadata file to not interfere with the WERReportMetadata.
File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
}
}
// Unified crash reporting
FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
FGenericCrashContext Context = GenericContext;
bool bContextDirty = false;
WERReportMetadata MetaData = FindMetadata(NewReportPath);
if (MetaData != null)
{
if (Context == null)
{
// Missing crash context
FReportData ReportData = new FReportData(MetaData, NewReportPath);
ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context);
bContextDirty = true;
}
else if (!Context.HasProcessedData())
{
// Missing data - try to get from WER metadata
FReportData ReportData = new FReportData(MetaData, NewReportPath);
GetErrorMessageFromMetadata(ReportData, Context);
bContextDirty = true;
}
}
if (Context == null)
{
CrashReporterProcessServicer.WriteFailure("! NoCntx : Path=" + NewReportPath);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
Context.CrashDirectory = NewReportPath;
Context.PrimaryCrashProperties.SetPlatformFullName();
// Added data from WER, save to the crash context file.
if (bContextDirty)
{
Context.ToFile();
}
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
uint BuiltFromCL = EngineVersion.Changelist;
string BranchName = EngineVersion.Branch;
if (string.IsNullOrEmpty(BranchName))
{
CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
Context.ToFile();
}
else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
{
CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
Context.ToFile();
}
// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
bool bHasDiagnostics = FindDiagnostics(NewReportPath);
if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
{
// For now ignore all locally made crashes.
CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
Context.ToFile();
}
// Check static reports index for duplicated reports
// This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router)
// This can be safely discontinued when all crashes come from the same SQS
// This DOES NOT scale to multiple CRP instances
lock (ReportIndexLock)
{
if (!ReportIndex.TryAddNewReport(ReportName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
}
if (ShouldDecimateNextReport())
{
CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded);
ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
return(false);
}
lock (NewReportsLock)
{
NewCrashContexts.Enqueue(Context);
}
EnqueueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent);
CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
return(true);
}
private static unsafe bool DecompressDataRouterContent(byte[] CompressedBufferArray, string InLandingZone)
{
// Decompress to landing zone
byte[] UncompressedBufferArray = new byte[Config.Default.MaxUncompressedS3RecordSize];
int UncompressedSize = 0;
fixed(byte *UncompressedBufferPtr = UncompressedBufferArray, CompressedBufferPtr = CompressedBufferArray)
{
Int32 UncompressResult = NativeMethods.UE4UncompressMemoryZLIB((IntPtr)UncompressedBufferPtr,
UncompressedBufferArray.Length,
(IntPtr)CompressedBufferPtr, CompressedBufferArray.Length);
if (UncompressResult < 0)
{
string FailString = "! DecompressDataRouterContent() failed in UE4UncompressMemoryZLIB() with " +
NativeMethods.GetUncompressError(UncompressResult);
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent);
return(false);
}
UncompressedSize = UncompressResult;
}
using (BinaryReader BinaryData = new BinaryReader(new MemoryStream(UncompressedBufferArray, 0, UncompressedSize, false)))
{
char[] MarkerChars = BinaryData.ReadChars(3);
if (MarkerChars[0] == 'C' && MarkerChars[1] == 'R' && MarkerChars[2] == '1')
{
CrashHeader CrashHeader = DataRouterReportQueue.CrashHeader.ParseCrashHeader(BinaryData);
// Create safe directory name and then create the directory on disk
string CrashFolderName = GetSafeFilename(CrashHeader.DirectoryName);
string CrashFolderPath = Path.Combine(InLandingZone, CrashFolderName);
// Early check for duplicate processed report
lock (ReportIndexLock)
{
if (ReportIndex.ContainsReport(CrashFolderName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"DataRouterReportQueue: Duplicate report skipped early {0} in a DataRouterReportQueue", CrashFolderPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
return(false); // this isn't an error so don't set error message
}
}
// Create target folder
int TryIndex = 1;
while (Directory.Exists(CrashFolderPath))
{
CrashFolderPath = Path.Combine(InLandingZone, string.Format("{0}_DUPE{1:D3}", CrashFolderName, TryIndex++));
}
Directory.CreateDirectory(CrashFolderPath);
if (UncompressedSize != CrashHeader.UncompressedSize)
{
CrashReporterProcessServicer.WriteEvent(
string.Format(
"DecompressDataRouterContent() warning UncompressedSize mismatch (embedded={0}, actual={1}) Path={2}",
CrashHeader.UncompressedSize, UncompressedSize, CrashFolderPath));
}
if (!ParseCrashFiles(BinaryData, CrashHeader.FileCount, CrashFolderPath))
{
string FailString = "! DecompressDataRouterContent() failed to write files Path=" + CrashFolderPath;
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent);
return(false);
}
}
else
{
#if ALLOWOLDCLIENTDATA
// Early Data Router upload format was broken.
// Should be [CR1][CrashHeader][File][File][File][File]...
// Actually [Undefined CrashHeader][File][File][File][File]...[CrashHeader]
// Seek to end minus header size
BinaryData.BaseStream.Position = UncompressedSize - DataRouterReportQueue.CrashHeader.FixedSize;
var CrashHeader = DataRouterReportQueue.CrashHeader.ParseCrashHeader(BinaryData);
// Create safe directory name and then create the directory on disk
string CrashFolderName = GetSafeFilename(CrashHeader.DirectoryName);
string CrashFolderPath = Path.Combine(InLandingZone, CrashFolderName);
// Early check for duplicate processed report
lock (ReportIndexLock)
{
if (ReportIndex.ContainsReport(CrashFolderName))
{
// Crash report not accepted by index
CrashReporterProcessServicer.WriteEvent(string.Format(
"DataRouterReportQueue: Duplicate report skipped early {0} in a DataRouterReportQueue", CrashFolderPath));
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
return(false); // this isn't an error so don't set error message
}
}
// Create target folder
int TryIndex = 1;
while (Directory.Exists(CrashFolderPath))
{
CrashFolderPath = Path.Combine(InLandingZone, string.Format("{0}_DUPE{1:D3}", CrashFolderName, TryIndex++));
}
Directory.CreateDirectory(CrashFolderPath);
if (UncompressedSize != CrashHeader.UncompressedSize + CrashHeader.FixedSize)
{
CrashReporterProcessServicer.WriteEvent(
string.Format(
"DecompressDataRouterContent() warning UncompressedSize mismatch (embedded={0}, actual={1}) Path={2}",
CrashHeader.UncompressedSize, UncompressedSize, CrashFolderPath));
}
// Seek to start of files (header size in from start)
BinaryData.BaseStream.Position = CrashHeader.FixedSize;
if (!ParseCrashFiles(BinaryData, CrashHeader.FileCount, CrashFolderPath))
{
string FailString = "! DecompressDataRouterContent() failed to write files Path=" + CrashFolderPath;
CrashReporterProcessServicer.WriteFailure(FailString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3RecordFailedEvent);
return(false);
}
#else
ErrorMessage = "! DecompressDataRouterContent() failed to read invalid data format. Corrupt or old format data received Path=" + CrashFolderPath;
return(false);
#endif
}
}
return(true);
}