/// <summary> ErrorMessage as string[] unescaped. </summary>
public string[] GetErrorMessage()
{
string UnescapedValue = FGenericCrashContext.UnescapeXMLString(ErrorMessage);
UnescapedValue = UnescapedValue.Substring(0, Math.Min(511, UnescapedValue.Length)); // Database limitation.
return(UnescapedValue.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries));
}
/// <summary> UserDescription as string[] unescaped. </summary>
public string[] GetUserDescription()
{
if (string.IsNullOrWhiteSpace(UserDescription))
{
return(new string[0]);
}
string UnescapedValue = FGenericCrashContext.UnescapeXMLString(UserDescription);
return(UnescapedValue.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries));
}
/// <summary>If available, will read CrashContext.runtime-xml.</summary>
public void ReadCrashContextIfAvailable()
{
try
{
//\\epicgames.net\Root\Projects\Paragon\QA_CrashReports
bool bHasCrashContext = HasCrashContextFile();
if (bHasCrashContext)
{
_crashContext = FGenericCrashContext.FromFile(SitePath + GetCrashContextUrl());
bool bTest = _crashContext != null && !string.IsNullOrEmpty(_crashContext.PrimaryCrashProperties.FullCrashDumpLocation);
if (bTest)
{
_bUseFullMinidumpPath = true;
//Some temporary code to redirect to the new file location for fulldumps for paragon.
//Consider removing this once fulldumps stop appearing in the old location.
if (_crashContext.PrimaryCrashProperties.FullCrashDumpLocation.ToLower()
.Contains("\\\\epicgames.net\\root\\dept\\gameqa\\paragon\\paragon_launchercrashdumps"))
{
//Files from old versions of the client may end up in the old location. Check for files there first.
if (!System.IO.Directory.Exists(_crashContext.PrimaryCrashProperties.FullCrashDumpLocation))
{
var suffix =
_crashContext.PrimaryCrashProperties.FullCrashDumpLocation.Substring("\\\\epicgames.net\\root\\dept\\gameqa\\paragon\\paragon_launchercrashdumps".Length);
_crashContext.PrimaryCrashProperties.FullCrashDumpLocation = String.Format("\\\\epicgames.net\\Root\\Projects\\Paragon\\QA_CrashReports{0}", suffix);
//If the file doesn't exist in the new location either then don't use the full minidump path.
_bUseFullMinidumpPath =
System.IO.Directory.Exists(_crashContext.PrimaryCrashProperties.FullCrashDumpLocation);
}
}
//End of temporary code.
FLogger.Global.WriteEvent("ReadCrashContextIfAvailable " + _crashContext.PrimaryCrashProperties.FullCrashDumpLocation + " is " + _bUseFullMinidumpPath);
}
}
}
catch (Exception Ex)
{
Debug.WriteLine("Exception in ReadCrashContextIfAvailable: " + Ex.ToString());
}
}
/// <summary> UserDescription as string[] unescaped. </summary>
public string[] GetUserDescription()
{
string UnescapedValue = FGenericCrashContext.UnescapeXMLString(UserDescription);
return(UnescapedValue.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries));
}
/// <summary> Callstack as string[] unescaped. </summary>
public string[] GetCallstack()
{
string UnescapedValue = FGenericCrashContext.UnescapeXMLString(CallStack);
return(UnescapedValue.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries));
}
void ProcessDumpFile( string DiagnosticsPath, FGenericCrashContext NewContext )
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: Waiting to run MDD on " + DiagnosticsPath);
if (CrashReporterProcessServicer.Symbolicator.Run(DiagnosticsPath, NewContext, ProcessorIndex))
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD finished running on " + DiagnosticsPath);
ReadDiagnosticsFile(NewContext);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessDumpFile: MDD didn't run on " + DiagnosticsPath);
}
}
double ReadProcessAddReport( DirectoryInfo DirInfo, FGenericCrashContext NewContext )
{
try
{
string DumpFileName = null;
string LogFileName = null;
string DiagnosticsFileName = "";
string VideoFileName = null;
string CrashContextName = "";
foreach( FileInfo Info in DirInfo.GetFiles() )
{
switch( Info.Extension.ToLower() )
{
case ".avi":
case ".mp4":
VideoFileName = Info.Name;
break;
case ".runtime-xml":
CrashContextName = Info.Name;
break;
case ".log":
LogFileName = Info.Name;
break;
case ".txt":
if (string.Compare( Info.Name, CrashReporterConstants.DiagnosticsFileName, true ) == 0)
{
DiagnosticsFileName = Info.Name;
ReadDiagnosticsFile( NewContext );
}
break;
case ".dmp":
case ".mdmp":
DumpFileName = Info.Name;
// Check to see if this has been processed locally
FileInfo DiagnosticsInfo = new FileInfo( Path.Combine( DirInfo.FullName, CrashReporterConstants.DiagnosticsFileName ) );
if (!DiagnosticsInfo.Exists && !NewContext.HasProcessedData())
{
ProcessDumpFile( Info.FullName, NewContext );
}
// Check to see if a new one has been created
DiagnosticsInfo.Refresh();
if( DiagnosticsInfo.Exists )
{
DiagnosticsFileName = DiagnosticsInfo.Name;
}
break;
default:
break;
}
}
// Check if the new context has processed data.
if (!NewContext.HasProcessedData())
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "% Warning no callstack or error msg : BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory);
NewContext.PrimaryCrashProperties.ProcessorFailedMessage = "No callstack or error message. Diagnostics missing or failed.";
}
Stopwatch WaitSW = Stopwatch.StartNew();
// Wait for previous task, should not really happen.
int AddReportTaskSlot = WaitForFreeAddReportTask();
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Starting AddReportTask running on slot {0} of {1} ({2} active)", AddReportTaskSlot, Config.Default.AddReportsPerProcessor, GetActiveAddReportTasks()));
double WaitTime = WaitSW.Elapsed.TotalSeconds;
//bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName );
// Save/update crash context to the file.
NewContext.ToFile();
AddReportTasks[AddReportTaskSlot] = Task.Factory.StartNew(() =>
{
bool bAdded = AddReport( DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName );
FinalizeReport( bAdded, DirInfo, NewContext );
} );
return WaitTime;
}
catch( Exception Ex )
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessReport: " + NewContext.CrashDirectory + "\n\n: " + Ex.ToString());
}
return 0.0;
}
private static void GetErrorMessageFromMetadata(FReportData ReportData, FGenericCrashContext CrashContext)
{
if (string.IsNullOrEmpty(CrashContext.PrimaryCrashProperties.ErrorMessage))
{
CrashContext.PrimaryCrashProperties.ErrorMessage = string.Join("\n", ReportData.ErrorMessage);
}
}
/// <summary>
/// A function to process a newly landed crash report.
/// </summary>
/// <param name="NewContext">An instance of the generic crash context</param>
private void ProcessReport( FGenericCrashContext NewContext )
{
Stopwatch ProcessReportSW = Stopwatch.StartNew();
// Just to verify that the report is still there.
DirectoryInfo DirInfo = new DirectoryInfo( NewContext.CrashDirectory );
if( !DirInfo.Exists )
{
// Something very odd happened.
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessReport: Directory not found: " + NewContext.CrashDirectory);
return;
}
double WaitTime = ReadProcessAddReport( DirInfo, NewContext );
// Make sure any log messages have been written to disk
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("ProcessReportTime={0} WaitTime={1}", ProcessReportSW.Elapsed.TotalSeconds.ToString("0.0"), WaitTime.ToString("0.00")));
}
/// <summary>
/// A function to add a report to the database, and rename the relevant files.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="LogFileName">The file name of the log file in the report.</param>
/// <param name="DumpFileName">The file name of the minidump in the report.</param>
/// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param>
private bool AddReport( DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName )
{
try
{
Stopwatch AddReportTime = Stopwatch.StartNew();
// Create an XML representation of a crash
string CrashDetails = CreateCrash( DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null );
if (CrashDetails == "")
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "! CreateCrash no payload: Path=" + NewContext.CrashDirectory);
return false;
}
// Upload the crash to the database, and retrieve the new row id
int ReportID = UploadCrash(CrashDetails);
lock (FailedUploadLock)
{
if (ReportID <= 0)
{
// Upload failed
ConsecutiveFailedUploads++;
if (ConsecutiveFailedUploads == Config.Default.ConsecutiveFailedWebAddLimit)
{
CrashReporterProcessServicer.StatusReporter.Alert("ReportProcessor.AddReport.NoContact", "Cannot contact Crash Report website.", Config.Default.SlackAlertRepeatMinimumMinutes);
CrashReporterProcessServicer.WriteFailure("Cannot contact Crash Report website.");
}
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "! NoUpload: Path=" + NewContext.CrashDirectory);
string PayloadFailedFileName = Path.Combine(DirInfo.FullName, "PayloadFailed.xml");
File.WriteAllText(PayloadFailedFileName, CrashDetails);
return false;
}
// Upload succeeded
ConsecutiveFailedUploads = 0;
}
string IDThenUnderscore = string.Format("{0}_", ReportID);
// Use the row id to name and move the files the way the web site requires
string DestinationFolder = Path.Combine(Config.Default.ProcessedReports, IDThenUnderscore);
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(DestinationFolder, Config.Default.DiskSpaceAlertPercent);
// Move report files to crash reporter file store
if (LogFileName != null)
{
LogFileName = Path.Combine(DirInfo.FullName, LogFileName);
FileInfo LogInfo = new FileInfo(LogFileName);
if(LogInfo.Exists)
{
LogInfo.MoveTo(DestinationFolder + "Launch.log");
}
}
string CrashContextRuntimeName = Path.Combine( DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName );
FileInfo CrashContextInfo = new FileInfo( CrashContextRuntimeName );
if (CrashContextInfo.Exists)
{
CrashContextInfo.MoveTo( DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName );
}
// WERMetaDataName = Path.Combine(DirInfo.FullName, WERMetaDataName);
// FileInfo MetaInfo = new FileInfo(WERMetaDataName);
// if (MetaInfo.Exists)
// {
// MetaInfo.MoveTo(DestinationFolder + "WERMeta.xml");
// }
if( DumpFileName != null )
{
DumpFileName = Path.Combine( DirInfo.FullName, DumpFileName );
FileInfo DumpInfo = new FileInfo( DumpFileName );
if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/ )
{
DumpInfo.MoveTo( DestinationFolder + "MiniDump.dmp" );
}
}
// DiagnosticsFileName = Path.Combine(DirInfo.FullName, DiagnosticsFileName);
// FileInfo DiagnosticsInfo = new FileInfo(DiagnosticsFileName);
// if (DiagnosticsInfo.Exists)
// {
// DiagnosticsInfo.MoveTo(DestinationFolder + CrashReporterConstants.DiagnosticsFileName);
// }
// Move the video (if it exists) to an alternate store
if (VideoFileName != null)
{
DestinationFolder = Path.Combine(Config.Default.ProcessedVideos, IDThenUnderscore);
VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName);
FileInfo VideoInfo = new FileInfo(VideoFileName);
if (VideoInfo.Exists)
{
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(DestinationFolder, Config.Default.DiskSpaceAlertPercent);
VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName);
}
}
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "# WebAdded: ReportID =" + string.Format("{0,7}", ReportID) + " Path=" + NewContext.CrashDirectory);
UpdateProcessedReports();
WebAddCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingSucceededEvent);
double Ratio = (double)WebAddCounter.TotalEvents / (double)ProcessedReports * 100;
double AddedPerDay = (double)WebAddCounter.TotalEvents / Timer.Elapsed.TotalDays;
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) +
string.Format(
"Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4} AddedPerMinute={5:N1}", (int) Ratio,
ProcessedReports, WebAddCounter.TotalEvents, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay, WebAddCounter.EventsPerSecond * 60));
return true;
}
catch( Exception Ex )
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "AddReport: " + DirInfo.Name + "\n\n" + Ex.ToString());
}
return false;
}
/// <summary>
/// Create an Xml payload representing a new crash.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="bHasVideoFile">Whether the report contains a video file.</param>
/// <param name="bHasLog">Whether the report contains a log file.</param>
/// <param name="bHasMinidump">Whether the report contains a minidump.</param>
/// <returns>A string of Xml payload representing the newly found crash report.</returns>
private string CreateCrash( DirectoryInfo DirInfo, FGenericCrashContext NewContext, bool bHasVideoFile, bool bHasLog, bool bHasMinidump )
{
string XmlPayload = "";
try
{
FEngineVersion EngineVersion = new FEngineVersion( NewContext.PrimaryCrashProperties.EngineVersion );
// Create a new crash description for uploading
var NewCrash = new CrashDescription();
NewCrash.CrashType = NewContext.PrimaryCrashProperties.GetCrashType();
NewCrash.BranchName = EngineVersion.GetCleanedBranch();
NewCrash.GameName = NewContext.PrimaryCrashProperties.GameName;
NewCrash.Platform = NewContext.PrimaryCrashProperties.PlatformFullName;
NewCrash.EngineMode = NewContext.PrimaryCrashProperties.EngineMode;
NewCrash.EngineVersion = EngineVersion.VersionNumber;
NewCrash.BuildVersion = NewContext.PrimaryCrashProperties.BuildVersion;
NewCrash.CommandLine = NewContext.PrimaryCrashProperties.CommandLine;
NewCrash.BaseDir = NewContext.PrimaryCrashProperties.BaseDir;
NewCrash.bProcessorFailed = !string.IsNullOrWhiteSpace(NewContext.PrimaryCrashProperties.ProcessorFailedMessage);
NewCrash.Language = NewContext.PrimaryCrashProperties.AppDefaultLocale;
// // Create a locate and get the system language.
// int SystemLanguageCode = 0;
// int.TryParse( ReportData.SystemLanguage, out SystemLanguageCode );
// try
// {
// if( SystemLanguageCode > 0 )
// {
// CultureInfo SystemLanguageCI = new CultureInfo( SystemLanguageCode );
// NewCrash.SystemLanguage = SystemLanguageCI.Name;
// }
// }
// catch( System.Exception )
// {
// // Default to en-US
// NewCrash.SystemLanguage = "en-US";
// }
NewCrash.MachineGuid = NewContext.PrimaryCrashProperties.MachineId; // Valid for all kind of builds, previously only for UE4 releases.
NewCrash.UserName = NewContext.PrimaryCrashProperties.UserName; // Only valid for non-UE4 releases.
NewCrash.EpicAccountId = NewContext.PrimaryCrashProperties.EpicAccountId; // Only valid for UE4 releases.
NewCrash.CallStack = NewContext.PrimaryCrashProperties.GetCallstack();
NewCrash.SourceContext = NewContext.PrimaryCrashProperties.GetSourceContext();
NewCrash.ErrorMessage = NewContext.PrimaryCrashProperties.GetErrorMessage();
NewCrash.UserDescription = NewContext.PrimaryCrashProperties.GetUserDescription();
NewCrash.UserActivityHint = NewContext.PrimaryCrashProperties.UserActivityHint;
NewCrash.CrashGUID = NewContext.PrimaryCrashProperties.CrashGUID;
// Iterate through all files and find a file with the earliest date.
DateTime TimeOfCrash = DateTime.UtcNow;
foreach( var File in DirInfo.GetFiles() )
{
if( File.CreationTimeUtc < TimeOfCrash )
{
TimeOfCrash = File.CreationTimeUtc;
}
}
//NewCrash.TimeofCrash = NewContext.PrimaryCrashProperties.TimeofCrash;
NewCrash.TimeofCrash = TimeOfCrash;
NewCrash.bHasMiniDump = bHasMinidump;
NewCrash.bHasLog = bHasLog;
NewCrash.bHasVideo = bHasVideoFile;
NewCrash.BuiltFromCL = (int)EngineVersion.Changelist;
NewCrash.bAllowToBeContacted = NewContext.PrimaryCrashProperties.bAllowToBeContacted;
// Ignore any callstack that is shorter than expected, usually the callstack is invalid.
if( NewCrash.CallStack.Length <= CrashReporterConstants.MinCallstackDepth )
{
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + "! BadStack: BuiltFromCL=" + string.Format( "{0,7}", NewContext.PrimaryCrashProperties.EngineVersion ) + " Path=" + NewContext.CrashDirectory );
NewContext.PrimaryCrashProperties.ProcessorFailedMessage = string.Format("Callstack was too small. {0} lines (minimum {1})", NewCrash.CallStack.Length, CrashReporterConstants.MinCallstackDepth);
NewContext.ToFile();
}
XmlPayload = XmlHandler.ToXmlString<CrashDescription>( NewCrash );
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "CreateCrash: " + Ex.ToString());
}
return XmlPayload;
}
/// <summary>
/// Finalizes report files.
/// Thread-safe.
/// </summary>
private void FinalizeReport( bool bAdded, DirectoryInfo DirInfo, FGenericCrashContext NewContext )
{
// Only remove if we added the report, otherwise leave all files to further investigation.
if( bAdded )
{
// Remove the report files as we're done with them.
CleanReport( DirInfo );
}
else
{
MoveReportToInvalid( NewContext.CrashDirectory, ReportQueueBase.GetSafeFilename(NewContext.GetAsFilename()) );
}
}
/// <summary>
/// Builds MDD command line args, waits for a MDD task slot, runs MDD and blocks for the result. Writes diag text into DiagnosticsPath folder if successful.
/// </summary>
/// <param name="DiagnosticsPath">Path of the minidump file</param>
/// <param name="Context">The crash context</param>
/// <param name="ProcessorIndex">Processor thread index for logging purposes</param>
/// <returns>True, if successful</returns>
public bool Run(string DiagnosticsPath, FGenericCrashContext Context, int ProcessorIndex)
{
// Use the latest MinidumpDiagnostics from the main branch.
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.DepotRoot, Config.Default.DiskSpaceAlertPercent);
string Win64BinariesDirectory = Path.Combine(Config.Default.DepotRoot, Config.Default.MDDBinariesFolderInDepot);
#if DEBUG
// Note: the debug executable must be built locally or synced from Perforce manually
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics-Win64-Debug.exe");
#else
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics.exe");
#endif
// Don't purge logs
// TODO: make this clear to logs once a day or something (without letting MDD check on every run!)
string PurgeLogsDays = "-1";
FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);
// Pass Windows variants (Win32/64) to MinidumpDiagnostics
string PlatformVariant = Context.PrimaryCrashProperties.PlatformName;
if (PlatformVariant != null && Context.PrimaryCrashProperties.PlatformFullName != null && PlatformVariant.ToUpper().Contains("WINDOWS"))
{
if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win32") ||
Context.PrimaryCrashProperties.PlatformFullName.Contains("32b"))
{
PlatformVariant = "Win32";
}
else if (Context.PrimaryCrashProperties.PlatformFullName.Contains("Win64") ||
Context.PrimaryCrashProperties.PlatformFullName.Contains("64b"))
{
PlatformVariant = "Win64";
}
}
// Build the absolute log file path for MinidumpDiagnostics
string BaseFolder = CrashReporterProcessServicer.SymbolicatorLogFolder;
string SubFolder = DateTime.UtcNow.ToString("yyyy_MM_dd");
string Folder = Path.Combine(BaseFolder, SubFolder);
string AbsLogPath = Path.Combine(Folder, Context.GetAsFilename() + ".log");
Directory.CreateDirectory(Folder);
List<string> MinidumpDiagnosticsParams = new List<string>
(
new string[]
{
"\""+DiagnosticsPath+"\"",
"-BranchName=" + EngineVersion.Branch, // Backward compatibility
"-BuiltFromCL=" + EngineVersion.Changelist, // Backward compatibility
"-GameName=" + Context.PrimaryCrashProperties.GameName,
"-EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion,
"-BuildVersion=" + (string.IsNullOrWhiteSpace(Context.PrimaryCrashProperties.BuildVersion) ?
string.Format("{0}-CL-{1}", EngineVersion.Branch, EngineVersion.Changelist).Replace('/', '+') :
Context.PrimaryCrashProperties.BuildVersion),
"-PlatformName=" + Context.PrimaryCrashProperties.PlatformName,
"-PlatformVariantName=" + PlatformVariant,
"-bUsePDBCache=true",
"-PDBCacheDepotRoot=" + Config.Default.DepotRoot,
"-PDBCachePath=" + Config.Default.MDDPDBCachePath,
"-PDBCacheSizeGB=" + Config.Default.MDDPDBCacheSizeGB,
"-PDBCacheMinFreeSpaceGB=" + Config.Default.MDDPDBCacheMinFreeSpaceGB,
"-PDBCacheFileDeleteDays=" + Config.Default.MDDPDBCacheFileDeleteDays,
"-MutexPDBCache",
"-PDBCacheLock=CrashReportProcessPDBCacheLock",
"-NoTrimCallstack",
"-SyncSymbols",
"-NoP4Symbols",
"-ForceUsePDBCache",
"-MutexSourceSync",
"-SourceSyncLock=CrashReportProcessSourceSyncLock",
"-SyncMicrosoftSymbols",
"-unattended",
"-AbsLog=" + AbsLogPath,
"-DepotIndex=" + Config.Default.DepotIndex,
"-P4User="******"-P4Client=" + Config.Default.P4Client,
"-ini:Engine:[LogFiles]:PurgeLogsDays=" + PurgeLogsDays + ",[LogFiles]:MaxLogFilesOnDisk=-1",
"-LOGTIMESINCESTART"
}
);
LaunchProcess.CaptureMessageDelegate CaptureMessageDelegate = null;
if (Environment.UserInteractive)
{
CaptureMessageDelegate = CrashReporterProcessServicer.WriteMDD;
}
else
{
MinidumpDiagnosticsParams.AddRange
(
new[]
{
"-buildmachine",
"-forcelogflush"
}
);
// Write some debugging message.
CrashReporterProcessServicer.WriteMDD(MinidumpDiagnosticsName + " Params: " + String.Join(" ", MinidumpDiagnosticsParams));
}
Task<bool> NewSymbolicatorTask = Task.FromResult(false);
Stopwatch WaitSW = Stopwatch.StartNew();
Double WaitForLockTime = 0.0;
lock (Tasks)
{
int TaskIdx = Task.WaitAny(Tasks);
Tasks[TaskIdx] = NewSymbolicatorTask = Task<bool>.Factory.StartNew(() =>
{
LaunchProcess ReportParser = new LaunchProcess(MinidumpDiagnosticsName, Path.GetDirectoryName(MinidumpDiagnosticsName), CaptureMessageDelegate,
MinidumpDiagnosticsParams.ToArray());
return ReportParser.WaitForExit(Config.Default.MDDTimeoutMinutes*1000*60) == EWaitResult.Ok;
});
WaitForLockTime = WaitSW.Elapsed.TotalSeconds;
}
NewSymbolicatorTask.Wait();
Double TotalMDDTime = WaitSW.Elapsed.TotalSeconds;
CrashReporterProcessServicer.WriteEvent(string.Format("PROC-{0} ", ProcessorIndex) + string.Format("Symbolicator.Run: Thread blocked for {0:N1}s then MDD ran for {1:N1}s", WaitForLockTime, TotalMDDTime - WaitForLockTime));
return NewSymbolicatorTask.Result;
}
/// <summary> Tries to dequeue a report from the list. </summary>
public bool TryDequeueReport(out FGenericCrashContext Context)
{
lock (NewReportsLock)
{
if (NewCrashContexts.Count > 0)
{
Context = NewCrashContexts.Dequeue();
DequeueCounter.AddEvent();
CrashReporterProcessServicer.StatusReporter.IncrementCount(QueueProcessingStartedEventName);
CrashReporterProcessServicer.WriteEvent(string.Format("- Dequeued: {0:N1}/minute BuiltFromCL={1,7} Path={2}", DequeueCounter.EventsPerSecond * 60, Context.PrimaryCrashProperties.EngineVersion, Context.CrashDirectory));
return true;
}
}
Context = null;
return false;
}
/// <summary> Creates a new instance of the crash report context from the specified file. </summary>
public static FGenericCrashContext FromString(string Text)
{
// The crash report context contains invalid characters, we need to fix them all in order to parse the XML.
HashSet <string> TagsToIgnore = new HashSet <string>()
{
"<FGenericCrashContext>", "<RuntimeProperties>", "<PlatformProperties>", "</FGenericCrashContext>", "</RuntimeProperties>", "</PlatformProperties>"
};
string[] XMLText = Text.Split(new string[] { Environment.NewLine }, StringSplitOptions.None);
string NodeOpen = "";
string NodeEnd = "";
List <string> Multiline = new List <string>();
List <int> MultiIndex = new List <int>();
int OpenEnd = 0;
int EndStart = 0;
for (int LineIndex = 1; LineIndex < XMLText.Length; LineIndex++)
{
string XMLLine = XMLText[LineIndex].Replace("\t", "");
bool bIgnore = TagsToIgnore.Contains(XMLLine);
if (bIgnore)
{
continue;
}
// XML node without data, skip.
if (XMLLine.Contains("/>"))
{
continue;
}
if (NodeOpen == "")
{
// Parse the tag.
//<Version>"1"</Version>
//<Version />
int OpenStart = XMLLine.IndexOf('<');
OpenEnd = XMLLine.IndexOf('>', OpenStart);
NodeOpen = XMLLine.Substring(OpenStart + 1, OpenEnd - OpenStart - 1);
}
EndStart = XMLLine.IndexOf("</");
int EndEnd = 0;
if (EndStart != -1)
{
EndEnd = XMLLine.IndexOf('>', EndStart);
NodeEnd = XMLLine.Substring(EndStart + 2, EndEnd - EndStart - 2);
}
else
{
Multiline.Add(XMLLine.Substring(OpenEnd != 0 ? OpenEnd + 1 : 0));
MultiIndex.Add(LineIndex);
OpenEnd = 0;
}
// Valid tag, fix invalid characters.
if (NodeOpen == NodeEnd)
{
if (Multiline.Count == 0)
{
string NodeValue = XMLLine.Substring(OpenEnd + 1, EndStart - OpenEnd - 1);
string EscapedXMLText = EscapeXMLString(UnescapeXMLString(NodeValue)); // Required for support old and new type format.
XMLText[LineIndex] = string.Format("<{0}>{1}</{0}>", NodeOpen, EscapedXMLText);
}
else
{
Multiline.Add(XMLLine.Substring(0, EndStart));
MultiIndex.Add(LineIndex);
for (int Inner = 0; Inner < Multiline.Count; Inner++)
{
string EscapedXMLText = EscapeXMLString(UnescapeXMLString(Multiline[Inner]));
if (Inner == 0)
{
XMLText[MultiIndex[Inner]] = string.Format("<{0}>{1}", NodeOpen, EscapedXMLText);
}
else if (Inner == Multiline.Count - 1)
{
XMLText[MultiIndex[Inner]] = string.Format("{1}</{0}>", NodeOpen, EscapedXMLText);
}
else
{
XMLText[MultiIndex[Inner]] = EscapedXMLText;
}
}
Multiline.Clear();
MultiIndex.Clear();
}
NodeOpen = NodeEnd = "";
}
}
string XMLTextJoined = string.Join(Environment.NewLine, XMLText);
FGenericCrashContext Context = XmlHandler.FromXmlString <FGenericCrashContext>(XMLTextJoined);
if (Context.UntypedXMLNodes != null)
{
var Typed = ((IEnumerable)Context.UntypedXMLNodes)
.Cast <XmlNode>()
.Select(X => new KeyValuePair <string, string>(X.Name, X.InnerText))
.ToList();
}
return(Context);
}
/// <summary> Reads callstack, source context and error message from the diagnostics file, if exists. </summary>
private void ReadDiagnosticsFile( FGenericCrashContext NewContext )
{
// Read the callstack and the source context from the diagnostics files.
string CrashDiagnosticsPath = Path.Combine( NewContext.CrashDirectory, CrashReporterConstants.DiagnosticsFileName );
if (File.Exists( CrashDiagnosticsPath ))
{
NewContext.PrimaryCrashProperties.CallStack = FGenericCrashContext.EscapeXMLString( string.Join( "\n", FReportData.GetCallStack( CrashDiagnosticsPath ) ) );
NewContext.PrimaryCrashProperties.SourceContext = FGenericCrashContext.EscapeXMLString( string.Join( "\n", FReportData.GetSourceContext( CrashDiagnosticsPath ) ) );
if (NewContext.PrimaryCrashProperties.ErrorMessage.Length == 0)
{
NewContext.PrimaryCrashProperties.ErrorMessage = FGenericCrashContext.EscapeXMLString( string.Join( "\n", FReportData.GetExceptionDescription( CrashDiagnosticsPath ) ) );
}
}
}
/// <summary> Converts WER metadata xml file to the crash context. </summary>
private static void ConvertMetadataToCrashContext(FReportData ReportData, string NewReportPath, ref FGenericCrashContext OutCrashContext)
{
if (OutCrashContext == null)
{
OutCrashContext = new FGenericCrashContext();
}
OutCrashContext.PrimaryCrashProperties.CrashVersion = (int)ECrashDescVersions.VER_1_NewCrashFormat;
//OutCrashContext.PrimaryCrashProperties.ProcessId = 0; don't overwrite valid ids, zero is default anyway
OutCrashContext.PrimaryCrashProperties.CrashGUID = new DirectoryInfo(NewReportPath).Name;
// OutCrashContext.PrimaryCrashProperties.IsInternalBuild
// OutCrashContext.PrimaryCrashProperties.IsPerforceBuild
// OutCrashContext.PrimaryCrashProperties.IsSourceDistribution
// OutCrashContext.PrimaryCrashProperties.SecondsSinceStart
OutCrashContext.PrimaryCrashProperties.GameName = ReportData.GameName;
// OutCrashContext.PrimaryCrashProperties.ExecutableName
// OutCrashContext.PrimaryCrashProperties.BuildConfiguration
// OutCrashContext.PrimaryCrashProperties.PlatformName
// OutCrashContext.PrimaryCrashProperties.PlatformNameIni
OutCrashContext.PrimaryCrashProperties.PlatformFullName = ReportData.Platform;
OutCrashContext.PrimaryCrashProperties.EngineMode = ReportData.EngineMode;
OutCrashContext.PrimaryCrashProperties.EngineVersion = ReportData.GetEngineVersion();
OutCrashContext.PrimaryCrashProperties.BuildVersion = ReportData.BuildVersion;
OutCrashContext.PrimaryCrashProperties.CommandLine = ReportData.CommandLine;
// OutCrashContext.PrimaryCrashProperties.LanguageLCID
// Create a locate and get the language.
int LanguageCode = 0;
int.TryParse(ReportData.Language, out LanguageCode);
try
{
if (LanguageCode > 0)
{
CultureInfo LanguageCI = new CultureInfo(LanguageCode);
OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = LanguageCI.Name;
}
}
catch (Exception)
{
OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = null;
}
if (string.IsNullOrEmpty(OutCrashContext.PrimaryCrashProperties.AppDefaultLocale))
{
// Default to en-US
OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = "en-US";
}
// OutCrashContext.PrimaryCrashProperties.IsUE4Release
string WERUserName = ReportData.UserName;
if (!string.IsNullOrEmpty(WERUserName) || string.IsNullOrEmpty(OutCrashContext.PrimaryCrashProperties.UserName))
{
OutCrashContext.PrimaryCrashProperties.UserName = WERUserName;
}
OutCrashContext.PrimaryCrashProperties.BaseDir = ReportData.BaseDir;
// OutCrashContext.PrimaryCrashProperties.RootDir
OutCrashContext.PrimaryCrashProperties.MachineId = ReportData.MachineId;
OutCrashContext.PrimaryCrashProperties.EpicAccountId = ReportData.EpicAccountId;
// OutCrashContext.PrimaryCrashProperties.CallStack
// OutCrashContext.PrimaryCrashProperties.SourceContext
OutCrashContext.PrimaryCrashProperties.UserDescription = string.Join("\n", ReportData.UserDescription);
// OutCrashContext.PrimaryCrashProperties.CrashDumpMode
// OutCrashContext.PrimaryCrashProperties.Misc.NumberOfCores
// OutCrashContext.PrimaryCrashProperties.Misc.NumberOfCoresIncludingHyperthreads
// OutCrashContext.PrimaryCrashProperties.Misc.Is64bitOperatingSystem
// OutCrashContext.PrimaryCrashProperties.Misc.CPUVendor
// OutCrashContext.PrimaryCrashProperties.Misc.CPUBrand
// OutCrashContext.PrimaryCrashProperties.Misc.PrimaryGPUBrand
// OutCrashContext.PrimaryCrashProperties.Misc.OSVersionMajor
// OutCrashContext.PrimaryCrashProperties.Misc.OSVersionMinor
// OutCrashContext.PrimaryCrashProperties.Misc.AppDiskTotalNumberOfBytes
// OutCrashContext.PrimaryCrashProperties.Misc.AppDiskNumberOfFreeBytes
// OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysical
// OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalVirtual
// OutCrashContext.PrimaryCrashProperties.MemoryStats.PageSize
// OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysicalGB
// OutCrashContext.PrimaryCrashProperties.MemoryStats.AvailablePhysical
// OutCrashContext.PrimaryCrashProperties.MemoryStats.AvailableVirtual
// OutCrashContext.PrimaryCrashProperties.MemoryStats.UsedPhysical
// OutCrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedPhysical
// OutCrashContext.PrimaryCrashProperties.MemoryStats.UsedVirtual
// OutCrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedVirtual
// OutCrashContext.PrimaryCrashProperties.MemoryStats.bIsOOM
// OutCrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationSize
// OutCrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationAlignment
OutCrashContext.PrimaryCrashProperties.TimeofCrash = new DateTime(ReportData.Ticks);
OutCrashContext.PrimaryCrashProperties.bAllowToBeContacted = ReportData.AllowToBeContacted;
OutCrashContext.PrimaryCrashProperties.DeploymentName = ReportData.DeploymentName;
OutCrashContext.PrimaryCrashProperties.IsEnsure = ReportData.IsEnsure;
OutCrashContext.PrimaryCrashProperties.IsAssert = ReportData.IsAssert;
OutCrashContext.PrimaryCrashProperties.CrashType = ReportData.CrashType;
GetErrorMessageFromMetadata(ReportData, OutCrashContext);
}