/// <summary>
/// Delete a report directory.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The directory to delete</param>
public static void CleanReport(DirectoryInfo DirInfo)
{
bool bWriteException = true;
for (int Retry = 0; Retry < 3; ++Retry)
{
try
{
foreach (FileInfo Info in DirInfo.GetFiles())
{
Info.IsReadOnly = false;
}
DirInfo.Delete(true /* delete contents */);
break;
}
catch (Exception Ex)
{
if (bWriteException)
{
CrashReporterProcessServicer.WriteException("CleanReport: " + Ex.ToString());
bWriteException = false;
}
}
System.Threading.Thread.Sleep(100);
}
}
private bool DequeueRecordSQS(out string OutRecordString)
{
OutRecordString = string.Empty;
try
{
#if DEBUG
bool bNoDeleteFromSQS = true;
#else
bool bNoDeleteFromSQS = false;
#endif
Message DequeuedMessage = CrashReporterProcessServicer.DataRouterAWS.SQSDequeueMessage(Config.Default.AWSSQSQueueInputUrl, bNoDeleteFromSQS);
if (DequeuedMessage != null)
{
OutRecordString = DequeuedMessage.Body;
return(true);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("DequeueRecordSQS: " + Ex, Ex);
}
return(false);
}
private bool DequeueRecordSQS(out string OutRecordString)
{
OutRecordString = string.Empty;
try
{
var ReceiveRequest = new ReceiveMessageRequest
{
QueueUrl = Config.Default.AWSSQSQueueUrl,
MaxNumberOfMessages = 1
};
var ReceiveResponse = SqsClient.ReceiveMessage(ReceiveRequest);
if (ReceiveResponse.Messages.Count == 1)
{
var Message = ReceiveResponse.Messages[0];
if (Message != null && TryDeleteRecordSQS(Message))
{
OutRecordString = Message.Body;
return(true);
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("DequeueRecordSQS: " + Ex.ToString());
}
return(false);
}
/// <summary>
/// Moves specified report to the directory where are stored invalid reports.
/// Thread-safe.
/// </summary>
public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename)
{
try
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingFailedEvent);
DirectoryInfo DirInfo = new DirectoryInfo(ReportName);
// Rename the report directory, so we should be able to quickly find the invalid reports.
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(Config.Default.InvalidReportsDirectory, Config.Default.DiskSpaceAlertPercent);
Directory.CreateDirectory(Config.Default.InvalidReportsDirectory);
string DestinationDirectory = Path.Combine(Config.Default.InvalidReportsDirectory, ReportNameAsFilename);
Directory.CreateDirectory(DestinationDirectory);
// Copy all files from the source directory. We can't use MoveTo due to different disc location.
foreach (FileInfo File in DirInfo.GetFiles())
{
string DestinationFilepath = Path.Combine(DestinationDirectory, File.Name);
File.CopyTo(DestinationFilepath, true);
}
DirInfo.Delete(true);
CrashReporterProcessServicer.WriteEvent(string.Format("Moved to {0}", DestinationDirectory));
UpdateProcessedReports();
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex);
}
}
private void TryGetNewS3Crashes(int CrashCount)
{
int NewCrashCount = 0;
while (NewCrashCount < CrashCount)
{
string SQSRecord = "<unset>";
try
{
if (!DequeueRecordSQS(out SQSRecord))
{
// Queue empty
break;
}
var RecordPair = SQSRecord.Split(',');
if (RecordPair.Length != 2)
{
CrashReporterProcessServicer.WriteFailure("TryGetNewS3Crashes: bad SQS message was " + SQSRecord);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
continue;
}
string S3BucketName = RecordPair[0];
string S3Key = RecordPair[1];
string ReadableRequestString = "Bucket=" + S3BucketName + " Key=" + S3Key;
var ObjectRequest = new GetObjectRequest
{
BucketName = S3BucketName,
Key = S3Key
};
using (Stream ProtocolBufferStream = new MemoryStream())
{
using (GetObjectResponse ObjectResponse = S3Client.GetObject(ObjectRequest))
{
using (Stream ResponseStream = ObjectResponse.ResponseStream)
{
if (!TryDecompResponseStream(ResponseStream, ProtocolBufferStream))
{
CrashReporterProcessServicer.WriteFailure("! GZip fail in DecompResponseStream(): " + ReadableRequestString);
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
continue;
}
}
}
NewCrashCount += UnpackRecordsFromDelimitedProtocolBuffers(ProtocolBufferStream, LandingZone, ReadableRequestString);
}
}
catch (Exception ex)
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReadS3FileFailedEvent);
CrashReporterProcessServicer.WriteException("TryGetNewS3Crashes: failure during processing SQS record " + SQSRecord +
"\n" + ex);
}
}
}
public static void WriteToFile()
{
try
{
if (File.Exists(Filepath))
{
File.Delete(Filepath + ".backup");
File.Move(Filepath, Filepath + ".backup");
}
using (var Writer = File.CreateText(Filepath))
{
var Today = DateTime.UtcNow.Date;
foreach (var Item in Index)
{
if (Today - Item.Value.Date <= Retention)
{
Writer.WriteLine(ItemToString(Item));
}
}
}
LastFlush = DateTime.UtcNow;
}
catch (Exception ex)
{
CrashReporterProcessServicer.WriteException(string.Format("Failed to write ReportIndex to {0}. Exception was: {1}", Filepath, ex));
CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.WriteToFile", string.Format("Failed to write ReportIndex to {0}", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes);
}
}
public void AlertOnLowDisk(string Filepath, float AlertThresholdPercent)
{
try
{
string Drive;
if (!StorageSpaceHelper.TryGetDriveLetter(Filepath, out Drive))
{
throw new CrashReporterException("Failed to get drive letter for path " + Filepath);
}
Int64 FreeSpace;
float FreePercent;
if (StorageSpaceHelper.TryGetSpaceAvailable(Drive, out FreeSpace, out FreePercent))
{
if (FreePercent < AlertThresholdPercent)
{
Alert("AlertOnLowDisk" + Drive, "Low disk space warning on " + Drive + " =>> " + GetDiskSpaceString(FreeSpace, FreePercent), 3 * Config.Default.SlackAlertRepeatMinimumMinutes);
}
}
else
{
CrashReporterProcessServicer.WriteEvent("Failed to read disk space for " + Drive);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("AlertOnLowDisk failed: " + Ex, Ex);
}
}
/// <summary>
/// Moves specified report to the directory where are stored invalid reports.
/// Thread-safe.
/// </summary>
public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename)
{
try
{
DirectoryInfo DirInfo = new DirectoryInfo(ReportName);
// Rename the report directory, so we should be able to quickly find the invalid reports.
Directory.CreateDirectory(Properties.Settings.Default.InvalidReportsDirectory);
string CleanFilename = String.Concat(ReportNameAsFilename.Split(Path.GetInvalidFileNameChars()));
string DestinationDirectory = Path.Combine(Properties.Settings.Default.InvalidReportsDirectory, CleanFilename);
Directory.CreateDirectory(DestinationDirectory);
// Copy all files from the source directory. We can't use MoveTo due to different disc location.
foreach (FileInfo File in DirInfo.GetFiles())
{
string DestinationFilepath = Path.Combine(DestinationDirectory, File.Name);
File.CopyTo(DestinationFilepath, true);
}
DirInfo.Delete(true);
CrashReporterProcessServicer.WriteEvent(string.Format("Moved to {0}", DestinationDirectory));
UpdateProcessedReports();
}
catch (System.Exception Ex)
{
CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex.ToString());
}
}
private static void UploadFileToS3(FileInfo FileInfo, string DestFilepath, bool bCompressed, string CompressedSuffix = null)
{
try
{
if (bCompressed)
{
string LocalCompressedFilepath = FileInfo.FullName;
if (CompressedSuffix != null)
{
LocalCompressedFilepath += CompressedSuffix;
}
else
{
LocalCompressedFilepath += ".gz";
}
byte[] UncompressedBytes = File.ReadAllBytes(FileInfo.FullName);
int ReturnCode = NativeMethods.CompressFileGZIP(LocalCompressedFilepath, UncompressedBytes);
if (ReturnCode < 0)
{
throw new CrashReporterException(string.Format("Failed to compress {0} to gzip file with error {1}", FileInfo.FullName, NativeMethods.GetZlibError(ReturnCode)));
}
string CompressedDestFilepath = DestFilepath;
if (CompressedSuffix != null)
{
CompressedDestFilepath += CompressedSuffix;
}
PutObjectResponse Response = CrashReporterProcessServicer.OutputAWS.PutS3ObjectFromFile(Config.Default.AWSS3OutputBucket,
CompressedDestFilepath,
LocalCompressedFilepath);
if (Response == null || Response.HttpStatusCode != HttpStatusCode.OK)
{
throw new CrashReporterException(string.Format("Failed to upload compressed gzip {0} to {1}", LocalCompressedFilepath, CompressedDestFilepath));
}
}
else
{
PutObjectResponse Response = CrashReporterProcessServicer.OutputAWS.PutS3ObjectFromFile(Config.Default.AWSS3OutputBucket,
DestFilepath,
FileInfo.FullName);
if (Response == null || Response.HttpStatusCode != HttpStatusCode.OK)
{
throw new CrashReporterException(string.Format("Failed to upload {0} to {1}", FileInfo.FullName, DestFilepath));
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("UploadFileToS3: " + Ex, Ex);
}
}
public void ReadFromFile()
{
if (!IsEnabled)
{
return;
}
try
{
Index = new Dictionary <string, DateTime>();
if (!File.Exists(Filepath))
{
if (File.Exists(Filepath + ".backup"))
{
CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read ReportIndex from {0}. Attempting to read from {1}", Filepath, Filepath + ".backup"));
CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.ReadFromFile.UsingBackup", string.Format("Failed to read ReportIndex from {0}. Using backup.", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes);
File.Move(Filepath + ".backup", Filepath);
}
else
{
CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read ReportIndex from {0}. Generating new one.", Filepath));
File.Create(Filepath).Close();
}
}
using (var Reader = File.OpenText(Filepath))
{
string ItemStringRaw;
while ((ItemStringRaw = Reader.ReadLine()) != null)
{
string ItemString = ItemStringRaw.Trim();
if (!string.IsNullOrWhiteSpace(ItemString))
{
KeyValuePair <string, DateTime> NewItem;
if (!TryParseItemString(ItemString, out NewItem))
{
CrashReporterProcessServicer.WriteFailure(string.Format("Failed to read line from ReportIndex: {0}.", ItemString));
continue;
}
Index.Add(NewItem.Key, NewItem.Value);
}
}
}
LastFlush = DateTime.UtcNow;
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("Failed to read ReportIndex from {0}. Exception was: {1}", Filepath, Ex), Ex);
CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.ReadFromFile", string.Format("Failed to read ReportIndex from {0}", Filepath), Config.Default.SlackAlertRepeatMinimumMinutes);
}
}
/// <summary>
/// Returns the count of items in the SQS
/// </summary>
private int GetSQSCount()
{
try
{
return(CrashReporterProcessServicer.DataRouterAWS.GetSQSQueueCount(Config.Default.AWSSQSQueueInputUrl));
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("GetSQSCount: " + Ex, Ex);
}
return(0);
}
/// <summary>
/// Main processing thread.
/// </summary>
/// <remarks>All exceptions are caught and written to the event log.</remarks>
private void Init()
{
ProcessorTask = new Task(() =>
{
// parse out hte list of blacklisted games.
string[] GameNamesToBlacklist = (Config.Default.GameNamesToBlacklist == null ? "" : Config.Default.GameNamesToBlacklist).ToLowerInvariant().Split(',');
while (!CancelSource.IsCancellationRequested)
{
try
{
bool bIdle = true;
foreach (var Queue in Watcher.ReportQueues)
{
FGenericCrashContext NewContext = null;
if (Queue.TryDequeueReport(out NewContext))
{
bool bIsBlacklistedGame = GameNamesToBlacklist.Contains(NewContext.PrimaryCrashProperties.GameName.ToLowerInvariant());
// if it's blacklisted, skip it, else process it.
if (bIsBlacklistedGame)
{
CrashReporterProcessServicer.WriteEvent(string.Format("Discarding crash from blacklisted GameName '{0}'", NewContext.PrimaryCrashProperties.GameName));
// delete the report from disk since we don't care about it.
FinalizeReport(AddReportResult.Added, new DirectoryInfo(NewContext.CrashDirectory), NewContext);
}
else
{
ProcessReport(NewContext);
}
// The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest
bIdle = false;
break;
}
}
if (bIdle)
{
// Don't use the CPU if we don't need.
Thread.Sleep(1000);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessNewReports: " + Ex, Ex);
}
TickStatic(Watcher);
}
});
}
/// <summary>
/// Call the web service function with an Xml payload to add a crash to the database.
/// </summary>
/// <param name="Payload">Xml representation of a new crash to upload.</param>
/// <returns>The database id of the newly added row.</returns>
private int UploadCrash(string Payload)
{
int NewID = -1;
try
{
// Simple suppression by blanking out the URL for local testing
if (Config.Default.CrashReportWebSite.Length > 0)
{
bool bDebug = false;
string RequestString;
if (!bDebug)
{
RequestString = "http://" + Config.Default.CrashReportWebSite + ":80/Crashes/AddCrash/-1";
}
else
{
RequestString = "http://localhost:80/Crashes/AddCrash/-1";
}
string ErrorMessage = string.Empty;
for (int Retry = 0; Retry < 3; ++Retry)
{
string ResponseString = SimpleWebRequest.GetWebServiceResponse(RequestString, Payload);
if (ResponseString.Length > 0)
{
// Convert response into a string
CrashReporterResult Result = XmlHandler.FromXmlString <CrashReporterResult>(ResponseString);
if (Result.ID > 0)
{
NewID = Result.ID;
break;
}
ErrorMessage = Result.Message;
}
Thread.Sleep(200);
}
if (NewID == -1)
{
CrashReporterProcessServicer.WriteFailure(string.Format("PROC-{0} ", ProcessorIndex) + "UploadCrash: " + ErrorMessage);
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "UploadCrash: " + Ex.ToString());
}
return(NewID);
}
/// <summary>
/// Delete a report directory.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The directory to delete</param>
public static void CleanReport(DirectoryInfo DirInfo)
{
const int MaxRetries = 3;
bool bWriteException = true;
for (int Retry = 0; Retry < MaxRetries; ++Retry)
{
try
{
if (!DirInfo.Exists)
{
break;
}
foreach (FileInfo Info in DirInfo.GetFiles())
{
Info.IsReadOnly = false;
Info.Attributes = FileAttributes.Normal;
}
DirInfo.Delete(true /* delete contents */);
// Random failures to delete with no exception seen regularly - retry
DirInfo.Refresh();
if (DirInfo.Exists)
{
CrashReporterProcessServicer.WriteEvent("CleanReport: Failed to delete folder without an Exception " + DirInfo);
Thread.Sleep(500);
continue;
}
break;
}
catch (Exception Ex)
{
if (bWriteException)
{
CrashReporterProcessServicer.WriteException("CleanReport: " + Ex, Ex);
bWriteException = false;
}
}
System.Threading.Thread.Sleep(100);
}
DirInfo.Refresh();
if (DirInfo.Exists)
{
CrashReporterProcessServicer.WriteEvent(string.Format("CleanReport: Failed to delete folder {0} after {1} retries", DirInfo, MaxRetries));
}
}
public void WriteToFile()
{
if (!IsEnabled)
{
return;
}
lock (FileLock)
{
try
{
if (File.Exists(Filepath))
{
File.Delete(Filepath + ".backup");
File.Move(Filepath, Filepath + ".backup");
}
using (MemoryStream MemStream = new MemoryStream())
{
using (StreamWriter Writer = new StreamWriter(MemStream))
{
var Today = DateTime.UtcNow.Date;
lock (IndexLock)
{
foreach (var Item in Index)
{
if (Today - Item.Value.Date <= Retention)
{
Writer.WriteLine(ItemToString(Item));
}
}
Writer.Flush();
}
File.WriteAllBytes(Filepath, MemStream.ToArray());
}
}
LastFlush = DateTime.UtcNow;
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("Failed to write ReportIndex to {0}. Exception was: {1}", Filepath, Ex), Ex);
CrashReporterProcessServicer.StatusReporter.Alert("ReportIndex.WriteToFile", string.Format("Failed to write ReportIndex to {0}", Filepath),
Config.Default.SlackAlertRepeatMinimumMinutes);
}
}
}
/// <summary>
/// Delete report folders older than a certain age to avoid unbounded growth of the crash repository.
/// </summary>
/// <remarks>The folders for the deduplication process older than the property 'DaysToSunsetReport' days old are deleted.</remarks>
private void CleanRepository()
{
try
{
foreach (var Queue in Watcher.ReportQueues)
{
Queue.CleanLandingZone();
}
CrashReporterProcessServicer.Log.CleanOutOldLogs(Properties.Settings.Default.DaysToSunsetReport);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CleanRepository: " + Ex.ToString());
}
}
/// <summary>
/// Delete report folders older than a certain age to avoid unbounded growth of the crash repository.
/// </summary>
/// <remarks>The folders for the deduplication process older than the property 'DeleteWaitingReportsDays' days old are deleted.</remarks>
private static void CleanRepository(ReportWatcher InWatcher)
{
try
{
foreach (var Queue in InWatcher.ReportQueues)
{
Queue.CleanLandingZone();
}
CrashReporterProcessServicer.Log.CleanOutOldLogs(Config.Default.DeleteWaitingReportsDays);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CleanRepository: " + Ex, Ex);
}
}
/// <summary>
/// Look for new report folders and add them to the publicly available thread-safe queue
/// </summary>
public void CheckForNewReports()
{
try
{
var NewReportPath = "";
var ReportsInLandingZone = new ReportIdSet();
// Check the landing zones for new reports
DirectoryInfo DirInfo = new DirectoryInfo(LandingZone);
// Couldn't find a landing zone, skip and try again later.
// Crash receiver will recreate them if needed.
if (!DirInfo.Exists)
{
CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone);
return;
}
// Add any new reports
foreach (var SubDirInfo in DirInfo.GetDirectories())
{
try
{
if (Directory.Exists(SubDirInfo.FullName))
{
NewReportPath = SubDirInfo.FullName;
ReportsInLandingZone.Add(NewReportPath);
if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath))
{
EnqueueNewReport(NewReportPath);
}
}
}
catch (System.Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex.ToString());
ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks);
}
}
//CrashReporterProcessServicer.WriteEvent( string.Format( "ReportsInLandingZone={0}, ReportsInLandingZoneLastTimeWeChecked={1}", ReportsInLandingZone.Count, ReportsInLandingZoneLastTimeWeChecked.Count ) );
ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone;
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex.ToString());
}
}
private void UploadFileToS3(FileInfo FileInfo, string DestFilename)
{
try
{
PutObjectResponse Response = CrashReporterProcessServicer.OutputAWS.PutS3ObjectFromFile(Config.Default.AWSS3OutputBucket,
Config.Default.AWSS3OutputKeyPrefix + DestFilename,
FileInfo.FullName);
if (Response == null || Response.HttpStatusCode != HttpStatusCode.OK)
{
throw new CrashReporterException(string.Format("Failed to upload {0} to {1}", FileInfo.FullName, DestFilename));
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("UploadFileToS3: " + Ex, Ex);
}
}
/// <summary>
/// Call the web service function with an Xml payload to add a crash to the database.
/// </summary>
/// <param name="Payload">Xml representation of a new crash to upload.</param>
/// <returns>The database id of the newly added row.</returns>
private int UploadCrash(string Payload)
{
int NewID = -1;
try
{
// Simple suppression by blanking out the URL for local testing
if (Properties.Settings.Default.CrashReportWebSite.Length > 0)
{
bool bDebug = false;
string RequestString;
if (!bDebug)
{
RequestString = "http://" + Properties.Settings.Default.CrashReportWebSite + ":80/Crashes/AddCrash/-1";
}
else
{
RequestString = "http://localhost:80/Crashes/AddCrash/-1";
}
string ResponseString = SimpleWebRequest.GetWebServiceResponse(RequestString, Payload);
if (ResponseString.Length > 0)
{
// Convert response into a string
CrashReporterResult Result = XmlHandler.FromXmlString <CrashReporterResult>(ResponseString);
if (Result.ID > 0)
{
NewID = Result.ID;
}
else
{
CrashReporterProcessServicer.WriteFailure("UploadCrash: " + Result.Message);
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("UploadCrash: " + Ex.ToString());
}
return(NewID);
}
/// <summary>
/// Moves specified report to the directory where are stored invalid reports.
/// Thread-safe.
/// </summary>
public static void MoveReportToInvalid(string ReportName, string ReportNameAsFilename)
{
try
{
CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ProcessingFailedEvent);
DirectoryInfo DirInfo = new DirectoryInfo(ReportName);
if (Config.Default.InvalidReportsToAWS)
{
string LeafReportName = Path.GetFileName(ReportName);
DateTime S3KeyTime = DateTime.UtcNow;
// Copy all files from the source directory. We can't use MoveTo due to different disc location.
int FilesMoved = 0;
foreach (FileInfo InvalidFile in DirInfo.GetFiles())
{
try
{
string S3IDPrefix = string.Format("/{0}_{1}_{2}/{3}/{4}/", S3KeyTime.Year, S3KeyTime.Month, S3KeyTime.Day, S3KeyTime.Hour, LeafReportName);
UploadFileToS3(InvalidFile, Config.Default.AWSS3InvalidKeyPrefix + S3IDPrefix + InvalidFile.Name, false);
FilesMoved++;
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteEvent("MoveReportToInvalid: Failed to write report file " + LeafReportName + ": " + InvalidFile.Name);
CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex);
}
}
CrashReporterProcessServicer.WriteEvent(string.Format("MoveReportToInvalid moved {0} file(s) from {1} to S3", FilesMoved, LeafReportName));
}
DirInfo.Delete(true);
UpdateProcessedReports();
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("MoveReportToInvalid: " + Ex, Ex);
}
}
public void AlertOnConsecutiveFails(string AlertKey, string AlertText, string RecoveredText, TimeSpan TimeSinceSuccessThreshold, bool bSucceeded)
{
try
{
lock (DataLock)
{
if (bSucceeded)
{
// Success - clear from list of failing keys
DateTime FailAlertStartTime;
if (FailAlertStartTimes.TryGetValue(AlertKey, out FailAlertStartTime))
{
if (FailAlertStartTime == DateTime.MaxValue)
{
Alert(AlertKey, RecoveredText, 0);
}
FailAlertStartTimes.Remove(AlertKey);
}
}
else if (!FailAlertStartTimes.ContainsKey(AlertKey))
{
// Failed but no record yet - first failure so note the time
FailAlertStartTimes.Add(AlertKey, DateTime.UtcNow);
}
else
{
// Failed and not the first consecutive fail
TimeSpan TimeSinceStartedFailing = DateTime.UtcNow - FailAlertStartTimes[AlertKey];
if (TimeSinceStartedFailing > TimeSinceSuccessThreshold)
{
FailAlertStartTimes[AlertKey] = DateTime.MaxValue; // forces future fails to make TimeSinceStartedFailing negative until at least one success
Alert(AlertKey, AlertText, 0);
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("AlertOnConsecutiveFails failed: " + Ex, Ex);
}
}
/// <summary>
/// Main processing thread.
/// </summary>
/// <remarks>All exceptions are caught and written to the event log.</remarks>
private void Init()
{
var Cancel = CancelSource.Token;
ProcessorTask = new Task(() =>
{
while (!Cancel.IsCancellationRequested)
{
try
{
bool bIdle = true;
foreach (var Queue in Watcher.ReportQueues)
{
FGenericCrashContext NewContext = null;
if (Queue.TryDequeueReport(out NewContext))
{
ProcessReport(NewContext);
// The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest
bIdle = false;
break;
}
}
if (bIdle)
{
// Don't use the CPU if we don't need.
Thread.Sleep(1000);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException(string.Format("PROC-{0} ", ProcessorIndex) + "ProcessNewReports: " + Ex, Ex);
}
TickStatic(Watcher);
}
});
}
/// <summary>
/// Returns the count of items in the SQS
/// </summary>
private int GetSQSCount()
{
try
{
var AttribRequest = new GetQueueAttributesRequest
{
QueueUrl = Config.Default.AWSSQSQueueUrl,
AttributeNames = new List <string>
{
"ApproximateNumberOfMessages"
}
};
var AttribResponse = SqsClient.GetQueueAttributes(AttribRequest);
return(AttribResponse.ApproximateNumberOfMessages);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("GetSQSCount: " + Ex.ToString());
}
return(0);
}
private bool TryDeleteRecordSQS(Message InMessage)
{
#if DEBUG
// Debug doesn't empty the queue - it's just reads records
return(true);
#else
try
{
var DeleteRequest = new DeleteMessageRequest
{
QueueUrl = Config.Default.AWSSQSQueueUrl,
ReceiptHandle = InMessage.ReceiptHandle
};
var DeleteResponse = SqsClient.DeleteMessage(DeleteRequest);
return(DeleteResponse.HttpStatusCode == HttpStatusCode.OK);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("TryDeleteRecordSQS: " + Ex.ToString());
}
return(false);
#endif
}
/// <summary>
/// A function to add a report to the database, and rename the relevant files.
/// Thread-safe.
/// </summary>
/// <param name="DirInfo">The DirectoryInfo of the report folder.</param>
/// <param name="NewContext">The generic crash context.</param>
/// <param name="LogFileName">The file name of the log file in the report.</param>
/// <param name="DumpFileName">The file name of the minidump in the report.</param>
/// <param name="VideoFileName">The file name of the video file in the report, or null if there isn't one.</param>
private bool AddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext, string LogFileName, string DumpFileName, string VideoFileName)
{
try
{
Stopwatch AddReportTime = Stopwatch.StartNew();
// Create an XML representation of a crash
string CrashDetails = CreateCrash(DirInfo, NewContext, VideoFileName != null, LogFileName != null, DumpFileName != null);
if (CrashDetails == "")
{
CrashReporterProcessServicer.WriteFailure("! NoDetail: Path=" + NewContext.CrashDirectory);
return(false);
}
// Upload the crash to the database, and retrieve the new row id
int ReportID = UploadCrash(CrashDetails);
if (ReportID <= 0)
{
CrashReporterProcessServicer.WriteFailure("! NoUpload: Path=" + NewContext.CrashDirectory);
return(false);
}
string IDThenUnderscore = string.Format("{0}_", ReportID);
// Use the row id to name and move the files the way the web site requires
string DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedReports, IDThenUnderscore);
// Move report files to crash reporter file store
if (LogFileName != null)
{
LogFileName = Path.Combine(DirInfo.FullName, LogFileName);
FileInfo LogInfo = new FileInfo(LogFileName);
if (LogInfo.Exists)
{
LogInfo.MoveTo(DestinationFolder + "Launch.log");
}
}
string CrashContextRuntimeName = Path.Combine(DirInfo.FullName, FGenericCrashContext.CrashContextRuntimeXMLName);
FileInfo CrashContextInfo = new FileInfo(CrashContextRuntimeName);
if (CrashContextInfo.Exists)
{
CrashContextInfo.MoveTo(DestinationFolder + FGenericCrashContext.CrashContextRuntimeXMLName);
}
// WERMetaDataName = Path.Combine(DirInfo.FullName, WERMetaDataName);
// FileInfo MetaInfo = new FileInfo(WERMetaDataName);
// if (MetaInfo.Exists)
// {
// MetaInfo.MoveTo(DestinationFolder + "WERMeta.xml");
// }
if (DumpFileName != null)
{
DumpFileName = Path.Combine(DirInfo.FullName, DumpFileName);
FileInfo DumpInfo = new FileInfo(DumpFileName);
if (DumpInfo.Exists && NewContext.PrimaryCrashProperties.CrashDumpMode != 1 /* ECrashDumpMode.FullDump = 1*/)
{
DumpInfo.MoveTo(DestinationFolder + "MiniDump.dmp");
}
}
// DiagnosticsFileName = Path.Combine(DirInfo.FullName, DiagnosticsFileName);
// FileInfo DiagnosticsInfo = new FileInfo(DiagnosticsFileName);
// if (DiagnosticsInfo.Exists)
// {
// DiagnosticsInfo.MoveTo(DestinationFolder + CrashReporterConstants.DiagnosticsFileName);
// }
// Move the video (if it exists) to an alternate store
if (VideoFileName != null)
{
DestinationFolder = Path.Combine(Properties.Settings.Default.ProcessedVideos, IDThenUnderscore);
VideoFileName = Path.Combine(DirInfo.FullName, VideoFileName);
FileInfo VideoInfo = new FileInfo(VideoFileName);
if (VideoInfo.Exists)
{
VideoInfo.MoveTo(DestinationFolder + CrashReporterConstants.VideoFileName);
}
}
CrashReporterProcessServicer.WriteEvent("# WebAdded: ReportID =" + string.Format("{0,7}", ReportID) + " Path=" + NewContext.CrashDirectory);
UpdateProcessedReports();
WebAddedReports++;
double Ratio = (double)WebAddedReports / (double)ProcessedReports * 100;
double AddedPerDay = (double)WebAddedReports / Timer.Elapsed.TotalDays;
CrashReporterProcessServicer.WriteEvent(string.Format("Ratio={0,2} Processed={1,7} WebAdded={2,7} AddReportTime={3} AddedPerDay={4}", (int)Ratio, ProcessedReports, WebAddedReports, AddReportTime.Elapsed.TotalSeconds.ToString("0.00"), (int)AddedPerDay));
return(true);
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("AddReport: " + DirInfo.Name + "\n\n" + Ex.ToString());
}
return(false);
}
double ReadProcessAddReport(DirectoryInfo DirInfo, FGenericCrashContext NewContext)
{
try
{
string DumpFileName = null;
string LogFileName = null;
string DiagnosticsFileName = "";
string VideoFileName = null;
string CrashContextName = "";
foreach (FileInfo Info in DirInfo.GetFiles())
{
switch (Info.Extension.ToLower())
{
case ".avi":
case ".mp4":
VideoFileName = Info.Name;
break;
case ".runtime-xml":
CrashContextName = Info.Name;
break;
case ".log":
LogFileName = Info.Name;
break;
case ".txt":
if (Info.Name == CrashReporterConstants.DiagnosticsFileName)
{
DiagnosticsFileName = Info.Name;
ReadDiagnosticsFile(NewContext);
}
break;
case ".dmp":
case ".mdmp":
DumpFileName = Info.Name;
// Check to see if this has been processed locally
FileInfo DiagnosticsInfo = new FileInfo(Path.Combine(DirInfo.FullName, CrashReporterConstants.DiagnosticsFileName));
if (!DiagnosticsInfo.Exists && !NewContext.HasProcessedData())
{
ProcessDumpFile(Info.FullName, NewContext);
}
// Check to see if a new one has been created
DiagnosticsInfo.Refresh();
if (DiagnosticsInfo.Exists)
{
DiagnosticsFileName = DiagnosticsInfo.Name;
}
break;
default:
break;
}
}
// Check if the new context has processed data.
if (NewContext.HasProcessedData())
{
Stopwatch WaitSW = Stopwatch.StartNew();
// Wait for previous task, should not really happen.
if (AddReportTask != null)
{
AddReportTask.Wait();
}
double WaitTime = WaitSW.Elapsed.TotalSeconds;
//bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName );
// Save/update crash context to the file.
NewContext.ToFile();
AddReportTask = Task.Factory.StartNew(() =>
{
bool bAdded = AddReport(DirInfo, NewContext, LogFileName, DumpFileName, VideoFileName);
FinalizeReport(bAdded, DirInfo, NewContext);
});
return(WaitTime);
}
else
{
CrashReporterProcessServicer.WriteFailure("! Invalid : BuiltFromCL=" + string.Format("{0,7}", NewContext.PrimaryCrashProperties.EngineVersion) + " Path=" + NewContext.CrashDirectory);
FinalizeReport(false, DirInfo, NewContext);
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("ProcessReport: " + NewContext.CrashDirectory + "\n\n: " + Ex.ToString());
}
return(0.0);
}
/// <summary>
/// Main processing thread.
/// </summary>
/// <remarks>All exceptions are caught and written to the event log.</remarks>
private void ProcessNewReports()
{
// Use the latest MinidumpDiagnostics from the main branch.
string Win64BinariesDirectory = Path.Combine(Properties.Settings.Default.DepotRoot, "UE4", "Engine", "Binaries", "Win64");
#if !DEBUG
string MinidumpDiagnosticsName = Path.Combine(Win64BinariesDirectory, "MinidumpDiagnostics*");
if (!SyncRequiredFiles(MinidumpDiagnosticsName))
{
return;
}
#endif
DateTime LastCleanupTime = DateTime.Now;
int LastDay = DateTime.UtcNow.Day;
var Cancel = CancelSource.Token;
ProcessorTask = Task.Factory.StartNew(() =>
{
while (!Cancel.IsCancellationRequested)
{
try
{
foreach (var Queue in Watcher.ReportQueues)
{
FGenericCrashContext NewContext = null;
if (Queue.TryDequeueReport(out NewContext))
{
ProcessReport(NewContext);
int CurrentDay = DateTime.UtcNow.Day;
if (CurrentDay > LastDay)
{
// Check the log and create a new one for a new day.
CrashReporterProcessServicer.Log.CreateNewLogFile();
LastDay = CurrentDay;
}
// The effect of this break is to prioritize ReportQueues by their order in the list, from highest to lowest
break;
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("ProcessNewReports: " + Ex.ToString());
}
// Remove all folders older than n days to prevent unbounded growth
if ((DateTime.Now - LastCleanupTime) > TimeSpan.FromMinutes(15))
{
CleanRepository();
LastCleanupTime = DateTime.Now;
}
// Don't use the CPU if we don't need.
Thread.Sleep(100);
}
});
}
/// <summary>
/// Look for new report folders and add them to the publicly available thread-safe queue.
/// </summary>
public override int CheckForNewReports()
{
try
{
if (QueueCount >= SkipQueueRefreshMin)
{
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: skipped busy queue size {0} in {1}", QueueCount, LandingZone));
return(LastQueueSizeOnDisk);
}
var NewReportPath = "";
var ReportsInLandingZone = new ReportIdSet();
// Check the landing zones for new reports
DirectoryInfo DirInfo = new DirectoryInfo(LandingZone);
// Couldn't find a landing zone, skip and try again later.
// Crash receiver will recreate them if needed.
if (!DirInfo.Exists)
{
CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone);
return(LastQueueSizeOnDisk);
}
var DirectoriesInLandingZone = DirInfo.GetDirectories().OrderBy(dirinfo => dirinfo.CreationTimeUtc).ToArray();
LastQueueSizeOnDisk = DirectoriesInLandingZone.Length;
int EnqueuedCount = 0;
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} reports in {1}", DirectoriesInLandingZone.Length, LandingZone));
// Add any new reports
for (int DirIndex = 0; DirIndex < DirectoriesInLandingZone.Length && QueueCount < QueueMax; DirIndex++)
{
var SubDirInfo = DirectoriesInLandingZone[DirIndex];
try
{
if (Directory.Exists(SubDirInfo.FullName))
{
NewReportPath = SubDirInfo.FullName;
ReportsInLandingZone.Add(NewReportPath);
if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath))
{
EnqueueNewReport(NewReportPath);
EnqueuedCount++;
}
}
}
catch (System.Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex.ToString());
ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks);
}
}
//CrashReporterProcessServicer.WriteEvent( string.Format( "ReportsInLandingZone={0}, ReportsInLandingZoneLastTimeWeChecked={1}", ReportsInLandingZone.Count, ReportsInLandingZoneLastTimeWeChecked.Count ) );
ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone;
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} enqueued to queue size {1} from {2}", EnqueuedCount, QueueCount, LandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: Enqueue rate {0:N1}/minute from {1}", EnqueueCounter.EventsPerSecond * 60, LandingZone));
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex.ToString());
}
return(LastQueueSizeOnDisk);
}
/// <summary>
/// Look for new report folders and add them to the publicly available thread-safe queue.
/// </summary>
public int CheckForNewReports()
{
try
{
if (QueueCount >= Config.Default.MinDesiredMemoryQueueSize)
{
CrashReporterProcessServicer.WriteEvent(string.Format(
"CheckForNewReports: {0} skipped busy queue size {1} in {2}", QueueName, QueueCount, LandingZone));
}
else
{
var NewReportPath = "";
var ReportsInLandingZone = new ReportIdSet();
CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(LandingZone, Config.Default.DiskSpaceAlertPercent);
DirectoryInfo[] DirectoriesInLandingZone;
if (GetCrashesFromLandingZone(out DirectoriesInLandingZone))
{
LastQueueSizeOnDisk = DirectoriesInLandingZone.Length;
int EnqueuedCount = 0;
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} reports in disk landing zone {1}",
DirectoriesInLandingZone.Length, LandingZone));
// Add any new reports
for (int DirIndex = 0; DirIndex < DirectoriesInLandingZone.Length && QueueCount < Config.Default.MaxMemoryQueueSize; DirIndex++)
{
var SubDirInfo = DirectoriesInLandingZone[DirIndex];
try
{
if (Directory.Exists(SubDirInfo.FullName))
{
NewReportPath = SubDirInfo.FullName;
ReportsInLandingZone.Add(NewReportPath);
if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath))
{
if (EnqueueNewReport(NewReportPath))
{
EnqueuedCount++;
}
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex, Ex);
ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks);
}
}
ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone;
CrashReporterProcessServicer.WriteEvent(string.Format(
"CheckForNewReports: {0} enqueued to queue size {1} from {2}", EnqueuedCount, QueueCount, LandingZone));
CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: Enqueue rate {0:N1}/minute from {1}",
EnqueueCounter.EventsPerSecond * 60, LandingZone));
}
else
{
LastQueueSizeOnDisk = 0;
}
}
}
catch (Exception Ex)
{
CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex, Ex);
}
return(GetTotalWaitingCount());
}