/// <summary>
/// Construct a processed callstack.
/// </summary>
/// <param name="CurrentCrash"></param>
public CallStackContainer(Crash CurrentCrash)
{
using (FAutoScopedLogTimer LogTimer = new FAutoScopedLogTimer(this.GetType().ToString() + "(CrashId=" + CurrentCrash.Id + ")"))
{
ParseCallStack(CurrentCrash);
}
}
//.*?\(.*?\).*?\[.*?\]
/// <summary>
/// Parse a raw callstack into a pattern
/// </summary>
/// <param name="CurrentCrash">The crash with a raw callstack to parse.</param>
private void ParseCallStack(Crash CurrentCrash)
{
// Everything is disabled by default
bDisplayUnformattedCallStack = false;
bDisplayModuleNames = false;
bDisplayFunctionNames = false;
bDisplayFileNames = false;
bDisplayFilePathNames = false;
bool bSkipping = false;
string LineToSkipUpto = "";
switch (CurrentCrash.CrashType)
{
case 2:
bSkipping = true;
LineToSkipUpto = "FDebug::AssertFailed";
break;
case 3:
bSkipping = true;
LineToSkipUpto = "FDebug::";
break;
}
if (string.IsNullOrEmpty(CurrentCrash.RawCallStack))
{
return;
}
CallStackEntries.Clear();
// Store off a pre split array of call stack lines
string[] RawCallStackLines = CurrentCrash.RawCallStack.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
int Middle = RawCallStackLines.Length / 2;
// Support older callstacks uploaded before UE4 upgrade
if (!NewCallstackFormat.Match(RawCallStackLines[Middle]).Success)
{
foreach (string CurrentLine in RawCallStackLines)
{
// Exit if we've hit the max number of lines we want
if (CallStackEntries.Count >= MaxLinesToParse)
{
break;
}
ParseUE3FormatCallstackLine(CurrentLine);
}
return;
}
foreach (string CurrentLine in RawCallStackLines)
{
// Exit if we've hit the max number of lines we want
if (CallStackEntries.Count >= MaxLinesToParse)
{
break;
}
if (bSkipping)
{
if (CurrentLine.Contains(LineToSkipUpto))
{
bSkipping = false;
}
}
if (bSkipping)
{
continue;
}
string ModuleName = "<Unknown>";
string FuncName = "<Unknown>";
string FilePath = "";
int LineNumber = 0;
//
// Generic sample line "UE4_Engine!UEngine::Exec() {+ 21105 bytes} [d:\depot\ue4\engine\source\runtime\engine\private\unrealengine.cpp:2777]"
//
// Mac
// thread_start() Address = 0x7fff87ae141d (filename not found) [in libsystem_pthread.dylib]
//
// Linux
// Unknown!AFortPlayerController::execServerSaveLoadoutData(FFrame&, void*) + some bytes
int ModuleSeparator = CurrentLine.IndexOf('!');
int PlusOffset = CurrentLine.IndexOf(" + ");
int OpenFuncSymbol = CurrentLine.IndexOf('(');
int CloseFuncSymbol = CurrentLine.IndexOf(')');
int OpenBracketOffset = CurrentLine.IndexOf('[');
int CloseBracketOffset = CurrentLine.LastIndexOf(']');
int MacModuleStart = CurrentLine.IndexOf("[in ");
int MacModuleEnd = MacModuleStart > 0 ? CurrentLine.IndexOf("]", MacModuleStart) : 0;
bool bLinux = CurrentCrash.PlatformName.Contains("Linux");
bool bMac = CurrentCrash.PlatformName.Contains("Mac");
bool bWindows = CurrentCrash.PlatformName.Contains("Windows");
// Parse out the juicy info from the line of the callstack
if (ModuleSeparator > 0)
{
ModuleName = CurrentLine.Substring(0, ModuleSeparator).Trim();
if (OpenFuncSymbol > ModuleSeparator && CloseFuncSymbol > OpenFuncSymbol)
{
// Grab the function name if it exists
FuncName = CurrentLine.Substring(ModuleSeparator + 1, OpenFuncSymbol - ModuleSeparator - 1).Trim();
FuncName += "()";
// Grab the source file
if (OpenBracketOffset > CloseFuncSymbol && CloseBracketOffset > OpenBracketOffset && (bWindows || bLinux))
{
string FileLinePath = CurrentLine.Substring(OpenBracketOffset + 1, CloseBracketOffset - OpenBracketOffset - 1).Trim();
FilePath = FileLinePath.TrimEnd("0123456789:".ToCharArray());
if (FileLinePath.Length > FilePath.Length + 1)
{
int SourceLine = 0;
if (int.TryParse(FileLinePath.Substring(FilePath.Length + 1), out SourceLine))
{
LineNumber = SourceLine;
}
}
}
}
}
else if (bWindows)
{
// Grab the module name if there is no function name
int WhiteSpacePos = CurrentLine.IndexOf(' ');
ModuleName = WhiteSpacePos > 0 ? CurrentLine.Substring(0, WhiteSpacePos) : CurrentLine;
}
if (bMac && MacModuleStart > 0 && MacModuleEnd > 0)
{
int AddressOffset = CurrentLine.IndexOf("Address =");
int OpenFuncSymbolMac = AddressOffset > 0 ? CurrentLine.Substring(0, AddressOffset).LastIndexOf('(') : 0;
if (OpenFuncSymbolMac > 0)
{
FuncName = CurrentLine.Substring(0, OpenFuncSymbolMac).Trim();
FuncName += "()";
}
ModuleName = CurrentLine.Substring(MacModuleStart + 3, MacModuleEnd - MacModuleStart - 3).Trim();
}
// Remove callstack entries that match any of these functions.
var FuncsToRemove = new HashSet <string>(new string[]
{
"RaiseException",
"FDebug::",
"Error::Serialize",
"FOutputDevice::Logf",
"FMsg::Logf",
"ReportCrash",
"NewReportEnsure",
"EngineCrashHandler",
"FLinuxPlatformStackWalk::CaptureStackBackTrac",
"FGenericPlatformStackWalk::StackWalkAndDump",
"FLinuxCrashContext::CaptureStackTrace",
"CommonLinuxCrashHandler",
// Generic crash handler for all platforms
});
bool Contains = FuncsToRemove.Contains(FuncName, new CustomFuncComparer());
if (!Contains)
{
CallStackEntries.Add(new CallStackEntry(CurrentLine, ModuleName, FilePath, FuncName, LineNumber));
}
}
}
/// <summary>
/// Create call stack pattern and either insert into database or match to existing.
/// Update Associate Buggs.
/// </summary>
/// <param name="newCrash"></param>
private void BuildPattern(Crash newCrash)
{
var callstack = new CallStackContainer(newCrash);
newCrash.Module = callstack.GetModuleName();
if (newCrash.PatternId == null)
{
var patternList = new List<string>();
try
{
foreach (var entry in callstack.CallStackEntries.Take(CallStackContainer.MaxLinesToParse))
{
FunctionCall currentFunctionCall;
var csEntry = entry;
if (_unitOfWork.FunctionRepository.Any(f => f.Call == csEntry.FunctionName))
{
currentFunctionCall = _unitOfWork.FunctionRepository.First(f => f.Call == csEntry.FunctionName);
}
else
{
currentFunctionCall = new FunctionCall { Call = csEntry.FunctionName };
_unitOfWork.FunctionRepository.Save(currentFunctionCall);
_unitOfWork.Save();
}
patternList.Add(currentFunctionCall.Id.ToString());
}
newCrash.Pattern = string.Join("+", patternList);
}
catch (Exception ex)
{
var messageBuilder = new StringBuilder();
FLogger.Global.WriteException("Build Pattern exception: " + ex.Message.ToString(CultureInfo.InvariantCulture));
messageBuilder.AppendLine("Exception was:");
messageBuilder.AppendLine(ex.ToString());
while (ex.InnerException != null)
{
ex = ex.InnerException;
messageBuilder.AppendLine(ex.ToString());
}
_slackWriter.Write("Build Pattern Exception : " + ex.Message.ToString(CultureInfo.InvariantCulture));
throw;
}
}
}
/// <summary>
/// Create a new crash data model object and insert it into the database
/// </summary>
/// <param name="description"></param>
/// <returns></returns>
private Crash CreateCrash(CrashDescription description)
{
var newCrash = new Crash
{
Branch = description.BranchName,
BaseDir = description.BaseDir,
BuildVersion = description.EngineVersion,
EngineVersion = description.BuildVersion,
ChangeListVersion = description.BuiltFromCL.ToString(),
CommandLine = description.CommandLine,
EngineMode = description.EngineMode,
ComputerName = description.MachineGuid
};
//if there's a valid username assign the associated UserNameId else use "anonymous".
var userName = (!string.IsNullOrEmpty(description.UserName)) ? description.UserName : UserNameAnonymous;
var user = _unitOfWork.UserRepository.GetByUserName(userName);
if (user != null)
newCrash.UserNameId = user.Id;
else
{
newCrash.User = new User() {UserName = description.UserName, UserGroupId = 5};
}
//If there's a valid EpicAccountId assign that.
if (!string.IsNullOrEmpty(description.EpicAccountId))
{
newCrash.EpicAccountId = description.EpicAccountId;
}
newCrash.Description = "";
if (description.UserDescription != null)
{
newCrash.Description = string.Join(Environment.NewLine, description.UserDescription);
}
newCrash.EngineMode = description.EngineMode;
newCrash.GameName = description.GameName;
newCrash.LanguageExt = description.Language; //Converted by the crash process.
newCrash.PlatformName = description.Platform;
if (description.ErrorMessage != null)
{
newCrash.Summary = string.Join("\n", description.ErrorMessage);
}
if (description.CallStack != null)
{
newCrash.RawCallStack = string.Join("\n", description.CallStack);
}
if (description.SourceContext != null)
{
newCrash.SourceContext = string.Join("\n", description.SourceContext);
}
newCrash.TimeOfCrash = description.TimeofCrash;
newCrash.Processed = description.bAllowToBeContacted;
newCrash.Jira = "";
newCrash.FixedChangeList = "";
newCrash.ProcessFailed = description.bProcessorFailed;
// Set the crash type
newCrash.CrashType = 1;
//if we have a crash type set the crash type
if (!string.IsNullOrEmpty(description.CrashType))
{
switch (description.CrashType.ToLower())
{
case "crash":
newCrash.CrashType = 1;
break;
case "assert":
newCrash.CrashType = 2;
break;
case "ensure":
newCrash.CrashType = 3;
break;
case "":
case null:
default:
newCrash.CrashType = 1;
break;
}
}
else //else fall back to the old behavior and try to determine type from RawCallStack
{
if (newCrash.RawCallStack != null)
{
if (newCrash.RawCallStack.Contains("FDebug::AssertFailed"))
{
newCrash.CrashType = 2;
}
else if (newCrash.RawCallStack.Contains("FDebug::Ensure"))
{
newCrash.CrashType = 3;
}
else if (newCrash.RawCallStack.Contains("FDebug::OptionallyLogFormattedEnsureMessageReturningFalse"))
{
newCrash.CrashType = 3;
}
else if (newCrash.RawCallStack.Contains("NewReportEnsure"))
{
newCrash.CrashType = 3;
}
}
}
// As we're adding it, the status is always new
newCrash.Status = "New";
/*
Unused Crashes' fields.
Title nchar(20)
Selected bit
Version int
AutoReporterID int
Processed bit -> renamed to AllowToBeContacted
HasDiagnosticsFile bit always true
HasNewLogFile bit
HasMetaData bit always true
*/
// Set the unused fields to the default values.
//NewCrash.Title = ""; removed from dbml
//NewCrash.Selected = false; removed from dbml
//NewCrash.Version = 4; removed from dbml
//NewCrash.AutoReporterID = 0; removed from dbml
//NewCrash.HasNewLogFile = false;removed from dbml
//NewCrash.HasDiagnosticsFile = true;
//NewCrash.HasMetaData = true;
newCrash.UserActivityHint = description.UserActivityHint;
BuildPattern(newCrash);
if(newCrash.CommandLine == null)
newCrash.CommandLine = "";
_unitOfWork.Dispose();
_unitOfWork = new UnitOfWork(new CrashReportEntities());
var callStackRepository = _unitOfWork.CallstackRepository;
try
{
var crashRepo = _unitOfWork.CrashRepository;
//if we don't have any callstack data then insert the crash and return
if (string.IsNullOrEmpty(newCrash.Pattern))
{
crashRepo.Save(newCrash);
_unitOfWork.Save();
return newCrash;
}
//If this isn't a new pattern then link it to our crash data model
if (callStackRepository.Any(data => data.Pattern == newCrash.Pattern))
{
var callstackPattern = callStackRepository.First(data => data.Pattern == newCrash.Pattern);
newCrash.PatternId = callstackPattern.id;
}
else
{
//if this is a new callstack pattern then insert into data model and create a new bugg.
var callstackPattern = new CallStackPattern { Pattern = newCrash.Pattern };
callStackRepository.Save(callstackPattern);
_unitOfWork.Save();
newCrash.PatternId = callstackPattern.id;
}
//Mask out the line number and File path from our error message.
var errorMessageString = description.ErrorMessage != null ? String.Join("", description.ErrorMessage) : "";
//Create our masking regular expressions
var fileRegex = new Regex(@"(\[File:).*?(])");//Match the filename out the file name
var lineRegex = new Regex(@"(\[Line:).*?(])");//Match the line no.
/**
* Regex to match ints of two characters or longer
* First term ((?<=\s)|(-)) : Positive look behind, match if preceeded by whitespace or if first character is '-'
* Second term (\d{3,}) match three or more decimal chracters in a row.
* Third term (?=(\s|$)) positive look ahead match if followed by whitespace or end of line/file.
*/
var intRegex = new Regex(@"-?\d{3,}");
/**
* Regular expression for masking out floats
*/
var floatRegex = new Regex(@"-?\d+\.\d+");
/**
* Regular expression for masking out hexadecimal numbers
*/
var hexRegex = new Regex(@"0x[\da-fA-F]+");
//mask out terms matches by our regex's
var trimmedError = fileRegex.Replace(errorMessageString, "");
trimmedError = lineRegex.Replace(trimmedError, "");
trimmedError = floatRegex.Replace(trimmedError, "");
trimmedError = hexRegex.Replace(trimmedError, "");
trimmedError = intRegex.Replace(trimmedError, "");
//Check to see if the masked error message is unique
ErrorMessage errorMessage = null;
if (_unitOfWork.ErrorMessageRepository.Any(data => data.Message.Contains(trimmedError)))
{
errorMessage =
_unitOfWork.ErrorMessageRepository.First(data => data.Message.Contains(trimmedError));
}
else
{
//if it's a new message then add it to the database.
errorMessage = new ErrorMessage() { Message = trimmedError };
_unitOfWork.ErrorMessageRepository.Save(errorMessage);
_unitOfWork.Save();
}
//Check for an existing bugg with this pattern and error message / no error message
if ( _unitOfWork.BuggRepository.Any(data =>
(data.PatternId == newCrash.PatternId || data.Pattern == newCrash.Pattern) && (newCrash.CrashType == 3 ||
(data.ErrorMessageId == errorMessage.Id || data.ErrorMessageId == null)) ))
{
//if a bugg exists for this pattern update the bugg data
var bugg = _unitOfWork.BuggRepository.First(data => data.PatternId == newCrash.PatternId) ??
_unitOfWork.BuggRepository.First(data => data.Pattern == newCrash.Pattern);
bugg.PatternId = newCrash.PatternId;
if (newCrash.CrashType != 3)
{
bugg.CrashType = newCrash.CrashType;
bugg.ErrorMessageId = errorMessage.Id;
}
//also update the bugg data while we're here
bugg.TimeOfLastCrash = newCrash.TimeOfCrash;
if (String.Compare(newCrash.BuildVersion, bugg.BuildVersion,
StringComparison.Ordinal) != 1)
bugg.BuildVersion = newCrash.BuildVersion;
_unitOfWork.Save();
//if a bugg exists update this crash from the bugg
//buggs are authoritative in this case
newCrash.Buggs.Add(bugg);
newCrash.Jira = bugg.TTPID;
newCrash.FixedChangeList = bugg.FixedChangeList;
newCrash.Status = bugg.Status;
_unitOfWork.CrashRepository.Save(newCrash);
_unitOfWork.Save();
}
else
{
//if there's no bugg for this pattern create a new bugg and insert into the data store.
var bugg = new Bugg();
bugg.TimeOfFirstCrash = newCrash.TimeOfCrash;
bugg.TimeOfLastCrash = newCrash.TimeOfCrash;
bugg.TTPID = newCrash.Jira;
bugg.Pattern = newCrash.Pattern;
bugg.PatternId = newCrash.PatternId;
bugg.NumberOfCrashes = 1;
bugg.NumberOfUsers = 1;
bugg.NumberOfUniqueMachines = 1;
bugg.BuildVersion = newCrash.BuildVersion;
bugg.CrashType = newCrash.CrashType;
bugg.Status = newCrash.Status;
bugg.FixedChangeList = newCrash.FixedChangeList;
bugg.ErrorMessageId = errorMessage.Id;
newCrash.Buggs.Add(bugg);
_unitOfWork.BuggRepository.Save(bugg);
_unitOfWork.CrashRepository.Save(newCrash);
_unitOfWork.Save();
}
}
catch (DbEntityValidationException dbentEx)
{
var messageBuilder = new StringBuilder();
messageBuilder.AppendLine("Db Entity Validation Exception Exception was:");
messageBuilder.AppendLine(dbentEx.ToString());
var innerEx = dbentEx.InnerException;
while (innerEx != null)
{
messageBuilder.AppendLine("Inner Exception : " + innerEx.Message);
innerEx = innerEx.InnerException;
}
if (dbentEx.EntityValidationErrors != null)
{
messageBuilder.AppendLine("Validation Errors : ");
foreach (var valErr in dbentEx.EntityValidationErrors)
{
messageBuilder.AppendLine(valErr.ValidationErrors.Select(data => data.ErrorMessage).Aggregate((current, next) => current + "; /n" + next));
}
}
FLogger.Global.WriteException(messageBuilder.ToString());
}
catch (Exception ex)
{
var messageBuilder = new StringBuilder();
messageBuilder.AppendLine("Create Crash Exception : ");
messageBuilder.AppendLine(ex.Message.ToString());
var innerEx = ex.InnerException;
while (innerEx != null)
{
messageBuilder.AppendLine("Inner Exception : " + innerEx.Message);
innerEx = innerEx.InnerException;
}
FLogger.Global.WriteException("Create Crash Exception : " +
messageBuilder.ToString());
_slackWriter.Write("Create Crash Exception : " + messageBuilder.ToString());
throw;
}
return newCrash;
}
