/// <summary>
/// Returns a hash that represents the failure results of this test. If the test failed this should be an empty string
/// </summary>
/// <returns></returns>
protected virtual string GetTestResultHash()
{
IEnumerable <string> RoleHashes = SessionArtifacts.Select(A => GetRoleResultHash(A)).OrderBy(S => S);
RoleHashes = RoleHashes.Where(S => S.Length > 0 && S != "0");
string Combined = string.Join("\n", RoleHashes);
string CombinedHash = Hasher.ComputeHash(Combined);
return(CombinedHash);
}
/// <summary>
/// Returns Warnings found during tests. By default only ensures are considered
/// </summary>
public override IEnumerable <string> GetWarnings()
{
if (SessionArtifacts == null)
{
return(new string[0]);
}
return(SessionArtifacts.SelectMany(A =>
{
return A.LogSummary.Ensures.Select(E => E.Message);
}));
}
/// <summary>
/// Returns a summary of this test
/// </summary>
/// <returns></returns>
public override string GetTestSummary()
{
// Handle case where there aren't any session artifacts, for example with device starvation
if (SessionArtifacts == null)
{
return("NoSummary");
}
MarkdownBuilder ReportBuilder = new MarkdownBuilder();
// add header
ReportBuilder.Append(GetTestSummaryHeader());
ReportBuilder.Append("--------");
StringBuilder SB = new StringBuilder();
// Get any artifacts with failures
var FailureArtifacts = GetArtifactsWithFailures();
// Any with warnings (ensures)
var WarningArtifacts = SessionArtifacts.Where(A => A.LogSummary.Ensures.Count() > 0);
// combine artifacts into order as Failures, Warnings, Other
var AllArtifacts = FailureArtifacts.Union(WarningArtifacts);
AllArtifacts = AllArtifacts.Union(SessionArtifacts);
// Add a summary of each
foreach (var Artifact in AllArtifacts)
{
string Summary = "NoSummary";
int ExitCode = GetRoleSummary(Artifact, out Summary);
if (SB.Length > 0)
{
SB.AppendLine();
}
SB.Append(Summary);
}
ReportBuilder.Append(SB.ToString());
return(ReportBuilder.ToString());
}
/// <summary>
/// Return all artifacts that are considered to have caused the test to fail
/// </summary>
/// <returns></returns>
protected virtual IEnumerable <UnrealRoleArtifacts> GetArtifactsWithFailures()
{
if (SessionArtifacts == null)
{
Log.Warning("SessionArtifacts was null, unable to check for failures");
return(new UnrealRoleArtifacts[0] {
});
}
bool DidKillClients = SessionArtifacts.Any(A => A.SessionRole.RoleType.IsClient() && A.AppInstance.WasKilled);
Dictionary <UnrealRoleArtifacts, int> ErrorCodes = new Dictionary <UnrealRoleArtifacts, int>();
var FailureList = SessionArtifacts.Where(A =>
{
// ignore anything we killed
if (A.AppInstance.WasKilled)
{
return(false);
}
string ExitReason;
int ExitCode = GetExitCodeAndReason(A, out ExitReason);
ErrorCodes.Add(A, ExitCode);
return(ExitCode != 0);
});
return(FailureList.OrderByDescending(A =>
{
int Score = 0;
if (A.LogSummary.FatalError != null || (ErrorCodes[A] != 0 && A.AppInstance.WasKilled == false))
{
Score += 100000;
}
Score += A.LogSummary.Ensures.Count();
return Score;
}).ToList());
}
/// <summary>
/// Override the summary report so we can insert a link to our report and the failed tests
/// </summary>
/// <returns></returns>
protected override string GetTestSummaryHeader()
{
MarkdownBuilder MB = new MarkdownBuilder(base.GetTestSummaryHeader());
string ReportLink = GetConfiguration().ReportOutputURL;
if (string.IsNullOrEmpty(ReportLink) == false)
{
MB.Paragraph(string.Format("Report: {0}", ReportLink));
}
var EditorArtifacts = SessionArtifacts.Where(A => A.SessionRole.RoleType == UnrealTargetRole.Editor).FirstOrDefault();
if (EditorArtifacts != null)
{
AutomationLogParser Parser = new AutomationLogParser(EditorArtifacts.LogParser);
IEnumerable <AutomationTestResult> AllTests = Parser.GetResults();
IEnumerable <AutomationTestResult> FailedTests = AllTests.Where(R => !R.Passed);
MB.Paragraph(string.Format("{0} of {1} tests passed", AllTests.Count() - FailedTests.Count(), AllTests.Count()));
if (FailedTests.Count() > 0)
{
MB.H3("Test Failures");
foreach (AutomationTestResult Result in FailedTests)
{
MB.H4(string.Format("{0} Failed", Result.DisplayName));
MB.UnorderedList(Result.Events);
}
}
}
return(MB.ToString());
}
/// <summary>
/// Parses the provided artifacts to determine the cause of an exit and whether it was abnormal
/// </summary>
/// <param name="InArtifacts"></param>
/// <param name="Reason"></param>
/// <param name="WasAbnormal"></param>
/// <returns></returns>
protected virtual int GetExitCodeAndReason(UnrealRoleArtifacts InArtifacts, out string ExitReason)
{
UnrealLogParser.LogSummary LogSummary = InArtifacts.LogSummary;
// Assume failure!
int ExitCode = -1;
ExitReason = "Unknown";
if (LogSummary.FatalError != null)
{
ExitReason = "Process encountered fatal error";
}
else if (LogSummary.Ensures.Count() > 0 && CachedConfig.FailOnEnsures)
{
ExitReason = string.Format("Process encountered {0} Ensures", LogSummary.Ensures.Count());
}
else if (InArtifacts.AppInstance.WasKilled)
{
ExitReason = "Process was killed";
}
else if (LogSummary.HasTestExitCode)
{
if (LogSummary.TestExitCode == 0)
{
ExitReason = "Process exited with code 0";
}
else
{
ExitReason = string.Format("Process exited with error code {0}", LogSummary.TestExitCode);
}
ExitCode = LogSummary.TestExitCode;
}
else if (LogSummary.RequestedExit)
{
ExitReason = string.Format("Process requested exit with no fatal errors");
ExitCode = 0;
}
else
{
// ok, process appears to have exited for no good reason so try to divine a result...
if (LogSummary.HasTestExitCode == false &&
InArtifacts.SessionRole.CommandLine.ToLower().Contains("-gauntlet"))
{
Log.Verbose("Role {0} had 0 exit code but used Gauntlet and no TestExitCode was found. Assuming failure", InArtifacts.SessionRole.RoleType);
ExitCode = -1;
ExitReason = "No test result from Gauntlet controller";
}
}
// Normal exits from server are not ok if we had clients running!
if (ExitCode == 0 && InArtifacts.SessionRole.RoleType.IsServer())
{
bool ClientsKilled = SessionArtifacts.Any(A => A.AppInstance.WasKilled && A.SessionRole.RoleType.IsClient());
if (ClientsKilled)
{
ExitCode = -1;
ExitReason = "Server exited while clients were running";
}
}
if (ExitCode == -1 && string.IsNullOrEmpty(ExitReason))
{
ExitReason = "Process exited with no indication of success";
}
return(ExitCode);
}
/// <summary>
/// Returns a summary of this test
/// </summary>
/// <returns></returns>
public override string GetTestSummary()
{
int AbnormalExits = 0;
int FatalErrors = 0;
int Ensures = 0;
int Errors = 0;
int Warnings = 0;
// Handle case where there aren't any session artifacts, for example with device starvation
if (SessionArtifacts == null)
{
return("NoSummary");
}
StringBuilder SB = new StringBuilder();
// Get any artifacts with failures
var FailureArtifacts = GetArtifactsWithFailures();
// Any with warnings (ensures)
var WarningArtifacts = SessionArtifacts.Where(A => A.LogSummary.Ensures.Count() > 0);
// combine artifacts into order as Failures, Warnings, Other
var AllArtifacts = FailureArtifacts.Union(WarningArtifacts);
AllArtifacts = AllArtifacts.Union(SessionArtifacts);
// create a quicck summary of total failures, ensures, errors, etc
foreach (var Artifact in AllArtifacts)
{
string Summary = "NoSummary";
int ExitCode = GetRoleSummary(Artifact, out Summary);
if (ExitCode != 0 && Artifact.AppInstance.WasKilled == false)
{
AbnormalExits++;
}
if (SB.Length > 0)
{
SB.AppendLine();
}
SB.Append(Summary);
FatalErrors += Artifact.LogSummary.FatalError != null ? 1 : 0;
Ensures += Artifact.LogSummary.Ensures.Count();
Errors += Artifact.LogSummary.Errors.Count();
Warnings += Artifact.LogSummary.Warnings.Count();
}
MarkdownBuilder MB = new MarkdownBuilder();
string WarningStatement = HasWarnings ? " With Warnings" : "";
// Create a summary
MB.H2(string.Format("{0} {1}{2}", Name, GetTestResult(), WarningStatement));
if (GetTestResult() != TestResult.Passed)
{
if (FailureArtifacts.Count() > 0)
{
foreach (var FailedArtifact in FailureArtifacts)
{
string FirstProcessCause = "";
int FirstExitCode = GetExitCodeAndReason(FailedArtifact, out FirstProcessCause);
MB.H3(string.Format("{0}: {1}", FailedArtifact.SessionRole.RoleType, FirstProcessCause));
if (FailedArtifact.LogSummary.FatalError != null)
{
MB.H4(FailedArtifact.LogSummary.FatalError.Message);
}
}
MB.Paragraph("See below for callstack and logs");
}
}
MB.Paragraph(string.Format("Context: {0}", Context.ToString()));
MB.Paragraph(string.Format("FatalErrors: {0}, Ensures: {1}, Errors: {2}, Warnings: {3}", FatalErrors, Ensures, Errors, Warnings));
MB.Paragraph(string.Format("ResultHash: {0}", GetTestResultHash()));
//MB.Paragraph(string.Format("Artifacts: {0}", CachedArtifactPath));
MB.Append("--------");
MB.Append(SB.ToString());
//SB.Clear();
//SB.AppendLine("begin: stack for UAT");
//SB.Append(MB.ToString());
//SB.Append("end: stack for UAT");
return(MB.ToString());
}
