/// <summary>
/// Reports the summary of this pass, including a sumamry for each test
/// </summary>
/// <param name="CurrentPass"></param>
/// <param name="NumPasses"></param>
/// <param name="Duration"></param>
/// <param name="AllInfo"></param>
/// <returns></returns>
void ReportMasterSummary(int CurrentPass, int NumPasses, TimeSpan Duration, IEnumerable <TestExecutionInfo> AllInfo)
{
MarkdownBuilder MB = new MarkdownBuilder();
int TestCount = AllInfo.Count();
int FailedCount = AllInfo.Where(T => T.FinalResult != TestResult.Passed).Count();
int WarningCount = AllInfo.Where(T => T.TestNode.HasWarnings).Count();
var SortedInfo = AllInfo;
// sort our tests by failed/warning/ok
if (FailedCount > 0 || WarningCount > 0)
{
SortedInfo = AllInfo.OrderByDescending(T =>
{
if (T.FinalResult != TestResult.Passed)
{
return(10);
}
if (T.TestNode.HasWarnings)
{
return(5);
}
return(0);
});
}
Log.Info("Completed test pass {0} of {1}.", CurrentPass, NumPasses);
MB.H2(string.Format("{0} of {1} Tests Passed in {2:mm\\:ss}. ({3} Failed, {4} Passed with Warnings)",
TestCount - FailedCount, TestCount, Duration, FailedCount, WarningCount));
// write out a list of tests and results
List <string> TestResults = new List <string>();
foreach (TestExecutionInfo Info in SortedInfo)
{
string WarningString = Info.TestNode.HasWarnings ? " With Warnings" : "";
TestResults.Add(string.Format("\t{0} result={1}{2}", Info, Info.FinalResult, WarningString));
}
MB.UnorderedList(TestResults);
string Summary = string.Format("Completed test pass {0} of {1}.", CurrentPass, NumPasses);
if (FailedCount > 0)
{
Log.Error("{0}", Summary);
}
else if (WarningCount > 0)
{
Log.Warning("{0}", Summary);
}
else
{
Log.Info("{0}", Summary);
}
// write the markdown out with each line indented
MB.ToString().Split('\n').ToList().ForEach(L => Log.Info(" " + L));
if (Options.DeferReports)
{
// write each tests full summary
foreach (TestExecutionInfo Info in SortedInfo)
{
ReportTestSummary(Info);
}
}
}
/// <summary>
/// Returns a summary of this test
/// </summary>
/// <returns></returns>
public override string GetTestSummary()
{
int AbnormalExits = 0;
int FatalErrors = 0;
int Ensures = 0;
int Errors = 0;
int Warnings = 0;
StringBuilder SB = new StringBuilder();
// Sort our artifacts so any missing processes are first
var ProblemArtifacts = GetArtifactsWithFailures();
var AllArtifacts = ProblemArtifacts.Union(SessionArtifacts);
// create a quicck summary of total failures, ensures, errors, etc
foreach (var Artifact in AllArtifacts)
{
string Summary = "NoSummary";
int ExitCode = GetRoleSummary(Artifact, out Summary);
if (ExitCode != 0 && Artifact.AppInstance.WasKilled == false)
{
AbnormalExits++;
}
if (SB.Length > 0)
{
SB.AppendLine();
}
SB.Append(Summary);
FatalErrors += Artifact.LogSummary.FatalError != null ? 1 : 0;
Ensures += Artifact.LogSummary.Ensures.Count();
Errors += Artifact.LogSummary.Errors.Count();
Warnings += Artifact.LogSummary.Warnings.Count();
}
MarkdownBuilder MB = new MarkdownBuilder();
// Create a summary
MB.H2(string.Format("{0} {1}", Name, GetTestResult()));
if (GetTestResult() != TestResult.Passed)
{
if (ProblemArtifacts.Count() > 0)
{
foreach (var FailedArtifact in ProblemArtifacts)
{
string FirstProcessCause = "";
int FirstExitCode = GetExitCodeAndReason(FailedArtifact, out FirstProcessCause);
MB.H3(string.Format("{0}: {1}", FailedArtifact.SessionRole.RoleType, FirstProcessCause));
if (FailedArtifact.LogSummary.FatalError != null)
{
MB.H4(FailedArtifact.LogSummary.FatalError.Message);
}
}
MB.Paragraph("See below for callstack and logs");
}
}
MB.Paragraph(string.Format("Context: {0}", Context.ToString()));
MB.Paragraph(string.Format("FatalErrors: {0}, Ensures: {1}, Errors: {2}, Warnings: {3}", FatalErrors, Ensures, Errors, Warnings));
//MB.Paragraph(string.Format("Artifacts: {0}", CachedArtifactPath));
MB.Append("--------");
MB.Append(SB.ToString());
//SB.Clear();
//SB.AppendLine("begin: stack for UAT");
//SB.Append(MB.ToString());
//SB.Append("end: stack for UAT");
return(MB.ToString());
}
/// <summary>
/// Parses the output of an application to try and determine a failure cause (if one exists). Returns
/// 0 for graceful shutdown
/// </summary>
/// <param name="Prefix"></param>
/// <param name="App"></param>
/// <returns></returns>
protected virtual int GetRoleSummary(UnrealRoleArtifacts InArtifacts, out string Summary)
{
const int MaxLogLines = 10;
const int MaxCallstackLines = 20;
UnrealLogParser.LogSummary LogSummary = InArtifacts.LogSummary;
string ExitReason = "Unknown";
int ExitCode = GetExitCodeAndReason(InArtifacts, out ExitReason);
MarkdownBuilder MB = new MarkdownBuilder();
MB.H3(string.Format("Role: {0} ({1} {2})", InArtifacts.SessionRole.RoleType, InArtifacts.SessionRole.Platform, InArtifacts.SessionRole.Configuration));
if (ExitCode != 0)
{
MB.H4(string.Format("Result: Abnormal Exit: {0}", ExitReason));
}
else
{
MB.H4(string.Format("Result: {0}", ExitReason));
}
int FatalErrors = LogSummary.FatalError != null ? 1 : 0;
if (LogSummary.FatalError != null)
{
MB.H4(string.Format("Fatal Error: {0}", LogSummary.FatalError.Message));
MB.UnorderedList(InArtifacts.LogSummary.FatalError.Callstack.Take(MaxCallstackLines));
if (InArtifacts.LogSummary.FatalError.Callstack.Count() > MaxCallstackLines)
{
MB.Paragraph("See log for full callstack");
}
}
if (LogSummary.Ensures.Count() > 0)
{
foreach (var Ensure in LogSummary.Ensures)
{
MB.H4(string.Format("Ensure: {0}", Ensure.Message));
MB.UnorderedList(Ensure.Callstack.Take(MaxCallstackLines));
if (Ensure.Callstack.Count() > MaxCallstackLines)
{
MB.Paragraph("See log for full callstack");
}
}
}
MB.Paragraph(string.Format("FatalErrors: {0}, Ensures: {1}, Errors: {2}, Warnings: {3}",
FatalErrors, LogSummary.Ensures.Count(), LogSummary.Errors.Count(), LogSummary.Warnings.Count()));
if (GetConfiguration().ShowErrorsInSummary&& InArtifacts.LogSummary.Errors.Count() > 0)
{
MB.H4("Errors");
MB.UnorderedList(LogSummary.Errors.Take(MaxLogLines));
if (LogSummary.Errors.Count() > MaxLogLines)
{
MB.Paragraph(string.Format("(First {0} of {1} errors)", MaxLogLines, LogSummary.Errors.Count()));
}
}
if (GetConfiguration().ShowWarningsInSummary&& InArtifacts.LogSummary.Warnings.Count() > 0)
{
MB.H4("Warnings");
MB.UnorderedList(LogSummary.Warnings.Take(MaxLogLines));
if (LogSummary.Warnings.Count() > MaxLogLines)
{
MB.Paragraph(string.Format("(First {0} of {1} warnings)", MaxLogLines, LogSummary.Warnings.Count()));
}
}
MB.H4("Artifacts");
MB.Paragraph(string.Format("Log: {0}", InArtifacts.LogPath));
MB.Paragraph(string.Format("Commandline: {0}", InArtifacts.AppInstance.CommandLine));
MB.Paragraph(InArtifacts.ArtifactPath);
Summary = MB.ToString();
return(ExitCode);
}
/// <summary>
/// Return the header for the test summary. The header is the first block of text and will be
/// followed by the summary of each individual role in the test
/// </summary>
/// <returns></returns>
protected virtual string GetTestSummaryHeader()
{
int AbnormalExits = 0;
int FatalErrors = 0;
int Ensures = 0;
int Errors = 0;
int Warnings = 0;
// create a quicck summary of total failures, ensures, errors, etc
foreach (var Artifact in SessionArtifacts)
{
string Summary;
int ExitCode = GetRoleSummary(Artifact, out Summary);
if (ExitCode != 0 && Artifact.AppInstance.WasKilled == false)
{
AbnormalExits++;
}
FatalErrors += Artifact.LogSummary.FatalError != null ? 1 : 0;
Ensures += Artifact.LogSummary.Ensures.Count();
Errors += Artifact.LogSummary.Errors.Count();
Warnings += Artifact.LogSummary.Warnings.Count();
}
MarkdownBuilder MB = new MarkdownBuilder();
string WarningStatement = HasWarnings ? " With Warnings" : "";
var FailureArtifacts = GetArtifactsWithFailures();
// Create a summary
MB.H2(string.Format("{0} {1}{2}", Name, GetTestResult(), WarningStatement));
if (GetTestResult() != TestResult.Passed)
{
if (FailureArtifacts.Count() > 0)
{
foreach (var FailedArtifact in FailureArtifacts)
{
string FirstProcessCause = "";
int FirstExitCode = GetExitCodeAndReason(FailedArtifact, out FirstProcessCause);
MB.H3(string.Format("{0}: {1}", FailedArtifact.SessionRole.RoleType, FirstProcessCause));
if (FailedArtifact.LogSummary.FatalError != null)
{
MB.H4(FailedArtifact.LogSummary.FatalError.Message);
}
}
MB.Paragraph("See below for logs and any callstacks");
}
}
MB.Paragraph(string.Format("Context: {0}", Context.ToString()));
MB.Paragraph(string.Format("FatalErrors: {0}, Ensures: {1}, Errors: {2}, Warnings: {3}", FatalErrors, Ensures, Errors, Warnings));
MB.Paragraph(string.Format("ResultHash: {0}", GetTestResultHash()));
MB.Paragraph(string.Format("Result: {0}", GetTestResult()));
//MB.Paragraph(string.Format("Artifacts: {0}", CachedArtifactPath));
return(MB.ToString());
}
/// <summary>
/// Append the provided Markdown to our body
/// </summary>
/// <param name="RHS"></param>
/// <returns></returns>
public MarkdownBuilder Append(MarkdownBuilder RHS)
{
EnsureEndsWithNewLine();
SB.Append(RHS.ToString());
return(this);
}
/// <summary>
/// Returns a summary of this test
/// </summary>
/// <returns></returns>
public override string GetTestSummary()
{
int AbnormalExits = 0;
int FatalErrors = 0;
int Ensures = 0;
int Errors = 0;
int Warnings = 0;
// Handle case where there aren't any session artifacts, for example with device starvation
if (SessionArtifacts == null)
{
return("NoSummary");
}
StringBuilder SB = new StringBuilder();
// Get any artifacts with failures
var FailureArtifacts = GetArtifactsWithFailures();
// Any with warnings (ensures)
var WarningArtifacts = SessionArtifacts.Where(A => A.LogSummary.Ensures.Count() > 0);
// combine artifacts into order as Failures, Warnings, Other
var AllArtifacts = FailureArtifacts.Union(WarningArtifacts);
AllArtifacts = AllArtifacts.Union(SessionArtifacts);
// create a quicck summary of total failures, ensures, errors, etc
foreach (var Artifact in AllArtifacts)
{
string Summary = "NoSummary";
int ExitCode = GetRoleSummary(Artifact, out Summary);
if (ExitCode != 0 && Artifact.AppInstance.WasKilled == false)
{
AbnormalExits++;
}
if (SB.Length > 0)
{
SB.AppendLine();
}
SB.Append(Summary);
FatalErrors += Artifact.LogSummary.FatalError != null ? 1 : 0;
Ensures += Artifact.LogSummary.Ensures.Count();
Errors += Artifact.LogSummary.Errors.Count();
Warnings += Artifact.LogSummary.Warnings.Count();
}
MarkdownBuilder MB = new MarkdownBuilder();
string WarningStatement = HasWarnings ? " With Warnings" : "";
// Create a summary
MB.H2(string.Format("{0} {1}{2}", Name, GetTestResult(), WarningStatement));
if (GetTestResult() != TestResult.Passed)
{
if (FailureArtifacts.Count() > 0)
{
foreach (var FailedArtifact in FailureArtifacts)
{
string FirstProcessCause = "";
int FirstExitCode = GetExitCodeAndReason(FailedArtifact, out FirstProcessCause);
MB.H3(string.Format("{0}: {1}", FailedArtifact.SessionRole.RoleType, FirstProcessCause));
if (FailedArtifact.LogSummary.FatalError != null)
{
MB.H4(FailedArtifact.LogSummary.FatalError.Message);
}
}
MB.Paragraph("See below for callstack and logs");
}
}
MB.Paragraph(string.Format("Context: {0}", Context.ToString()));
MB.Paragraph(string.Format("FatalErrors: {0}, Ensures: {1}, Errors: {2}, Warnings: {3}", FatalErrors, Ensures, Errors, Warnings));
MB.Paragraph(string.Format("ResultHash: {0}", GetTestResultHash()));
//MB.Paragraph(string.Format("Artifacts: {0}", CachedArtifactPath));
MB.Append("--------");
MB.Append(SB.ToString());
//SB.Clear();
//SB.AppendLine("begin: stack for UAT");
//SB.Append(MB.ToString());
//SB.Append("end: stack for UAT");
return(MB.ToString());
}
