/// <summary> /// Return the header for the test summary. The header is the first block of text and will be /// followed by the summary of each individual role in the test /// </summary> /// <returns></returns> protected virtual string GetTestSummaryHeader() { int AbnormalExits = 0; int FatalErrors = 0; int Ensures = 0; int Errors = 0; int Warnings = 0; // create a quicck summary of total failures, ensures, errors, etc foreach (var Artifact in SessionArtifacts) { string Summary; int ExitCode = GetRoleSummary(Artifact, out Summary); if (ExitCode != 0 && Artifact.AppInstance.WasKilled == false) { AbnormalExits++; } FatalErrors += Artifact.LogSummary.FatalError != null ? 1 : 0; Ensures += Artifact.LogSummary.Ensures.Count(); Errors += Artifact.LogSummary.Errors.Count(); Warnings += Artifact.LogSummary.Warnings.Count(); } MarkdownBuilder MB = new MarkdownBuilder(); string WarningStatement = HasWarnings ? " With Warnings" : ""; var FailureArtifacts = GetArtifactsWithFailures(); // Create a summary MB.H2(string.Format("{0} {1}{2}", Name, GetTestResult(), WarningStatement)); if (GetTestResult() != TestResult.Passed) { if (FailureArtifacts.Count() > 0) { foreach (var FailedArtifact in FailureArtifacts) { string FirstProcessCause = ""; int FirstExitCode = GetExitCodeAndReason(FailedArtifact, out FirstProcessCause); MB.H3(string.Format("{0}: {1}", FailedArtifact.SessionRole.RoleType, FirstProcessCause)); if (FailedArtifact.LogSummary.FatalError != null) { MB.H4(FailedArtifact.LogSummary.FatalError.Message); } } MB.Paragraph("See below for logs and any callstacks"); } } MB.Paragraph(string.Format("Context: {0}", Context.ToString())); MB.Paragraph(string.Format("FatalErrors: {0}, Ensures: {1}, Errors: {2}, Warnings: {3}", FatalErrors, Ensures, Errors, Warnings)); MB.Paragraph(string.Format("Result: {0}, ResultHash: {1}", GetTestResult(), GetTestResultHash())); //MB.Paragraph(string.Format("Artifacts: {0}", CachedArtifactPath)); return(MB.ToString()); }
/// <summary> /// Returns a summary of this test /// </summary> /// <returns></returns> public override string GetTestSummary() { int AbnormalExits = 0; int FatalErrors = 0; int Ensures = 0; int Errors = 0; int Warnings = 0; StringBuilder SB = new StringBuilder(); // Sort our artifacts so any missing processes are first var ProblemArtifacts = GetArtifactsWithFailures(); var AllArtifacts = ProblemArtifacts.Union(SessionArtifacts); // create a quicck summary of total failures, ensures, errors, etc foreach (var Artifact in AllArtifacts) { string Summary = "NoSummary"; int ExitCode = GetRoleSummary(Artifact, out Summary); if (ExitCode != 0 && Artifact.AppInstance.WasKilled == false) { AbnormalExits++; } if (SB.Length > 0) { SB.AppendLine(); } SB.Append(Summary); FatalErrors += Artifact.LogSummary.FatalError != null ? 1 : 0; Ensures += Artifact.LogSummary.Ensures.Count(); Errors += Artifact.LogSummary.Errors.Count(); Warnings += Artifact.LogSummary.Warnings.Count(); } MarkdownBuilder MB = new MarkdownBuilder(); // Create a summary MB.H2(string.Format("{0} {1}", Name, GetTestResult())); if (GetTestResult() != TestResult.Passed) { if (ProblemArtifacts.Count() > 0) { foreach (var FailedArtifact in ProblemArtifacts) { string FirstProcessCause = ""; int FirstExitCode = GetExitCodeAndReason(FailedArtifact, out FirstProcessCause); MB.H3(string.Format("{0}: {1}", FailedArtifact.SessionRole.RoleType, FirstProcessCause)); if (FailedArtifact.LogSummary.FatalError != null) { MB.H4(FailedArtifact.LogSummary.FatalError.Message); } } MB.Paragraph("See below for callstack and logs"); } } MB.Paragraph(string.Format("Context: {0}", Context.ToString())); MB.Paragraph(string.Format("FatalErrors: {0}, Ensures: {1}, Errors: {2}, Warnings: {3}", FatalErrors, Ensures, Errors, Warnings)); //MB.Paragraph(string.Format("Artifacts: {0}", CachedArtifactPath)); MB.Append("--------"); MB.Append(SB.ToString()); //SB.Clear(); //SB.AppendLine("begin: stack for UAT"); //SB.Append(MB.ToString()); //SB.Append("end: stack for UAT"); return(MB.ToString()); }
/// <summary> /// Parses the output of an application to try and determine a failure cause (if one exists). Returns /// 0 for graceful shutdown /// </summary> /// <param name="Prefix"></param> /// <param name="App"></param> /// <returns></returns> protected virtual int GetRoleSummary(UnrealRoleArtifacts InArtifacts, out string Summary) { const int MaxLogLines = 10; const int MaxCallstackLines = 20; UnrealLogParser.LogSummary LogSummary = InArtifacts.LogSummary; string ExitReason = "Unknown"; int ExitCode = GetExitCodeAndReason(InArtifacts, out ExitReason); MarkdownBuilder MB = new MarkdownBuilder(); MB.H3(string.Format("Role: {0} ({1} {2})", InArtifacts.SessionRole.RoleType, InArtifacts.SessionRole.Platform, InArtifacts.SessionRole.Configuration)); if (ExitCode != 0) { MB.H4(string.Format("Result: Abnormal Exit: {0}", ExitReason)); } else { MB.H4(string.Format("Result: {0}", ExitReason)); } int FatalErrors = LogSummary.FatalError != null ? 1 : 0; if (LogSummary.FatalError != null) { MB.H4(string.Format("Fatal Error: {0}", LogSummary.FatalError.Message)); MB.UnorderedList(InArtifacts.LogSummary.FatalError.Callstack.Take(MaxCallstackLines)); if (InArtifacts.LogSummary.FatalError.Callstack.Count() > MaxCallstackLines) { MB.Paragraph("See log for full callstack"); } } if (LogSummary.Ensures.Count() > 0) { foreach (var Ensure in LogSummary.Ensures) { MB.H4(string.Format("Ensure: {0}", Ensure.Message)); MB.UnorderedList(Ensure.Callstack.Take(MaxCallstackLines)); if (Ensure.Callstack.Count() > MaxCallstackLines) { MB.Paragraph("See log for full callstack"); } } } MB.Paragraph(string.Format("FatalErrors: {0}, Ensures: {1}, Errors: {2}, Warnings: {3}", FatalErrors, LogSummary.Ensures.Count(), LogSummary.Errors.Count(), LogSummary.Warnings.Count())); if (GetConfiguration().ShowErrorsInSummary&& InArtifacts.LogSummary.Errors.Count() > 0) { MB.H4("Errors"); MB.UnorderedList(LogSummary.Errors.Take(MaxLogLines)); if (LogSummary.Errors.Count() > MaxLogLines) { MB.Paragraph(string.Format("(First {0} of {1} errors)", MaxLogLines, LogSummary.Errors.Count())); } } if (GetConfiguration().ShowWarningsInSummary&& InArtifacts.LogSummary.Warnings.Count() > 0) { MB.H4("Warnings"); MB.UnorderedList(LogSummary.Warnings.Take(MaxLogLines)); if (LogSummary.Warnings.Count() > MaxLogLines) { MB.Paragraph(string.Format("(First {0} of {1} warnings)", MaxLogLines, LogSummary.Warnings.Count())); } } MB.H4("Artifacts"); MB.Paragraph(string.Format("Log: {0}", InArtifacts.LogPath)); MB.Paragraph(string.Format("Commandline: {0}", InArtifacts.AppInstance.CommandLine)); MB.Paragraph(InArtifacts.ArtifactPath); Summary = MB.ToString(); return(ExitCode); }
/// <summary> /// Returns a summary of this test /// </summary> /// <returns></returns> public override string GetTestSummary() { int AbnormalExits = 0; int FatalErrors = 0; int Ensures = 0; int Errors = 0; int Warnings = 0; // Handle case where there aren't any session artifacts, for example with device starvation if (SessionArtifacts == null) { return("NoSummary"); } StringBuilder SB = new StringBuilder(); // Get any artifacts with failures var FailureArtifacts = GetArtifactsWithFailures(); // Any with warnings (ensures) var WarningArtifacts = SessionArtifacts.Where(A => A.LogSummary.Ensures.Count() > 0); // combine artifacts into order as Failures, Warnings, Other var AllArtifacts = FailureArtifacts.Union(WarningArtifacts); AllArtifacts = AllArtifacts.Union(SessionArtifacts); // create a quicck summary of total failures, ensures, errors, etc foreach (var Artifact in AllArtifacts) { string Summary = "NoSummary"; int ExitCode = GetRoleSummary(Artifact, out Summary); if (ExitCode != 0 && Artifact.AppInstance.WasKilled == false) { AbnormalExits++; } if (SB.Length > 0) { SB.AppendLine(); } SB.Append(Summary); FatalErrors += Artifact.LogSummary.FatalError != null ? 1 : 0; Ensures += Artifact.LogSummary.Ensures.Count(); Errors += Artifact.LogSummary.Errors.Count(); Warnings += Artifact.LogSummary.Warnings.Count(); } MarkdownBuilder MB = new MarkdownBuilder(); string WarningStatement = HasWarnings ? " With Warnings" : ""; // Create a summary MB.H2(string.Format("{0} {1}{2}", Name, GetTestResult(), WarningStatement)); if (GetTestResult() != TestResult.Passed) { if (FailureArtifacts.Count() > 0) { foreach (var FailedArtifact in FailureArtifacts) { string FirstProcessCause = ""; int FirstExitCode = GetExitCodeAndReason(FailedArtifact, out FirstProcessCause); MB.H3(string.Format("{0}: {1}", FailedArtifact.SessionRole.RoleType, FirstProcessCause)); if (FailedArtifact.LogSummary.FatalError != null) { MB.H4(FailedArtifact.LogSummary.FatalError.Message); } } MB.Paragraph("See below for callstack and logs"); } } MB.Paragraph(string.Format("Context: {0}", Context.ToString())); MB.Paragraph(string.Format("FatalErrors: {0}, Ensures: {1}, Errors: {2}, Warnings: {3}", FatalErrors, Ensures, Errors, Warnings)); MB.Paragraph(string.Format("ResultHash: {0}", GetTestResultHash())); //MB.Paragraph(string.Format("Artifacts: {0}", CachedArtifactPath)); MB.Append("--------"); MB.Append(SB.ToString()); //SB.Clear(); //SB.AppendLine("begin: stack for UAT"); //SB.Append(MB.ToString()); //SB.Append("end: stack for UAT"); return(MB.ToString()); }