/// <summary> /// Returns a summary of this test /// </summary> /// <returns></returns> public override string GetTestSummary() { // Handle case where there aren't any session artifacts, for example with device starvation if (SessionArtifacts == null) { return("NoSummary"); } MarkdownBuilder ReportBuilder = new MarkdownBuilder(); // add header ReportBuilder.Append(GetTestSummaryHeader()); ReportBuilder.Append("--------"); StringBuilder SB = new StringBuilder(); // Get any artifacts with failures var FailureArtifacts = GetArtifactsWithFailures(); // Any with warnings (ensures) var WarningArtifacts = SessionArtifacts.Where(A => A.LogSummary.Ensures.Count() > 0); // combine artifacts into order as Failures, Warnings, Other var AllArtifacts = FailureArtifacts.Union(WarningArtifacts); AllArtifacts = AllArtifacts.Union(SessionArtifacts); // Add a summary of each foreach (var Artifact in AllArtifacts) { string Summary = "NoSummary"; int ExitCode = GetRoleSummary(Artifact, out Summary); if (SB.Length > 0) { SB.AppendLine(); } SB.Append(Summary); } ReportBuilder.Append(SB.ToString()); return(ReportBuilder.ToString()); }
/// <summary> /// Return all artifacts that are considered to have caused the test to fail /// </summary> /// <returns></returns> protected virtual IEnumerable <UnrealRoleArtifacts> GetArtifactsWithFailures() { if (SessionArtifacts == null) { Log.Warning("SessionArtifacts was null, unable to check for failures"); return(new UnrealRoleArtifacts[0] { }); } bool DidKillClients = SessionArtifacts.Any(A => A.SessionRole.RoleType.IsClient() && A.AppInstance.WasKilled); Dictionary <UnrealRoleArtifacts, int> ErrorCodes = new Dictionary <UnrealRoleArtifacts, int>(); var FailureList = SessionArtifacts.Where(A => { // ignore anything we killed if (A.AppInstance.WasKilled) { return(false); } string ExitReason; int ExitCode = GetExitCodeAndReason(A, out ExitReason); ErrorCodes.Add(A, ExitCode); return(ExitCode != 0); }); return(FailureList.OrderByDescending(A => { int Score = 0; if (A.LogSummary.FatalError != null || (ErrorCodes[A] != 0 && A.AppInstance.WasKilled == false)) { Score += 100000; } Score += A.LogSummary.Ensures.Count(); return Score; }).ToList()); }
/// <summary> /// Override the summary report so we can insert a link to our report and the failed tests /// </summary> /// <returns></returns> protected override string GetTestSummaryHeader() { MarkdownBuilder MB = new MarkdownBuilder(base.GetTestSummaryHeader()); string ReportLink = GetConfiguration().ReportOutputURL; if (string.IsNullOrEmpty(ReportLink) == false) { MB.Paragraph(string.Format("Report: {0}", ReportLink)); } var EditorArtifacts = SessionArtifacts.Where(A => A.SessionRole.RoleType == UnrealTargetRole.Editor).FirstOrDefault(); if (EditorArtifacts != null) { AutomationLogParser Parser = new AutomationLogParser(EditorArtifacts.LogParser); IEnumerable <AutomationTestResult> AllTests = Parser.GetResults(); IEnumerable <AutomationTestResult> FailedTests = AllTests.Where(R => !R.Passed); MB.Paragraph(string.Format("{0} of {1} tests passed", AllTests.Count() - FailedTests.Count(), AllTests.Count())); if (FailedTests.Count() > 0) { MB.H3("Test Failures"); foreach (AutomationTestResult Result in FailedTests) { MB.H4(string.Format("{0} Failed", Result.DisplayName)); MB.UnorderedList(Result.Events); } } } return(MB.ToString()); }
/// <summary> /// Returns a summary of this test /// </summary> /// <returns></returns> public override string GetTestSummary() { int AbnormalExits = 0; int FatalErrors = 0; int Ensures = 0; int Errors = 0; int Warnings = 0; // Handle case where there aren't any session artifacts, for example with device starvation if (SessionArtifacts == null) { return("NoSummary"); } StringBuilder SB = new StringBuilder(); // Get any artifacts with failures var FailureArtifacts = GetArtifactsWithFailures(); // Any with warnings (ensures) var WarningArtifacts = SessionArtifacts.Where(A => A.LogSummary.Ensures.Count() > 0); // combine artifacts into order as Failures, Warnings, Other var AllArtifacts = FailureArtifacts.Union(WarningArtifacts); AllArtifacts = AllArtifacts.Union(SessionArtifacts); // create a quicck summary of total failures, ensures, errors, etc foreach (var Artifact in AllArtifacts) { string Summary = "NoSummary"; int ExitCode = GetRoleSummary(Artifact, out Summary); if (ExitCode != 0 && Artifact.AppInstance.WasKilled == false) { AbnormalExits++; } if (SB.Length > 0) { SB.AppendLine(); } SB.Append(Summary); FatalErrors += Artifact.LogSummary.FatalError != null ? 1 : 0; Ensures += Artifact.LogSummary.Ensures.Count(); Errors += Artifact.LogSummary.Errors.Count(); Warnings += Artifact.LogSummary.Warnings.Count(); } MarkdownBuilder MB = new MarkdownBuilder(); string WarningStatement = HasWarnings ? " With Warnings" : ""; // Create a summary MB.H2(string.Format("{0} {1}{2}", Name, GetTestResult(), WarningStatement)); if (GetTestResult() != TestResult.Passed) { if (FailureArtifacts.Count() > 0) { foreach (var FailedArtifact in FailureArtifacts) { string FirstProcessCause = ""; int FirstExitCode = GetExitCodeAndReason(FailedArtifact, out FirstProcessCause); MB.H3(string.Format("{0}: {1}", FailedArtifact.SessionRole.RoleType, FirstProcessCause)); if (FailedArtifact.LogSummary.FatalError != null) { MB.H4(FailedArtifact.LogSummary.FatalError.Message); } } MB.Paragraph("See below for callstack and logs"); } } MB.Paragraph(string.Format("Context: {0}", Context.ToString())); MB.Paragraph(string.Format("FatalErrors: {0}, Ensures: {1}, Errors: {2}, Warnings: {3}", FatalErrors, Ensures, Errors, Warnings)); MB.Paragraph(string.Format("ResultHash: {0}", GetTestResultHash())); //MB.Paragraph(string.Format("Artifacts: {0}", CachedArtifactPath)); MB.Append("--------"); MB.Append(SB.ToString()); //SB.Clear(); //SB.AppendLine("begin: stack for UAT"); //SB.Append(MB.ToString()); //SB.Append("end: stack for UAT"); return(MB.ToString()); }