private static Dictionary <string, MachineSummaryEntry> ProduceMachineSummaries(TestRecords tests)
{
Dictionary <string, MachineSummaryEntry> SummaryTable = new Dictionary <string, MachineSummaryEntry>(StringComparer.OrdinalIgnoreCase);
//Go through VariationRecords in each test to build up SummaryStats
foreach (TestRecord test in tests.TestCollection)
{
MachineSummaryEntry entry;
string name = ReportingUtilities.ReportMachine(test.Machine);
//Create Entry if machine doesn't exist yet, else load it up
if (!SummaryTable.ContainsKey(name))
{
entry = new MachineSummaryEntry();
entry.TestExecutionTime = TimeSpan.Zero;
entry.Name = name;
SummaryTable.Add(name, entry);
}
else
{
entry = SummaryTable[name];
}
bool hasBugs = (test.TestInfo.Bugs != null && test.TestInfo.Bugs.Count > 0);
foreach (VariationRecord variation in test.Variations)
{
entry.TotalVariations += ReportingUtilities.OneForCountable(variation.Result);
entry.FailedVariations += ReportingUtilities.OneForFail(variation.Result);
entry.FailedVariationsWithBugs += ReportingUtilities.OneForFailOnBug(variation.Result, hasBugs);
}
entry.TestExecutionTime += ReportingUtilities.GetTestDuration(test);
if (test.Variations.Count == 0)
{
entry.TestsWithoutVariation += 1;
}
}
return(SummaryTable);
}
private static void Generate(TestRecords records, string path)
{
using (XmlTableWriter tableWriter = new XmlTableWriter(path))
{
tableWriter.AddXsl(@"DrtReport.xsl");
tableWriter.WriteStartElement("Variations");
tableWriter.WriteAttributeString("PassRate", ReportingUtilities.CalculatePassRate(records));
foreach (TestRecord test in FilterNonPassingTests(records))
{
TestInfo testInfo = test.TestInfo;
{
tableWriter.WriteStartElement("Variation");
tableWriter.WriteAttributeString("Area", testInfo.Area);
tableWriter.WriteAttributeString("TestName", testInfo.Name);
tableWriter.WriteAttributeString("Variation", "Test Level Summary");
tableWriter.WriteAttributeString("Duration", ReportingUtilities.FormatTimeSpanAsSeconds(ReportingUtilities.GetTestDuration(test))); //Total test execution Time
tableWriter.WriteAttributeString("Result", ReportingUtilities.InterpretTestOutcome(test).ToString());
tableWriter.WriteAttributeString("Log", test.Log);
tableWriter.WriteAttributeString("LogDir", ReportingUtilities.ReportPaths(test.LoggedFiles));
tableWriter.WriteEndElement();
}
foreach (VariationRecord variation in test.Variations)
{
tableWriter.WriteStartElement("Variation");
tableWriter.WriteAttributeString("Area", testInfo.Area);
tableWriter.WriteAttributeString("TestName", testInfo.Name);
tableWriter.WriteAttributeString("Variation", variation.VariationName);
tableWriter.WriteAttributeString("Duration", ReportingUtilities.FormatTimeSpanAsSeconds(ReportingUtilities.GetVariationDuration(variation)));
tableWriter.WriteAttributeString("Result", variation.Result.ToString());
tableWriter.WriteAttributeString("Log", variation.Log);
tableWriter.WriteAttributeString("LogDir", ReportingUtilities.ReportPaths(variation.LoggedFiles));
tableWriter.WriteEndElement();
}
}
tableWriter.WriteEndElement();
}
}
internal static void Generate(TestRecords records, DirectoryInfo ReportRoot)
{
// See https://xunit.net/docs/format-xml-v2.html for the documentation on how to format xunit logs
var root = new XElement("assemblies");
// We run our tests by Area in Helix. The SummaryReportGenerator does a nice job of aggregating the TestRecords
// by area already, so we'l reuse that. This ensures we aren't making too many assummptions (like that all records are in the same Area)
// that could break later.
var resultsByArea = SummaryReportGenerator.ProduceAreaSummaries(records);
foreach (SummaryReportGenerator.AreaSummaryEntry areaEntry in resultsByArea.Values)
{
string assemblyName = areaEntry.AssociatedTests.FirstOrDefault()?.TestInfo.DriverParameters["exe"];
var assembly = new XElement("assembly");
assembly.SetAttributeValue("name", assemblyName);
assembly.SetAttributeValue("test-framework", "QualityVault");
assembly.SetAttributeValue("run-date", DateTime.Now.ToString("yyyy-mm-dd"));
assembly.SetAttributeValue("run-time", areaEntry.StartTime.ToString(@"hh\:mm\:ss"));
assembly.SetAttributeValue("time", (areaEntry.EndTime - areaEntry.StartTime).TotalSeconds);
assembly.SetAttributeValue("total", areaEntry.TotalVariations);
assembly.SetAttributeValue("passed", areaEntry.TotalVariations - areaEntry.FailedVariations - areaEntry.IgnoredVariations);
assembly.SetAttributeValue("failed", areaEntry.FailedVariations);
assembly.SetAttributeValue("skipped", areaEntry.IgnoredVariations);
assembly.SetAttributeValue("errors", 0);
root.Add(assembly);
foreach (TestRecord testRecord in areaEntry.AssociatedTests)
{
var collection = new XElement("collection");
int testPassedCount = testRecord.Variations.Where(variation => variation.Result == Result.Pass).Count();
int testFailedCount = testRecord.Variations.Where(variation => variation.Result == Result.Fail).Count();
int testSkippedCount = testRecord.Variations.Where(variation => variation.Result == Result.Ignore).Count();
collection.SetAttributeValue("total", testRecord.Variations.Count);
collection.SetAttributeValue("passed", testPassedCount);
collection.SetAttributeValue("failed", testFailedCount);
collection.SetAttributeValue("skipped", testSkippedCount);
collection.SetAttributeValue("name", testRecord.TestInfo.Name);
collection.SetAttributeValue("time", ReportingUtilities.FormatTimeSpanAsSeconds(ReportingUtilities.GetTestDuration(testRecord)));
assembly.Add(collection);
foreach (VariationRecord variation in testRecord.Variations)
{
var test = new XElement("test");
string className = testRecord.TestInfo.DriverParameters["class"];
string methodName = testRecord.TestInfo.DriverParameters["method"];
test.SetAttributeValue("type", className);
test.SetAttributeValue("method", methodName);
test.SetAttributeValue("name", variation.VariationName);
test.SetAttributeValue("time", testRecord.Duration.TotalSeconds);
test.SetAttributeValue("result", variation.Result == Result.Ignore ? "Skip" : variation.Result.ToString());
if (variation.Result != Result.Pass)
{
var failure = new XElement("failure");
failure.SetAttributeValue("exception-type", "Exception");
var message = new XElement("message");
StringBuilder errorMessage = new StringBuilder();
errorMessage.AppendLine("Error Log: ");
errorMessage.AppendLine(testRecord.Log);
message.Add(new XCData(errorMessage.ToString()));
failure.Add(message);
test.Add(failure);
}
collection.Add(test);
}
}
}
string xunitOutputPath = Path.Combine(ReportRoot.FullName, "testResults.xml");
File.WriteAllText(xunitOutputPath, root.ToString());
}
/// <summary>
/// Write a Test node under Test, to hold information for the Test: log, test result, duration, Test info, which
/// won't show for every Variation in Xml viewer.
/// </summary>
/// <param name="tableWriter">Writer</param>
/// <param name="test">test</param>
/// <param name="testInfo">testInfo</param>
/// <param name="testLogPath">test log path</param>
private static void WriteChildTestNode(XmlTableWriter tableWriter, TestRecord test, TestInfo testInfo, string testLogPath)
{
bool logTruncated = false;
tableWriter.WriteStartElement("Test");
tableWriter.WriteAttributeString("Duration", ReportingUtilities.FormatTimeSpanAsSeconds(ReportingUtilities.GetTestDuration(test))); //Total test execution Time
tableWriter.WriteAttributeString("Result", ReportingUtilities.InterpretTestOutcome(test).ToString());
tableWriter.WriteAttributeString("Log", ReportingUtilities.ProcessLongLog(test.Log, testLogPath, ref logTruncated));
if (logTruncated)
{
tableWriter.WriteAttributeString("LogPath", Path.Combine(ReportingUtilities.TestLogsDir, Path.GetFileName(testLogPath)));
}
tableWriter.WriteAttributeString("LogDir", ReportingUtilities.ReportPaths(test.LoggedFiles));
int failed = 0;
int failedOnBug = 0;
bool hasBugs = ReportingUtilities.TestHasBugs(testInfo);
foreach (VariationRecord variation in test.Variations)
{
failed += ReportingUtilities.OneForFail(variation.Result);
failedOnBug += ReportingUtilities.OneForFailOnBug(variation.Result, hasBugs);
}
tableWriter.WriteAttributeString("Failures", string.Format("{0}", failed - failedOnBug));
tableWriter.WriteAttributeString("Total", string.Format("{0}", test.Variations.Count));
tableWriter.WriteEndElement();
}