/// <summary>
/// Converts the given results file to TestRunData object
/// </summary>
/// <param name="filePath">File path</param>
/// <param name="runName">Run Name</param>
/// <returns>TestRunData</returns>
public TestRunData ReadResultsFromFile(TestRunContext runContext, string filePath, string runName)
{
ArgUtil.NotNull(runContext, nameof(runContext));
Trace.Entering();
runContext.RunName = runName;
return(_resultReader.ReadResults(_executionContext, filePath, runContext));
}
private void ReadResults(TestRunContext runContext = null, bool attachRunLevelAttachments = true)
{
_fileName = Path.Combine(Path.GetTempPath(), Path.GetTempFileName());
File.WriteAllText(_fileName, _nunitResultsToBeRead);
_nUnitReader = new NUnitResultReader();
_nUnitReader.AddResultsFileToRunLevelAttachments = attachRunLevelAttachments;
_testRunData = _nUnitReader.ReadResults(_ec.Object, _fileName, runContext);
}
public async Task <bool> PublishAsync(TestRunContext runContext, List <string> testResultFiles, string runTitle, int?buildId, bool mergeResults)
{
if (mergeResults)
{
return(await PublishAllTestResultsToSingleTestRunAsync(testResultFiles, _testRunPublisher, runContext, _resultReader.Name, runTitle, buildId, _executionContext.CancellationToken));
}
else
{
return(await PublishToNewTestRunPerTestResultFileAsync(testResultFiles, _testRunPublisher, runContext, _resultReader.Name, runTitle, PublishBatchSize, _executionContext.CancellationToken));
}
}
private string ParseRunName(TestRunContext runContext)
{
string runName = ParserName + " Test Run";
if (runContext != null)
{
runName = !String.IsNullOrWhiteSpace(runContext.RunName)
? runContext.RunName
: $"{runName} {runContext.Configuration} {runContext.Platform}";
}
return(runName);
}
private void InitialiseRunUserIdRef(TestRunContext runContext)
{
string runUser = "";
if (runContext?.Owner != null)
{
runUser = runContext.Owner;
}
_runUserIdRef = new IdentityRef()
{
DisplayName = runUser
};
}
/// <summary>
/// Publish separate test run for each result file that has results.
/// </summary>
private async Task <bool> PublishToNewTestRunPerTestResultFileAsync(List <string> resultFiles,
ITestRunPublisher publisher,
TestRunContext runContext,
string resultReader,
string runTitle,
int batchSize,
CancellationToken cancellationToken)
{
bool isTestRunOutcomeFailed = false;
try
{
IList <TestRun> publishedRuns = new List <TestRun>();
var groupedFiles = resultFiles
.Select((resultFile, index) => new { Index = index, file = resultFile })
.GroupBy(pair => pair.Index / batchSize)
.Select(bucket => bucket.Select(pair => pair.file).ToList())
.ToList();
bool changeTestRunTitle = resultFiles.Count > 1;
TestRunSummary testRunSummary = new TestRunSummary();
foreach (var files in groupedFiles)
{
// Publish separate test run for each result file that has results.
var publishTasks = files.Select(async resultFile =>
{
cancellationToken.ThrowIfCancellationRequested();
string runName = runTitle;
if (!string.IsNullOrWhiteSpace(runTitle) && changeTestRunTitle)
{
runName = GetRunName(runTitle);
}
_executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
TestRunData testRunData = publisher.ReadResultsFromFile(runContext, resultFile, runName);
testRunData.PipelineReference = runContext.PipelineReference;
isTestRunOutcomeFailed = isTestRunOutcomeFailed || GetTestRunOutcome(testRunData, testRunSummary);
cancellationToken.ThrowIfCancellationRequested();
if (testRunData != null)
{
if (testRunData.Results != null && testRunData.Results.Length > 0)
{
testRunData.AddCustomField(_testRunSystemCustomFieldName, runContext.TestRunSystem);
AddTargetBranchInfoToRunCreateModel(testRunData, runContext.TargetBranchName);
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, testRunData.Results, _executionContext.CancellationToken);
TestRun updatedRun = await publisher.EndTestRunAsync(testRunData, testRun.Id, cancellationToken: _executionContext.CancellationToken);
publishedRuns.Add(updatedRun);
}
else
{
_executionContext.Output(StringUtil.Loc("NoResultFound", resultFile));
}
}
else
{
_executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
}
});
await Task.WhenAll(publishTasks);
}
// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
{
isTestRunOutcomeFailed = runOutcome.Value;
}
}
StoreTestRunSummaryInEnvVar(testRunSummary);
}
catch (Exception ex) when(!(ex is OperationCanceledException && _executionContext.CancellationToken.IsCancellationRequested))
{
// Not catching all the operationcancelled exceptions, as the pipeline cancellation should cancel the command as well.
// Do not fail the task.
LogPublishTestResultsFailureWarning(ex);
}
return(isTestRunOutcomeFailed);
}
/// <summary>
/// Publish separate test run for each result file that has results.
/// </summary>
private async Task <bool> PublishToNewTestRunPerTestResultFileAsync(List <string> resultFiles,
ITestRunPublisher publisher,
TestRunContext runContext,
string resultReader,
string runTitle,
int batchSize,
CancellationToken cancellationToken)
{
bool isTestRunOutcomeFailed = false;
try
{
var groupedFiles = resultFiles
.Select((resultFile, index) => new { Index = index, file = resultFile })
.GroupBy(pair => pair.Index / batchSize)
.Select(bucket => bucket.Select(pair => pair.file).ToList())
.ToList();
bool changeTestRunTitle = resultFiles.Count > 1;
TestRunSummary testRunSummary = new TestRunSummary();
foreach (var files in groupedFiles)
{
// Publish separate test run for each result file that has results.
var publishTasks = files.Select(async resultFile =>
{
cancellationToken.ThrowIfCancellationRequested();
string runName = runTitle;
if (!string.IsNullOrWhiteSpace(runTitle) && changeTestRunTitle)
{
runName = GetRunName(runTitle);
}
_executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
TestRunData testRunData = publisher.ReadResultsFromFile(runContext, resultFile, runName);
testRunData.PipelineReference = runContext.PipelineReference;
isTestRunOutcomeFailed = isTestRunOutcomeFailed || GetTestRunOutcome(testRunData, testRunSummary);
cancellationToken.ThrowIfCancellationRequested();
if (testRunData != null)
{
if (testRunData.Results != null && testRunData.Results.Length > 0)
{
testRunData.AddCustomField(_testRunSystemCustomFieldName, runContext.TestRunSystem);
AddTargetBranchInfoToRunCreateModel(testRunData, runContext.TargetBranchName);
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, testRunData.Results, _executionContext.CancellationToken);
await publisher.EndTestRunAsync(testRunData, testRun.Id, cancellationToken: _executionContext.CancellationToken);
}
else
{
_executionContext.Output(StringUtil.Loc("NoResultFound", resultFile));
}
}
else
{
_executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
}
});
await Task.WhenAll(publishTasks);
}
StoreTestRunSummaryInEnvVar(testRunSummary);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
//Do not fail the task.
LogPublishTestResultsFailureWarning(ex);
}
return(isTestRunOutcomeFailed);
}
/// <summary>
/// Converts the given results file to TestRunData object
/// </summary>
/// <param name="filePath">File path</param>
/// <returns>TestRunData</returns>
public TestRunData ReadResultsFromFile(TestRunContext runContext, string filePath)
{
Trace.Entering();
return(_resultReader.ReadResults(_executionContext, filePath, runContext));
}
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
var doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Ignore
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return(null);
}
INUnitResultsXmlReader nunitResultsReader = null;
var testResultsNode = doc.SelectSingleNode("test-results");
if (testResultsNode == null)
{
testResultsNode = doc.SelectSingleNode("test-run");
if (testResultsNode == null)
{
throw new NotSupportedException(StringUtil.Loc("InvalidFileFormat"));
}
nunitResultsReader = new NUnit3ResultsXmlReader();
}
else
{
nunitResultsReader = new NUnit2ResultsXmlReader();
}
return(nunitResultsReader.GetTestRunData(filePath, doc, testResultsNode, runContext, AddResultsFileToRunLevelAttachments));
}
/// <summary>
/// Reads a JUnit results file from disk, converts it into a TestRunData object.
/// </summary>
/// <param name="filePath"></param>
/// <param name="runContext"></param>
/// <returns></returns>
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
// http://windyroad.com.au/dl/Open%20Source/JUnit.xsd
XmlDocument doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Ignore
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return(null);
}
//init test run summary information - run name, host name, start time
TestSuiteSummary runSummary = new TestSuiteSummary(Name);
IdentityRef runUserIdRef = null;
string runUser = runContext != null ? runContext.Owner : string.Empty;
if (!string.IsNullOrEmpty(runUser))
{
runUserIdRef = new IdentityRef()
{
DisplayName = runUser
};
}
var presentTime = DateTime.UtcNow;
runSummary.TimeStamp = DateTime.MaxValue;
var maxCompletedTime = DateTime.MinValue;
//read data from testsuite nodes
XmlNode testSuitesNode = doc.SelectSingleNode("testsuites");
if (testSuitesNode != null)
{
//found testsuites node - some plugins generate it like karma junit plugin
XmlNodeList testSuiteNodeList = doc.SelectNodes("/testsuites/testsuite");
if (testSuiteNodeList != null)
{
foreach (XmlNode testSuiteNode in testSuiteNodeList)
{
//for each available suites get all suite details
TestSuiteSummary testSuiteSummary = ReadTestSuite(testSuiteNode, runUserIdRef);
// sum up testsuite durations and test case durations, decision on what to use will be taken later
runSummary.TotalTestCaseDuration = runSummary.TotalTestCaseDuration.Add(testSuiteSummary.TotalTestCaseDuration);
runSummary.TestSuiteDuration = runSummary.TestSuiteDuration.Add(testSuiteSummary.TestSuiteDuration);
runSummary.SuiteTimeDataAvailable = runSummary.SuiteTimeDataAvailable && testSuiteSummary.SuiteTimeDataAvailable;
runSummary.SuiteTimeStampAvailable = runSummary.SuiteTimeStampAvailable && testSuiteSummary.SuiteTimeStampAvailable;
runSummary.Host = testSuiteSummary.Host;
runSummary.Name = testSuiteSummary.Name;
//stop calculating timestamp information, if timestamp data is not avilable for even one test suite
if (testSuiteSummary.SuiteTimeStampAvailable)
{
runSummary.TimeStamp = runSummary.TimeStamp > testSuiteSummary.TimeStamp ? testSuiteSummary.TimeStamp : runSummary.TimeStamp;
DateTime completedTime = testSuiteSummary.TimeStamp.AddTicks(testSuiteSummary.TestSuiteDuration.Ticks);
maxCompletedTime = maxCompletedTime < completedTime ? completedTime : maxCompletedTime;
}
runSummary.Results.AddRange(testSuiteSummary.Results);
}
if (testSuiteNodeList.Count > 1)
{
runSummary.Name = Name + "_" + Path.GetFileName(filePath);
}
}
}
else
{
XmlNode testSuiteNode = doc.SelectSingleNode("testsuite");
if (testSuiteNode != null)
{
runSummary = ReadTestSuite(testSuiteNode, runUserIdRef);
//only if start time is available then only we need to calculate completed time
if (runSummary.TimeStamp != DateTime.MaxValue)
{
DateTime completedTime = runSummary.TimeStamp.AddTicks(runSummary.TestSuiteDuration.Ticks);
maxCompletedTime = maxCompletedTime < completedTime ? completedTime : maxCompletedTime;
}
else
{
runSummary.SuiteTimeStampAvailable = false;
}
}
}
if (runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName))
{
runSummary.Name = runContext.RunName;
}
if (!runSummary.SuiteTimeStampAvailable)
{
executionContext.Output("Timestamp is not available for one or more testsuites. Total run duration is being calculated as the sum of time durations of detected testsuites");
if (!runSummary.SuiteTimeDataAvailable)
{
executionContext.Output("Time is not available for one or more testsuites. Total run duration is being calculated as the sum of time durations of detected testcases");
}
}
//if start time is not calculated then it should be initialized as present time
runSummary.TimeStamp = runSummary.TimeStamp == DateTime.MaxValue
? presentTime
: runSummary.TimeStamp;
//if suite timestamp data is not available even for single testsuite, then fallback to testsuite run time
//if testsuite run time is not available even for single testsuite, then fallback to total test case duration
maxCompletedTime = !runSummary.SuiteTimeStampAvailable || maxCompletedTime == DateTime.MinValue
? runSummary.TimeStamp.Add(runSummary.SuiteTimeDataAvailable ? runSummary.TestSuiteDuration
: runSummary.TotalTestCaseDuration) : maxCompletedTime;
//create test run data
var testRunData = new TestRunData(
name: runSummary.Name,
startedDate: runSummary.TimeStamp != DateTime.MinValue ? runSummary.TimeStamp.ToString("o") : null,
completedDate: maxCompletedTime != DateTime.MinValue ? maxCompletedTime.ToString("o") : null,
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: runContext != null ? runContext.Platform : string.Empty,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
)
{
Results = runSummary.Results.ToArray(),
Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0]
};
return(testRunData);
}
/// <summary>
/// Reads a trx file from disk, converts it into a TestRunData object.
/// </summary>
/// <param name="filePath">File path</param>
/// <returns>TestRunData</returns>
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext)
{
_executionContext = executionContext;
_definitions = new Dictionary <string, TestCaseDefinition>();
List <TestCaseResultData> results = new List <TestCaseResultData>();
string xmlContents = File.ReadAllText(filePath);
xmlContents = xmlContents.Replace("xmlns", "ns");
XmlDocument doc = new XmlDocument();
doc.LoadXml(xmlContents);
string runName = Name + " Test Run";
if (runContext != null)
{
if (!String.IsNullOrWhiteSpace(runContext.RunName))
{
runName = runContext.RunName;
}
else
{
runName = StringUtil.Format("{0} {1} {2}", runName, runContext.Configuration, runContext.Platform);
}
}
string runUser = "";
if (runContext.Owner != null)
{
runUser = runContext.Owner;
}
_runUserIdRef = new IdentityRef();
_runUserIdRef.DisplayName = runUser;
//Parse the run start and finish times. If either of it is not available, send neither.
XmlNode node = doc.SelectSingleNode("/TestRun/Times");
DateTime runStartDate = DateTime.MinValue;
DateTime runFinishDate = DateTime.MinValue;
if (node != null && node.Attributes["start"] != null && node.Attributes["finish"] != null)
{
if (DateTime.TryParse(node.Attributes["start"].Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out runStartDate))
{
_executionContext.Debug(string.Format(CultureInfo.InvariantCulture, "Setting run start and finish times."));
//Only if there is a valid start date.
DateTime.TryParse(node.Attributes["finish"].Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out runFinishDate);
if (runFinishDate < runStartDate)
{
runFinishDate = runStartDate = DateTime.MinValue;
_executionContext.Debug("Run finish date is less than start date. Resetting to min value.");
}
}
}
TestRunData testRunData = new TestRunData(
name: runName,
buildId: runContext.BuildId,
startedDate: runStartDate != DateTime.MinValue ? runStartDate.ToString("o") : null,
completedDate: runFinishDate != DateTime.MinValue ? runFinishDate.ToString("o") : null,
state: TestRunState.InProgress.ToString(),
isAutomated: true,
dueDate: string.Empty,
type: string.Empty,
buildFlavor: runContext.Configuration,
buildPlatform: runContext.Platform,
releaseUri: runContext.ReleaseUri,
releaseEnvironmentUri: runContext.ReleaseEnvironmentUri
);
//Parse the Deployment node for the runDeploymentRoot - this is the attachment root. Required for .NET Core
XmlNode deploymentNode = doc.SelectSingleNode("/TestRun/TestSettings/Deployment");
var _attachmentRoot = string.Empty;
if (deploymentNode != null && deploymentNode.Attributes["runDeploymentRoot"] != null)
{
_attachmentRoot = deploymentNode.Attributes["runDeploymentRoot"].Value;
}
else
{
_attachmentRoot = Path.GetFileNameWithoutExtension(filePath); // This for platform v1
}
_attachmentLocation = Path.Combine(Path.GetDirectoryName(filePath), _attachmentRoot, "In");
_executionContext.Debug(string.Format(CultureInfo.InvariantCulture, "Attachment location: {0}", _attachmentLocation));
AddRunLevelAttachments(filePath, doc, testRunData);
// Create a dictionary of testcase definitions to be used when iterating over the results
Dictionary <string, TestCaseDefinition> definitions = new Dictionary <string, TestCaseDefinition>();
if (doc.SelectNodes("/TestRun/TestDefinitions").Count > 0)
{
foreach (XmlNode definitionNode in doc.SelectNodes("/TestRun/TestDefinitions")[0])
{
IdentityRef owner = null;
string priority = null, storage = null;
if (definitionNode.Attributes["storage"] != null && definitionNode.Attributes["storage"].Value != null)
{
storage = Path.GetFileName(definitionNode.Attributes["storage"].Value);
}
XmlAttribute priorityAttribute = definitionNode.Attributes["priority"];
if (priorityAttribute != null)
{
priority = priorityAttribute.Value;
}
XmlNode ownerNode = definitionNode.SelectSingleNode("./Owners/Owner");
if (ownerNode != null)
{
IdentityRef ownerIdRef = new IdentityRef();
ownerIdRef.DisplayName = ownerNode.Attributes["name"].Value;
ownerIdRef.DirectoryAlias = ownerNode.Attributes["name"].Value;
owner = ownerIdRef;
}
// The automated test name should be FQDN, if we are unable to figure it out like in case of a webtest,
// set it as "name" from the parent (where it is always present)
XmlNode testResultNode = definitionNode.SelectSingleNode("./TestMethod");
string automatedTestName = null;
if (testResultNode != null && testResultNode.Attributes["className"] != null && testResultNode.Attributes["name"] != null)
{
// At times the class names are coming as
// className="MS.TF.Test.AgileX.VSTests.WiLinking.UI.WiLinkingUIQueryTests"
// at other times, they are as
// className="UnitTestProject3.UnitTest1, UnitTestProject3, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null"
string className = testResultNode.Attributes["className"].Value.Split(',')[0];
automatedTestName = className + "." + testResultNode.Attributes["name"].Value;
}
else if (definitionNode.Attributes["name"] != null)
{
automatedTestName = definitionNode.Attributes["name"].Value;
}
_definitions.Add(definitionNode.Attributes["id"].Value, new TestCaseDefinition(automatedTestName, owner, priority, storage));
}
}
// Read UnitTestResults, WebTestResults and OrderedTestResults
XmlNodeList resultsNodes = doc.SelectNodes("/TestRun/Results/UnitTestResult");
XmlNodeList webTestResultNodes = doc.SelectNodes("/TestRun/Results/WebTestResult");
XmlNodeList orderedTestResultNodes = doc.SelectNodes("/TestRun/Results/TestResultAggregation");
results.AddRange(ReadActualResults(resultsNodes, TestType.UnitTest));
results.AddRange(ReadActualResults(webTestResultNodes, TestType.WebTest));
results.AddRange(ReadActualResults(orderedTestResultNodes, TestType.OrderedTest));
testRunData.Results = results.ToArray <TestCaseResultData>();
_executionContext.Debug(string.Format(CultureInfo.InvariantCulture, "Total test results: {0}", testRunData.Results.Length));
return(testRunData);
}
public TestRunData GetTestRunData(string filePath, XmlDocument doc, XmlNode testResultsNode, TestRunContext runContext, bool addResultsAsAttachments)
{
var results = new List <TestCaseResultData>();
//read test run summary information - run name, start time
string runName = "NUnit Test Run";
DateTime runStartTime = DateTime.MinValue; //Use local time instead of UTC as TestRunData uses local time for defaults. Also assuming timestamp is local is more accurate in cases where tests were run on build machine
TimeSpan totalRunDuration = TimeSpan.Zero;
TimeSpan totalTestCaseDuration = TimeSpan.Zero;
if (testResultsNode != null)
{
//get test run summary information
if (testResultsNode.Attributes["name"] != null)
{
runName = testResultsNode.Attributes["name"].Value;
}
//run times
DateTime dateFromXml = DateTime.MinValue.Date; //Use local time instead of UTC as TestRunData uses local time for defaults.
if (testResultsNode.Attributes["date"] != null)
{
DateTime.TryParse(testResultsNode.Attributes["date"].Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.None, out dateFromXml);
}
TimeSpan timeFromXml = TimeSpan.Zero;
if (testResultsNode.Attributes["time"] != null)
{
TimeSpan.TryParse(testResultsNode.Attributes["time"].Value, CultureInfo.InvariantCulture, out timeFromXml);
}
//assume runtimes from xml are current local time since timezone information is not in the xml, if xml datetime > current local time, fallback to local start time
DateTime runStartDateTimeFromXml = new DateTime(dateFromXml.Ticks).AddTicks(timeFromXml.Ticks);
if (runStartTime == DateTime.MinValue)
{
runStartTime = runStartDateTimeFromXml;
}
}
//run environment - platform, config and hostname
string platform = runContext != null ? runContext.Platform : string.Empty;
string config = runContext != null ? runContext.Configuration : string.Empty;
string runUser = runContext != null ? runContext.Owner : string.Empty;
string hostName = string.Empty;
XmlNode envNode = doc.SelectSingleNode(RootNodeName + "/environment");
if (envNode != null)
{
if (envNode.Attributes["machine-name"] != null && envNode.Attributes["machine-name"].Value != null)
{
hostName = envNode.Attributes["machine-name"].Value;
}
if (envNode.Attributes["platform"] != null && envNode.Attributes["platform"].Value != null && runContext != null && runContext.BuildId > 0)
{
//We cannot publish platform information without a valid build id.
platform = envNode.Attributes["platform"].Value;
}
}
//run owner
IdentityRef runUserIdRef = null;
if (!string.IsNullOrEmpty(runUser))
{
runUserIdRef = new IdentityRef()
{
DisplayName = runUser
};
}
//get all test assemblies
if (testResultsNode != null)
{
XmlNodeList testAssemblyNodes = testResultsNode.SelectNodes("test-suite");
if (testAssemblyNodes != null)
{
foreach (XmlNode testAssemblyNode in testAssemblyNodes)
{
var assemblyStartTime = (runStartTime == DateTime.MinValue) ? DateTime.MinValue : runStartTime + totalTestCaseDuration;
List <TestCaseResultData> testCases = FindTestCaseNodes(testAssemblyNode, hostName, runUserIdRef, assemblyStartTime);
if (testCases != null)
{
results.AddRange(testCases);
testCases.ForEach(x => totalTestCaseDuration += TimeSpan.FromMilliseconds(x.DurationInMs));
}
}
}
}
if (TimeSpan.Compare(totalRunDuration, totalTestCaseDuration) < 0)
{
totalRunDuration = totalTestCaseDuration; //run duration may not be set in the xml, so use total test case duration
}
if (runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName))
{
runName = runContext.RunName;
}
//create test run data
TestRunData testRunData = new TestRunData(
name: runName,
startedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.ToString("o"),
completedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.Add(totalRunDuration).ToString("o"),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: config,
buildPlatform: platform,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.Results = results.ToArray();
testRunData.Attachments = addResultsAsAttachments ? new string[] { filePath } : new string[0];
return(testRunData);
}
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext)
{
try
{
string jsonTestSummary = File.ReadAllText(filePath);
if (string.IsNullOrWhiteSpace(jsonTestSummary))
{
return(null);
}
JsonTestSummary testSummary = StringUtil.ConvertFromJson <JsonTestSummary>(jsonTestSummary);
// Adding the minimum details from the JSON.
TestRunData testRunData = new TestRunData(name: "Container Structure Test",
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: runContext != null ? runContext.Platform : string.Empty,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null);
List <TestCaseResultData> results = new List <TestCaseResultData>();
foreach (JsonTestResult result in testSummary.Results)
{
TestCaseResultData resultCreateModel = new TestCaseResultData();
resultCreateModel.TestCaseTitle = result.Name;
resultCreateModel.AutomatedTestName = result.Name;
bool outcome = result.Pass.Equals("true", StringComparison.OrdinalIgnoreCase);
if (!outcome)
{
resultCreateModel.ErrorMessage = string.Join("|", result.Errors);
}
resultCreateModel.State = "Completed";
resultCreateModel.AutomatedTestType = Name;
resultCreateModel.Outcome = outcome ? TestOutcome.Passed.ToString() : TestOutcome.Failed.ToString();
results.Add(resultCreateModel);
}
testRunData.Results = results.ToArray();
return(testRunData);
}
catch (Exception ex)
{
executionContext.Output("Error occured in reading results : " + ex);
}
return(null);
}
/// <summary>
/// Reads a CTest results file from disk, converts it into a TestRunData object.
/// </summary>
/// <param name="filePath"></param>
/// <param name="runContext"></param>
/// <returns></returns>
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
_ec = executionContext;
// Read Xml File
XmlDocument doc = new XmlDocument();
try
{
XmlReaderSettings settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Ignore
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return(null);
}
// init test run summary information - run name, host name, start time
List <TestCaseResultData> results = new List <TestCaseResultData>();
string runName = ParseRunName(runContext);
InitialiseRunUserIdRef(runContext);
//run environment - platform, config and hostname
string platform = runContext != null ? runContext.Platform : string.Empty;
string config = runContext != null ? runContext.Configuration : string.Empty;
string runUser = runContext != null ? runContext.Owner : string.Empty;
string hostName = string.Empty;
//Parse the run start and finish times.
XmlNode node = doc.SelectSingleNode("/Site/Testing");
_runStartDate = DateTime.MinValue;
_runFinishDate = DateTime.MinValue;
ParseRunStartAndFinishDates(node);
//create test run data object
TestRunData testRunData = new TestRunData(
name: runName,
startedDate: (_runStartDate == DateTime.MinValue) ? string.Empty : _runStartDate.ToString("o"),
completedDate: (_runFinishDate == DateTime.MinValue) ? string.Empty : _runFinishDate.ToString("o"),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: config,
buildPlatform: platform,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
)
{
Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0]
};
// Read results
XmlNodeList resultsNodes = doc.SelectNodes("/Site/Testing/Test");
results.AddRange(ReadActualResults(resultsNodes));
testRunData.Results = results.ToArray();
return(testRunData);
}
/// <summary>
/// Publish single test run
/// </summary>
private async Task <bool> PublishAllTestResultsToSingleTestRunAsync(List <string> resultFiles, ITestRunPublisher publisher, TestRunContext runContext, string resultReader, string runTitle, int?buildId, CancellationToken cancellationToken)
{
bool isTestRunOutcomeFailed = false;
try
{
//use local time since TestRunData defaults to local times
DateTime minStartDate = DateTime.MaxValue;
DateTime maxCompleteDate = DateTime.MinValue;
DateTime presentTime = DateTime.UtcNow;
bool dateFormatError = false;
TimeSpan totalTestCaseDuration = TimeSpan.Zero;
List <string> runAttachments = new List <string>();
List <TestCaseResultData> runResults = new List <TestCaseResultData>();
TestRunSummary testRunSummary = new TestRunSummary();
//read results from each file
foreach (string resultFile in resultFiles)
{
cancellationToken.ThrowIfCancellationRequested();
//test case results
_executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
TestRunData resultFileRunData = publisher.ReadResultsFromFile(runContext, resultFile);
isTestRunOutcomeFailed = isTestRunOutcomeFailed || GetTestRunOutcome(resultFileRunData, testRunSummary);
if (resultFileRunData != null)
{
if (resultFileRunData.Results != null && resultFileRunData.Results.Length > 0)
{
try
{
if (string.IsNullOrEmpty(resultFileRunData.StartDate) || string.IsNullOrEmpty(resultFileRunData.CompleteDate))
{
dateFormatError = true;
}
//As per discussion with Manoj(refer bug 565487): Test Run duration time should be minimum Start Time to maximum Completed Time when merging
if (!string.IsNullOrEmpty(resultFileRunData.StartDate))
{
DateTime startDate = DateTime.Parse(resultFileRunData.StartDate, null, DateTimeStyles.RoundtripKind);
minStartDate = minStartDate > startDate ? startDate : minStartDate;
if (!string.IsNullOrEmpty(resultFileRunData.CompleteDate))
{
DateTime endDate = DateTime.Parse(resultFileRunData.CompleteDate, null, DateTimeStyles.RoundtripKind);
maxCompleteDate = maxCompleteDate < endDate ? endDate : maxCompleteDate;
}
}
}
catch (FormatException)
{
_executionContext.Warning(StringUtil.Loc("InvalidDateFormat", resultFile, resultFileRunData.StartDate, resultFileRunData.CompleteDate));
dateFormatError = true;
}
//continue to calculate duration as a fallback for case: if there is issue with format or dates are null or empty
foreach (TestCaseResultData tcResult in resultFileRunData.Results)
{
int durationInMs = Convert.ToInt32(tcResult.DurationInMs);
totalTestCaseDuration = totalTestCaseDuration.Add(TimeSpan.FromMilliseconds(durationInMs));
}
runResults.AddRange(resultFileRunData.Results);
//run attachments
if (resultFileRunData.Attachments != null)
{
runAttachments.AddRange(resultFileRunData.Attachments);
}
}
else
{
_executionContext.Output(StringUtil.Loc("NoResultFound", resultFile));
}
}
else
{
_executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
}
}
//publish run if there are results.
if (runResults.Count > 0)
{
string runName = string.IsNullOrWhiteSpace(runTitle)
? StringUtil.Format("{0}_TestResults_{1}", _resultReader.Name, buildId)
: runTitle;
if (DateTime.Compare(minStartDate, maxCompleteDate) > 0)
{
_executionContext.Warning(StringUtil.Loc("InvalidCompletedDate", maxCompleteDate, minStartDate));
dateFormatError = true;
}
minStartDate = DateTime.Equals(minStartDate, DateTime.MaxValue) ? presentTime : minStartDate;
maxCompleteDate = dateFormatError || DateTime.Equals(maxCompleteDate, DateTime.MinValue) ? minStartDate.Add(totalTestCaseDuration) : maxCompleteDate;
// create test run
TestRunData testRunData = new TestRunData(
name: runName,
startedDate: minStartDate.ToString("o"),
completedDate: maxCompleteDate.ToString("o"),
state: "InProgress",
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: runContext != null ? runContext.Platform : string.Empty,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.PipelineReference = runContext.PipelineReference;
testRunData.Attachments = runAttachments.ToArray();
testRunData.AddCustomField(_testRunSystemCustomFieldName, runContext.TestRunSystem);
AddTargetBranchInfoToRunCreateModel(testRunData, runContext.TargetBranchName);
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, runResults.ToArray(), _executionContext.CancellationToken);
TestRun updatedRun = await publisher.EndTestRunAsync(testRunData, testRun.Id, true, _executionContext.CancellationToken);
// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
IList <TestRun> publishedRuns = new List <TestRun>();
publishedRuns.Add(updatedRun);
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
{
isTestRunOutcomeFailed = runOutcome.Value;
}
}
StoreTestRunSummaryInEnvVar(testRunSummary);
}
}
catch (Exception ex) when(!(ex is OperationCanceledException && _executionContext.CancellationToken.IsCancellationRequested))
{
// Not catching all the operationcancelled exceptions, as the pipeline cancellation should cancel the command as well.
// Do not fail the task.
LogPublishTestResultsFailureWarning(ex);
}
return(isTestRunOutcomeFailed);
}
public new TestRunData GetTestRunData(string filePath, XmlDocument doc, XmlNode testResultsNode, TestRunContext runContext, bool addResultsAsAttachments)
{
this.InnerTestSuiteNodeName = "results/test-suite";
this.TestCaseNodeName = "results/test-case";
this.RootNodeName = "test-results";
return(base.GetTestRunData(filePath, doc, testResultsNode, runContext, addResultsAsAttachments));
}
public TestRunData GetTestRunData(string filePath, XmlDocument doc, XmlNode testResultsNode, TestRunContext runContext, bool addResultsAsAttachments)
{
var testRunNode = doc.SelectSingleNode("test-run");
var testRunStartedOn = DateTime.MinValue;
var testRunEndedOn = DateTime.MinValue;
var testCaseResults = new List <TestCaseResultData>();
List <string> runLevelAttachments = new List <string>();
_runUserIdRef = runContext != null ? new IdentityRef()
{
DisplayName = runContext.Owner
} : null;
_platform = runContext != null ? runContext.Platform : string.Empty;
if (testRunNode.Attributes["start-time"] != null)
{
DateTime.TryParse(testRunNode.Attributes["start-time"]?.Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out testRunStartedOn);
DateTime.TryParse(testRunNode.Attributes["end-time"]?.Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out testRunEndedOn);
var testAssemblyNodes = testRunNode.SelectNodes("//test-suite[@type='Assembly']");
if (testAssemblyNodes != null)
{
foreach (XmlNode testAssemblyNode in testAssemblyNodes)
{
var environmentNode = testAssemblyNode.SelectSingleNode("environment");
var hostname = String.Empty;
var assemblyName = (testAssemblyNode.Attributes["name"] != null ? testAssemblyNode.Attributes["name"].Value : null);
if (environmentNode != null)
{
if (environmentNode.Attributes["machine-name"] != null)
{
hostname = environmentNode.Attributes["machine-name"].Value;
}
if (environmentNode.Attributes["platform"] != null && runContext?.BuildId > 0)
{
// override platform only if there's valid build
// We cannot publish platform information without a valid build id.
_platform = environmentNode.Attributes["platform"].Value;
}
}
var testCaseNodes = testAssemblyNode.SelectNodes(".//test-case");
if (testCaseNodes != null)
{
foreach (XmlNode testCaseNode in testCaseNodes)
{
testCaseResults.Add(getTestCaseResultData(testCaseNode, assemblyName, hostname));
}
}
}
}
// Adding run level attachments.
runLevelAttachments = this.GetTestRunLevelAttachments(testRunNode);
// Adding results file itself as run level attachment.
runLevelAttachments.Add(filePath);
}
TestRunData testRunData = new TestRunData(
name: runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName) ? runContext.RunName : _defaultRunName,
startedDate: (testRunStartedOn == DateTime.MinValue ? string.Empty : testRunStartedOn.ToString("o")),
completedDate: (testRunEndedOn == DateTime.MinValue ? string.Empty : testRunEndedOn.ToString("o")),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: _platform,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.Results = testCaseResults.ToArray();
testRunData.Attachments = addResultsAsAttachments ? runLevelAttachments.ToArray() : new string[0];
return(testRunData);
}
public TestRunPublisherTests()
{
_attachmentFilePath = "attachment.txt";
File.WriteAllText(_attachmentFilePath, "asdf");
_testRunContext = new TestRunContext("owner", "platform", "config", 1, "builduri", "releaseuri", "releaseenvuri");
_reader = new Mock <IResultReader>();
_reader.Setup(x => x.ReadResults(It.IsAny <IExecutionContext>(), It.IsAny <string>(), It.IsAny <TestRunContext>()))
.Callback <IExecutionContext, string, TestRunContext>
((executionContext, filePath, runContext) =>
{
_runContext = runContext;
_resultsFilepath = filePath;
})
.Returns((IExecutionContext executionContext, string filePath, TestRunContext runContext) =>
{
TestRunData trd = new TestRunData(
name: "xyz",
buildId: runContext.BuildId,
completedDate: "",
state: "InProgress",
isAutomated: true,
dueDate: "",
type: "",
buildFlavor: runContext.Configuration,
buildPlatform: runContext.Platform,
releaseUri: runContext.ReleaseUri,
releaseEnvironmentUri: runContext.ReleaseEnvironmentUri
);
trd.Attachments = new string[] { "attachment.txt" };
return(trd);
});
_testResultServer = new Mock <ITestResultsServer>();
_testResultServer.Setup(x => x.InitializeServer(It.IsAny <VssConnection>(), It.IsAny <IExecutionContext>()));
_testResultServer.Setup(x => x.AddTestResultsToTestRunAsync(It.IsAny <TestCaseResult[]>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <CancellationToken>()))
.Callback <TestCaseResult[], string, int, CancellationToken>
((currentBatch, projectName, testRunId, cancellationToken) =>
{
_batchSizes.Add(currentBatch.Length);
_resultCreateModels = currentBatch;
})
.Returns(() =>
{
List <TestCaseResult> resultsList = new List <TestCaseResult>();
int i = 0;
int j = 1;
foreach (TestCaseResult resultCreateModel in _resultCreateModels)
{
List <TestSubResult> SubResults = null;
if (resultCreateModel.SubResults != null)
{
SubResults = new List <TestSubResult>();
foreach (var subresultdata in resultCreateModel.SubResults)
{
var subResult = new TestSubResult();
subResult.Id = j++;
SubResults.Add(subResult);
}
}
resultsList.Add(new TestCaseResult()
{
Id = ++i, SubResults = SubResults
});
}
return(Task.FromResult(resultsList));
});
_testResultServer.Setup(x => x.CreateTestRunAsync(It.IsAny <string>(), It.IsAny <RunCreateModel>(), It.IsAny <CancellationToken>()))
.Callback <string, RunCreateModel, CancellationToken>
((projectName, testRunData, cancellationToken) =>
{
_projectId = projectName;
_testRun = (TestRunData)testRunData;
})
.Returns(Task.FromResult(new TestRun()
{
Name = "TestRun", Id = 1
}));
_testResultServer.Setup(x => x.UpdateTestRunAsync(It.IsAny <string>(), It.IsAny <int>(), It.IsAny <RunUpdateModel>(), It.IsAny <CancellationToken>()))
.Callback <string, int, RunUpdateModel, CancellationToken>
((projectName, testRunId, updateModel, cancellationToken) =>
{
_runId = testRunId;
_projectId = projectName;
_updateProperties = updateModel;
})
.Returns(Task.FromResult(new TestRun()
{
Name = "TestRun", Id = 1
}));
_testResultServer.Setup(x => x.CreateTestRunAttachmentAsync(
It.IsAny <TestAttachmentRequestModel>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <CancellationToken>()))
.Callback <TestAttachmentRequestModel, string, int, CancellationToken>
((reqModel, projectName, testRunId, cancellationToken) =>
{
_attachmentRequestModel = reqModel;
_projectId = projectName;
_runId = testRunId;
})
.Returns(Task.FromResult(new TestAttachmentReference()));
_testResultServer.Setup(x => x.CreateTestResultAttachmentAsync(It.IsAny <TestAttachmentRequestModel>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <int>(), It.IsAny <CancellationToken>()))
.Callback <TestAttachmentRequestModel, string, int, int, CancellationToken>
((reqModel, projectName, testRunId, testCaseResultId, cancellationToken) =>
{
if (_resultsLevelAttachments.ContainsKey(testCaseResultId))
{
_resultsLevelAttachments[testCaseResultId].Add(reqModel);
}
else
{
_resultsLevelAttachments.Add(testCaseResultId, new List <TestAttachmentRequestModel>()
{
reqModel
});
}
})
.Returns(Task.FromResult(new TestAttachmentReference()));
_testResultServer.Setup(x => x.CreateTestSubResultAttachmentAsync(It.IsAny <TestAttachmentRequestModel>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <int>(), It.IsAny <int>(), It.IsAny <CancellationToken>()))
.Callback <TestAttachmentRequestModel, string, int, int, int, CancellationToken>
((reqModel, projectName, testRunId, testCaseResultId, testSubResultId, cancellationToken) =>
{
if (_subResultsLevelAttachments.ContainsKey(testCaseResultId))
{
if (_subResultsLevelAttachments[testCaseResultId].ContainsKey(testSubResultId))
{
_subResultsLevelAttachments[testCaseResultId][testSubResultId].Add(reqModel);
}
else
{
_subResultsLevelAttachments[testCaseResultId].Add(testSubResultId, new List <TestAttachmentRequestModel>()
{
reqModel
});
}
}
else
{
Dictionary <int, List <TestAttachmentRequestModel> > subResulAtt = new Dictionary <int, List <TestAttachmentRequestModel> >();
subResulAtt.Add(testSubResultId, new List <TestAttachmentRequestModel>()
{
reqModel
});
_subResultsLevelAttachments.Add(testCaseResultId, subResulAtt);
}
})
.Returns(Task.FromResult(new TestAttachmentReference()));
}
//Based on the XUnit V2 format: http://xunit.github.io/docs/format-xml-v2.html
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
List <TestCaseResultData> results = new List <TestCaseResultData>();
XmlDocument doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Ignore
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return(null);
}
string runName = Name + " Test Run";
string runUser = "";
if (runContext != null)
{
if (!string.IsNullOrWhiteSpace(runContext.RunName))
{
runName = runContext.RunName;
}
else
{
runName = string.Format(CultureInfo.CurrentCulture, "{0} {1} {2}", runName, runContext.Configuration, runContext.Platform);
}
if (runContext.Owner != null)
{
runUser = runContext.Owner;
}
}
IdentityRef runUserIdRef = new IdentityRef();
runUserIdRef.DisplayName = runUser;
var minStartTime = DateTime.MaxValue;
var maxCompletedTime = DateTime.MinValue;
bool dateTimeParseError = true;
bool assemblyRunDateTimeAttributesNotPresent = false;
bool assemblyTimeAttributeNotPresent = false;
double assemblyRunDuration = 0;
double testRunDuration = 0;
XmlNodeList assemblyNodes = doc.SelectNodes("/assemblies/assembly");
foreach (XmlNode assemblyNode in assemblyNodes)
{
var assemblyRunStartTimeStamp = DateTime.MinValue;
if (assemblyNode.Attributes["run-date"] != null && assemblyNode.Attributes["run-time"] != null)
{
string runDate = assemblyNode.Attributes["run-date"].Value;
string runTime = assemblyNode.Attributes["run-time"].Value;
var startDate = DateTime.Now;
var startTime = TimeSpan.Zero;
if (DateTime.TryParse(runDate, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out startDate) &&
TimeSpan.TryParse(runTime, CultureInfo.InvariantCulture, out startTime))
{
dateTimeParseError = false;
}
assemblyRunStartTimeStamp = startDate + startTime;
if (minStartTime > assemblyRunStartTimeStamp)
{
minStartTime = assemblyRunStartTimeStamp;
}
}
else
{
assemblyRunDateTimeAttributesNotPresent = true;
}
if (!assemblyTimeAttributeNotPresent && assemblyNode.Attributes["time"] != null)
{
double assemblyDuration = 0;
Double.TryParse(assemblyNode.Attributes["time"].Value, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out assemblyDuration);
assemblyRunDuration = assemblyRunDuration + assemblyDuration;
var durationFromSeconds = TimeSpan.FromSeconds(assemblyDuration);
// no assemblystarttime available so dont calculate assemblycompletedtime
if (assemblyRunStartTimeStamp != DateTime.MinValue)
{
DateTime assemblyRunCompleteTimeStamp =
assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks);
//finding maximum comleted time
if (maxCompletedTime < assemblyRunCompleteTimeStamp)
{
maxCompletedTime = assemblyRunCompleteTimeStamp;
}
}
}
else
{
assemblyTimeAttributeNotPresent = true;
}
XmlNodeList testCaseNodeList = assemblyNode.SelectNodes("./collection/test");
foreach (XmlNode testCaseNode in testCaseNodeList)
{
TestCaseResultData resultCreateModel = new TestCaseResultData()
{
Priority = TestManagementConstants.UnspecifiedPriority, //Priority is int type so if no priority set then its 255.
};
//Test storage.
if (assemblyNode.Attributes["name"] != null)
{
resultCreateModel.AutomatedTestStorage = Path.GetFileName(assemblyNode.Attributes["name"].Value);
}
//Fully Qualified Name.
if (testCaseNode.Attributes["name"] != null)
{
resultCreateModel.AutomatedTestName = testCaseNode.Attributes["name"].Value;
}
//Test Method Name.
if (testCaseNode.Attributes["method"] != null)
{
resultCreateModel.TestCaseTitle = testCaseNode.Attributes["method"].Value;
}
//Test duration.
if (testCaseNode.Attributes["time"] != null && testCaseNode.Attributes["time"].Value != null)
{
double duration = 0;
double.TryParse(testCaseNode.Attributes["time"].Value, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out duration);
var durationFromSeconds = TimeSpan.FromSeconds(duration);
resultCreateModel.DurationInMs = durationFromSeconds.TotalMilliseconds;
// no assemblystarttime available so dont set testcase start and completed
if (assemblyRunStartTimeStamp != DateTime.MinValue)
{
resultCreateModel.StartedDate = assemblyRunStartTimeStamp;
resultCreateModel.CompletedDate = assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks);
assemblyRunStartTimeStamp = assemblyRunStartTimeStamp.AddTicks(1) + durationFromSeconds;
//next start time
}
//Calculate overall run duration.
testRunDuration += duration;
}
//Test outcome.
if (testCaseNode.SelectSingleNode("./failure") != null)
{
resultCreateModel.Outcome = TestOutcome.Failed.ToString();
//Error message.
XmlNode failureMessageNode = testCaseNode.SelectSingleNode("./failure/message");
if (failureMessageNode != null && !string.IsNullOrWhiteSpace(failureMessageNode.InnerText))
{
resultCreateModel.ErrorMessage = failureMessageNode.InnerText;
}
//Stack trace.
XmlNode failureStackTraceNode = testCaseNode.SelectSingleNode("./failure/stack-trace");
if (failureStackTraceNode != null && !string.IsNullOrWhiteSpace(failureStackTraceNode.InnerText))
{
resultCreateModel.StackTrace = failureStackTraceNode.InnerText;
}
// Console log
resultCreateModel.AttachmentData = new AttachmentData();
XmlNode consoleLog = testCaseNode.SelectSingleNode("./output");
if (consoleLog != null && !string.IsNullOrWhiteSpace(consoleLog.InnerText))
{
resultCreateModel.AttachmentData.ConsoleLog = consoleLog.InnerText;
}
}
else if (testCaseNode.Attributes["result"] != null && string.Equals(testCaseNode.Attributes["result"].Value, "pass", StringComparison.OrdinalIgnoreCase))
{
resultCreateModel.Outcome = TestOutcome.Passed.ToString();
}
else
{
resultCreateModel.Outcome = TestOutcome.NotExecuted.ToString();
}
//Test priority.
XmlNode priorityTrait = testCaseNode.SelectSingleNode("./traits/trait[@name='priority']");
if (priorityTrait != null && priorityTrait.Attributes["value"] != null)
{
var priorityValue = priorityTrait.Attributes["value"].Value;
resultCreateModel.Priority = !string.IsNullOrEmpty(priorityValue) ? Convert.ToInt32(priorityValue)
: TestManagementConstants.UnspecifiedPriority;
}
//Test owner.
XmlNode ownerNode = testCaseNode.SelectSingleNode("./traits/trait[@name='owner']");
if (ownerNode != null && ownerNode.Attributes["value"] != null && ownerNode.Attributes["value"].Value != null)
{
IdentityRef ownerIdRef = new IdentityRef();
ownerIdRef.DisplayName = ownerNode.Attributes["value"].Value;
ownerIdRef.DirectoryAlias = ownerNode.Attributes["value"].Value;
resultCreateModel.Owner = ownerIdRef;
}
resultCreateModel.RunBy = runUserIdRef;
resultCreateModel.State = "Completed";
resultCreateModel.AutomatedTestType = Name;
if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle))
{
results.Add(resultCreateModel);
}
}
}
if (dateTimeParseError || assemblyRunDateTimeAttributesNotPresent)
{
executionContext.Warning("Atleast for one assembly start time was not obtained due to tag not present or parsing issue, total run duration will now be summation of time taken by each assembly");
if (assemblyTimeAttributeNotPresent)
{
executionContext.Warning("Atleast for one assembly time tag is not present, total run duration will now be summation of time from all test runs");
}
}
//if minimum start time is not available then set it to present time
minStartTime = minStartTime == DateTime.MaxValue ? DateTime.UtcNow : minStartTime;
//if start time cannot be obtained even for one assembly then fallback duration to sum of assembly run time
//if assembly run time cannot be obtained even for one assembly then fallback duration to total test run
maxCompletedTime = dateTimeParseError || assemblyRunDateTimeAttributesNotPresent || maxCompletedTime == DateTime.MinValue ? minStartTime.Add(assemblyTimeAttributeNotPresent ? TimeSpan.FromSeconds(testRunDuration) : TimeSpan.FromSeconds(assemblyRunDuration)) : maxCompletedTime;
executionContext.Output(string.Format("Obtained XUnit Test Run Start Date: {0} and Completed Date: {1}", minStartTime.ToString("o"), maxCompletedTime.ToString("o")));
TestRunData testRunData = new TestRunData(
name: runName,
buildId: runContext != null ? runContext.BuildId : 0,
startedDate: minStartTime != DateTime.MinValue ? minStartTime.ToString("o") : null,
completedDate: maxCompletedTime != DateTime.MinValue ? maxCompletedTime.ToString("o") : null,
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildFlavor: runContext != null ? runContext.Configuration : null,
buildPlatform: runContext != null ? runContext.Platform : null,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.Results = results.ToArray();
testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];
return(testRunData);
}