private void ProcessPublishTestResultsCommand(IExecutionContext context, Dictionary<string, string> eventProperties, string data)
{
ArgUtil.NotNull(context, nameof(context));
_executionContext = context;
LoadPublishTestResultsInputs(eventProperties, data);
string teamProject = context.Variables.System_TeamProject;
string owner = context.Variables.Build_RequestedFor;
string buildUri = context.Variables.Build_BuildUri;
int buildId = context.Variables.Build_BuildId ?? 0;
//Temporary fix to support publish in RM scenarios where there might not be a valid Build ID associated.
//TODO: Make a cleaner fix after TCM User Story 401703 is completed.
if (buildId == 0)
{
_platform = _configuration = null;
}
string releaseUri = null;
string releaseEnvironmentUri = null;
// Check to identify if we are in the Release management flow; if not, then release fields will be kept null while publishing to TCM
if (!string.IsNullOrWhiteSpace(context.Variables.Release_ReleaseUri))
{
releaseUri = context.Variables.Release_ReleaseUri;
releaseEnvironmentUri = context.Variables.Release_ReleaseEnvironmentUri;
}
IResultReader resultReader = GetTestResultReader(_testRunner);
TestRunContext runContext = new TestRunContext(owner, _platform, _configuration, buildId, buildUri, releaseUri, releaseEnvironmentUri);
Client.VssConnection connection = WorkerUtilies.GetVssConnection(_executionContext);
var publisher = HostContext.GetService<ITestRunPublisher>();
publisher.InitializePublisher(context, connection, teamProject, resultReader);
var commandContext = HostContext.CreateService<IAsyncCommandContext>();
commandContext.InitializeCommandContext(context, StringUtil.Loc("PublishTestResults"));
if (_mergeResults)
{
commandContext.Task = PublishAllTestResultsToSingleTestRunAsync(_testResultFiles, publisher, buildId, runContext, resultReader.Name, context.CancellationToken);
}
else
{
commandContext.Task = PublishToNewTestRunPerTestResultFileAsync(_testResultFiles, publisher, runContext, resultReader.Name, context.CancellationToken);
}
_executionContext.AsyncCommands.Add(commandContext);
}
public TestRunPublisherTests()
{
_attachmentFilePath = "attachment.txt";
File.WriteAllText(_attachmentFilePath, "asdf");
_testRunContext = new TestRunContext("owner", "platform", "config", 1, "builduri", "releaseuri", "releaseenvuri");
_reader = new Mock<IResultReader>();
_reader.Setup(x => x.ReadResults(It.IsAny<IExecutionContext>(), It.IsAny<string>(), It.IsAny<TestRunContext>()))
.Callback<IExecutionContext, string, TestRunContext>
((executionContext, filePath, runContext) =>
{
_runContext = runContext;
_resultsFilepath = filePath;
})
.Returns((IExecutionContext executionContext, string filePath, TestRunContext runContext) =>
{
TestRunData trd = new TestRunData(
name: "xyz",
buildId: runContext.BuildId,
completedDate: "",
state: "InProgress",
isAutomated: true,
dueDate: "",
type: "",
buildFlavor: runContext.Configuration,
buildPlatform: runContext.Platform,
releaseUri: runContext.ReleaseUri,
releaseEnvironmentUri: runContext.ReleaseEnvironmentUri
);
trd.Attachments = new string[] { "attachment.txt" };
return trd;
});
_testResultServer = new Mock<ITestResultsServer>();
_testResultServer.Setup(x => x.InitializeServer(It.IsAny<Client.VssConnection>()));
_testResultServer.Setup(x => x.AddTestResultsToTestRunAsync(It.IsAny<TestResultCreateModel[]>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
.Callback<TestResultCreateModel[], string, int, CancellationToken>
((currentBatch, projectName, testRunId, cancellationToken) =>
{
_batchSizes.Add(currentBatch.Length);
_resultCreateModels = currentBatch;
})
.Returns(() =>
{
List<TestCaseResult> resultsList = new List<TestCaseResult>();
int i = 0;
foreach (TestResultCreateModel resultCreateModel in _resultCreateModels)
{
resultsList.Add(new TestCaseResult() { Id = ++i });
}
return Task.FromResult(resultsList);
});
_testResultServer.Setup(x => x.CreateTestRunAsync(It.IsAny<string>(), It.IsAny<RunCreateModel>(), It.IsAny<CancellationToken>()))
.Callback<string, RunCreateModel, CancellationToken>
((projectName, testRunData, cancellationToken) =>
{
_projectId = projectName;
_testRun = (TestRunData)testRunData;
})
.Returns(Task.FromResult(new TestRun() { Name = "TestRun", Id = 1 }));
_testResultServer.Setup(x => x.UpdateTestRunAsync(It.IsAny<string>(), It.IsAny<int>(), It.IsAny<RunUpdateModel>(), It.IsAny<CancellationToken>()))
.Callback<string, int, RunUpdateModel, CancellationToken>
((projectName, testRunId, updateModel, cancellationToken) =>
{
_runId = testRunId;
_projectId = projectName;
_updateProperties = updateModel;
})
.Returns(Task.FromResult(new TestRun() { Name = "TestRun", Id = 1 }));
_testResultServer.Setup(x => x.CreateTestRunAttachmentAsync(
It.IsAny<TestAttachmentRequestModel>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
.Callback<TestAttachmentRequestModel, string, int, CancellationToken>
((reqModel, projectName, testRunId, cancellationToken) =>
{
_attachmentRequestModel = reqModel;
_projectId = projectName;
_runId = testRunId;
})
.Returns(Task.FromResult(new TestAttachmentReference()));
_testResultServer.Setup(x => x.CreateTestResultAttachmentAsync(It.IsAny<TestAttachmentRequestModel>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
.Callback<TestAttachmentRequestModel, string, int, int, CancellationToken>
((reqModel, projectName, testRunId, testCaseResultId, cancellationToken) =>
{
if (_resultsLevelAttachments.ContainsKey(testCaseResultId))
{
_resultsLevelAttachments[testCaseResultId].Add(reqModel);
}
else
{
_resultsLevelAttachments.Add(testCaseResultId, new List<TestAttachmentRequestModel>() { reqModel });
}
})
.Returns(Task.FromResult(new TestAttachmentReference()));
}
/// <summary>
/// Reads a JUnit results file from disk, converts it into a TestRunData object.
/// </summary>
/// <param name="filePath"></param>
/// <param name="runContext"></param>
/// <returns></returns>
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
// http://windyroad.com.au/dl/Open%20Source/JUnit.xsd
XmlDocument doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Prohibit
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return null;
}
//init test run summary information - run name, host name, start time
TestSuiteSummary runSummary = new TestSuiteSummary(Name);
IdentityRef runUserIdRef = null;
string runUser = runContext != null ? runContext.Owner : string.Empty;
if (!string.IsNullOrEmpty(runUser))
{
runUserIdRef = new IdentityRef() { DisplayName = runUser };
}
//read data from testsuite nodes
XmlNode testSuitesNode = doc.SelectSingleNode("testsuites");
if (testSuitesNode != null)
{
//found testsuites node - some plugins generate it like karma junit plugin
XmlNodeList testSuiteNodeList = doc.SelectNodes("/testsuites/testsuite");
if (testSuiteNodeList != null)
{
foreach (XmlNode testSuiteNode in testSuiteNodeList)
{
TestSuiteSummary testSuiteSummary = ReadTestSuite(testSuiteNode, runUserIdRef);
runSummary.Duration = runSummary.Duration.Add(testSuiteSummary.Duration);
runSummary.Results.AddRange(testSuiteSummary.Results);
runSummary.Host = testSuiteSummary.Host;
runSummary.Name = testSuiteSummary.Name;
}
if (testSuiteNodeList.Count > 1)
{
runSummary.Name = Name + "_" + Path.GetFileName(filePath);
}
}
}
else
{
XmlNode testSuiteNode = doc.SelectSingleNode("testsuite");
if (testSuiteNode != null)
{
runSummary = ReadTestSuite(testSuiteNode, runUserIdRef);
}
}
if (runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName))
{
runSummary.Name = runContext.RunName;
}
if (runSummary.Results.Count > 0)
{
//first testsuite starteddate is the starteddate of the run
runSummary.TimeStamp = DateTime.Parse(runSummary.Results[0].StartedDate);
}
//create test run data
var testRunData = new TestRunData(
name: runSummary.Name,
startedDate: runSummary.TimeStamp.ToString("o"),
completedDate: runSummary.TimeStamp.Add(runSummary.Duration).ToString("o"),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: runContext != null ? runContext.Platform : string.Empty,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
)
{
Results = runSummary.Results.ToArray(),
Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0]
};
return testRunData;
}
private async Task PublishTestRunDataAsync(VssConnection connection, String teamProject, TestRunContext testRunContext)
{
bool isTestRunOutcomeFailed = false;
var featureFlagService = HostContext.GetService <IFeatureFlagService>();
featureFlagService.InitializeFeatureService(_executionContext, connection);
var publishTestResultsLibFeatureState = featureFlagService.GetFeatureFlagState(TestResultsConstants.UsePublishTestResultsLibFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
_telemetryProperties.Add("UsePublishTestResultsLib", publishTestResultsLibFeatureState);
//This check is to determine to use "Microsoft.TeamFoundation.PublishTestResults" Library or the agent code to parse and publish the test results.
if (publishTestResultsLibFeatureState)
{
var publisher = HostContext.GetService <ITestDataPublisher>();
publisher.InitializePublisher(_executionContext, teamProject, connection, _testRunner);
isTestRunOutcomeFailed = await publisher.PublishAsync(testRunContext, _testResultFiles, GetPublishOptions(), _executionContext.CancellationToken);
}
else
{
var publisher = HostContext.GetService <ILegacyTestRunDataPublisher>();
publisher.InitializePublisher(_executionContext, teamProject, connection, _testRunner, _publishRunLevelAttachments);
isTestRunOutcomeFailed = await publisher.PublishAsync(testRunContext, _testResultFiles, _runTitle, _executionContext.Variables.Build_BuildId, _mergeResults);
}
if (isTestRunOutcomeFailed && _failTaskOnFailedTests)
{
_executionContext.Result = TaskResult.Failed;
_executionContext.Error(StringUtil.Loc("FailedTestsInResults"));
}
await PublishEventsAsync(connection);
}
//Based on the XUnit V2 format: http://xunit.github.io/docs/format-xml-v2.html
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
List <TestCaseResultData> results = new List <TestCaseResultData>();
XmlDocument doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Ignore
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return(null);
}
string runName = Name + " Test Run";
string runUser = "";
if (runContext != null)
{
if (!string.IsNullOrWhiteSpace(runContext.RunName))
{
runName = runContext.RunName;
}
else
{
runName = string.Format(CultureInfo.CurrentCulture, "{0} {1} {2}", runName, runContext.Configuration, runContext.Platform);
}
if (runContext.Owner != null)
{
runUser = runContext.Owner;
}
}
IdentityRef runUserIdRef = new IdentityRef();
runUserIdRef.DisplayName = runUser;
var minStartTime = DateTime.MaxValue;
var maxCompletedTime = DateTime.MinValue;
bool dateTimeParseError = true;
bool assemblyRunDateTimeAttributesNotPresent = false;
bool assemblyTimeAttributeNotPresent = false;
double assemblyRunDuration = 0;
double testRunDuration = 0;
XmlNodeList assemblyNodes = doc.SelectNodes("/assemblies/assembly");
foreach (XmlNode assemblyNode in assemblyNodes)
{
var assemblyRunStartTimeStamp = DateTime.MinValue;
if (assemblyNode.Attributes["run-date"] != null && assemblyNode.Attributes["run-time"] != null)
{
string runDate = assemblyNode.Attributes["run-date"].Value;
string runTime = assemblyNode.Attributes["run-time"].Value;
var startDate = DateTime.Now;
var startTime = TimeSpan.Zero;
if (DateTime.TryParse(runDate, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.None, out startDate) &&
TimeSpan.TryParse(runTime, CultureInfo.InvariantCulture, out startTime))
{
dateTimeParseError = false;
}
assemblyRunStartTimeStamp = startDate + startTime;
if (minStartTime > assemblyRunStartTimeStamp)
{
minStartTime = assemblyRunStartTimeStamp;
}
}
else
{
assemblyRunDateTimeAttributesNotPresent = true;
}
if (!assemblyTimeAttributeNotPresent && assemblyNode.Attributes["time"] != null)
{
double assemblyDuration = 0;
Double.TryParse(assemblyNode.Attributes["time"].Value, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out assemblyDuration);
assemblyRunDuration = assemblyRunDuration + assemblyDuration;
var durationFromSeconds = TimeSpan.FromSeconds(assemblyDuration);
// no assemblystarttime available so dont calculate assemblycompletedtime
if (assemblyRunStartTimeStamp != DateTime.MinValue)
{
DateTime assemblyRunCompleteTimeStamp =
assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks);
//finding maximum comleted time
if (maxCompletedTime < assemblyRunCompleteTimeStamp)
{
maxCompletedTime = assemblyRunCompleteTimeStamp;
}
}
}
else
{
assemblyTimeAttributeNotPresent = true;
}
XmlNodeList testCaseNodeList = assemblyNode.SelectNodes("./collection/test");
foreach (XmlNode testCaseNode in testCaseNodeList)
{
TestCaseResultData resultCreateModel = new TestCaseResultData()
{
Priority = TestManagementConstants.UnspecifiedPriority, //Priority is int type so if no priority set then its 255.
};
//Test storage.
if (assemblyNode.Attributes["name"] != null)
{
resultCreateModel.AutomatedTestStorage = Path.GetFileName(assemblyNode.Attributes["name"].Value);
}
//Fully Qualified Name.
if (testCaseNode.Attributes["name"] != null)
{
resultCreateModel.AutomatedTestName = testCaseNode.Attributes["name"].Value;
}
//Test Method Name.
if (testCaseNode.Attributes["method"] != null)
{
resultCreateModel.TestCaseTitle = testCaseNode.Attributes["method"].Value;
}
//Test duration.
if (testCaseNode.Attributes["time"] != null && testCaseNode.Attributes["time"].Value != null)
{
double duration = 0;
double.TryParse(testCaseNode.Attributes["time"].Value, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out duration);
var durationFromSeconds = TimeSpan.FromSeconds(duration);
resultCreateModel.DurationInMs = durationFromSeconds.TotalMilliseconds;
// no assemblystarttime available so dont set testcase start and completed
if (assemblyRunStartTimeStamp != DateTime.MinValue)
{
resultCreateModel.StartedDate = assemblyRunStartTimeStamp;
resultCreateModel.CompletedDate = assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks);
assemblyRunStartTimeStamp = assemblyRunStartTimeStamp.AddTicks(1) + durationFromSeconds;
//next start time
}
//Calculate overall run duration.
testRunDuration += duration;
}
//Test outcome.
if (testCaseNode.SelectSingleNode("./failure") != null)
{
resultCreateModel.Outcome = TestOutcome.Failed.ToString();
//Error message.
XmlNode failureMessageNode = testCaseNode.SelectSingleNode("./failure/message");
if (failureMessageNode != null && !string.IsNullOrWhiteSpace(failureMessageNode.InnerText))
{
resultCreateModel.ErrorMessage = failureMessageNode.InnerText;
}
//Stack trace.
XmlNode failureStackTraceNode = testCaseNode.SelectSingleNode("./failure/stack-trace");
if (failureStackTraceNode != null && !string.IsNullOrWhiteSpace(failureStackTraceNode.InnerText))
{
resultCreateModel.StackTrace = failureStackTraceNode.InnerText;
}
}
else if (testCaseNode.Attributes["result"] != null && string.Equals(testCaseNode.Attributes["result"].Value, "pass", StringComparison.OrdinalIgnoreCase))
{
resultCreateModel.Outcome = TestOutcome.Passed.ToString();
}
else
{
resultCreateModel.Outcome = TestOutcome.NotExecuted.ToString();
}
//Test priority.
XmlNode priorityTrait = testCaseNode.SelectSingleNode("./traits/trait[@name='priority']");
if (priorityTrait != null && priorityTrait.Attributes["value"] != null)
{
var priorityValue = priorityTrait.Attributes["value"].Value;
resultCreateModel.Priority = !string.IsNullOrEmpty(priorityValue) ? Convert.ToInt32(priorityValue)
: TestManagementConstants.UnspecifiedPriority;
}
//Test owner.
XmlNode ownerNode = testCaseNode.SelectSingleNode("./traits/trait[@name='owner']");
if (ownerNode != null && ownerNode.Attributes["value"] != null && ownerNode.Attributes["value"].Value != null)
{
IdentityRef ownerIdRef = new IdentityRef();
ownerIdRef.DisplayName = ownerNode.Attributes["value"].Value;
ownerIdRef.DirectoryAlias = ownerNode.Attributes["value"].Value;
resultCreateModel.Owner = ownerIdRef;
}
resultCreateModel.RunBy = runUserIdRef;
resultCreateModel.State = "Completed";
resultCreateModel.AutomatedTestType = Name;
if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle))
{
results.Add(resultCreateModel);
}
}
}
if (dateTimeParseError || assemblyRunDateTimeAttributesNotPresent)
{
executionContext.Warning("Atleast for one assembly start time was not obtained due to tag not present or parsing issue, total run duration will now be summation of time taken by each assembly");
if (assemblyTimeAttributeNotPresent)
{
executionContext.Warning("Atleast for one assembly time tag is not present, total run duration will now be summation of time from all test runs");
}
}
//if minimum start time is not available then set it to present time
minStartTime = minStartTime == DateTime.MaxValue ? DateTime.UtcNow : minStartTime;
//if start time cannot be obtained even for one assembly then fallback duration to sum of assembly run time
//if assembly run time cannot be obtained even for one assembly then fallback duration to total test run
maxCompletedTime = dateTimeParseError || assemblyRunDateTimeAttributesNotPresent || maxCompletedTime == DateTime.MinValue ? minStartTime.Add(assemblyTimeAttributeNotPresent ? TimeSpan.FromSeconds(testRunDuration) : TimeSpan.FromSeconds(assemblyRunDuration)) : maxCompletedTime;
executionContext.Output(string.Format("Obtained XUnit Test Run Start Date: {0} and Completed Date: {1}", minStartTime.ToString("o"), maxCompletedTime.ToString("o")));
TestRunData testRunData = new TestRunData(
name: runName,
buildId: runContext != null ? runContext.BuildId : 0,
startedDate: minStartTime.ToString("o"),
completedDate: maxCompletedTime.ToString("o"),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildFlavor: runContext != null ? runContext.Configuration : null,
buildPlatform: runContext != null ? runContext.Platform : null,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.Results = results.ToArray();
testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];
return(testRunData);
}
/// <summary>
/// Reads a trx file from disk, converts it into a TestRunData object.
/// </summary>
/// <param name="filePath">File path</param>
/// <returns>TestRunData</returns>
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext)
{
_executionContext = executionContext;
_definitions = new Dictionary <string, TestCaseDefinition>();
List <TestCaseResultData> results = new List <TestCaseResultData>();
string xmlContents = File.ReadAllText(filePath);
xmlContents = xmlContents.Replace("xmlns", "ns");
XmlDocument doc = new XmlDocument();
doc.LoadXml(xmlContents);
string runName = Name + " Test Run";
if (runContext != null)
{
if (!String.IsNullOrWhiteSpace(runContext.RunName))
{
runName = runContext.RunName;
}
else
{
runName = StringUtil.Format("{0} {1} {2}", runName, runContext.Configuration, runContext.Platform);
}
}
string runUser = "";
if (runContext.Owner != null)
{
runUser = runContext.Owner;
}
_runUserIdRef = new IdentityRef();
_runUserIdRef.DisplayName = runUser;
//Parse the run start and finish times. If either of it is not available, send neither.
XmlNode node = doc.SelectSingleNode("/TestRun/Times");
DateTime runStartDate = DateTime.MinValue;
DateTime runFinishDate = DateTime.MinValue;
if (node != null && node.Attributes["start"] != null && node.Attributes["finish"] != null)
{
if (DateTime.TryParse(node.Attributes["start"].Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out runStartDate))
{
_executionContext.Debug(string.Format(CultureInfo.InvariantCulture, "Setting run start and finish times."));
//Only if there is a valid start date.
DateTime.TryParse(node.Attributes["finish"].Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out runFinishDate);
if (runFinishDate < runStartDate)
{
runFinishDate = runStartDate = DateTime.MinValue;
_executionContext.Debug("Run finish date is less than start date. Resetting to min value.");
}
}
}
TestRunData testRunData = new TestRunData(
name: runName,
buildId: runContext.BuildId,
startedDate: runStartDate != DateTime.MinValue ? runStartDate.ToString("o") : null,
completedDate: runFinishDate != DateTime.MinValue ? runFinishDate.ToString("o") : null,
state: TestRunState.InProgress.ToString(),
isAutomated: true,
dueDate: string.Empty,
type: string.Empty,
buildFlavor: runContext.Configuration,
buildPlatform: runContext.Platform,
releaseUri: runContext.ReleaseUri,
releaseEnvironmentUri: runContext.ReleaseEnvironmentUri
);
//Parse the Deployment node for the runDeploymentRoot - this is the attachment root. Required for .NET Core
XmlNode deploymentNode = doc.SelectSingleNode("/TestRun/TestSettings/Deployment");
var _attachmentRoot = string.Empty;
if (deploymentNode != null && deploymentNode.Attributes["runDeploymentRoot"] != null)
{
_attachmentRoot = deploymentNode.Attributes["runDeploymentRoot"].Value;
}
else
{
_attachmentRoot = Path.GetFileNameWithoutExtension(filePath); // This for platform v1
}
_attachmentLocation = Path.Combine(Path.GetDirectoryName(filePath), _attachmentRoot, "In");
_executionContext.Debug(string.Format(CultureInfo.InvariantCulture, "Attachment location: {0}", _attachmentLocation));
AddRunLevelAttachments(filePath, doc, testRunData);
// Create a dictionary of testcase definitions to be used when iterating over the results
Dictionary <string, TestCaseDefinition> definitions = new Dictionary <string, TestCaseDefinition>();
if (doc.SelectNodes("/TestRun/TestDefinitions").Count > 0)
{
foreach (XmlNode definitionNode in doc.SelectNodes("/TestRun/TestDefinitions")[0])
{
IdentityRef owner = null;
string priority = null, storage = null;
if (definitionNode.Attributes["storage"] != null && definitionNode.Attributes["storage"].Value != null)
{
storage = Path.GetFileName(definitionNode.Attributes["storage"].Value);
}
XmlAttribute priorityAttribute = definitionNode.Attributes["priority"];
if (priorityAttribute != null)
{
priority = priorityAttribute.Value;
}
XmlNode ownerNode = definitionNode.SelectSingleNode("./Owners/Owner");
if (ownerNode != null)
{
IdentityRef ownerIdRef = new IdentityRef();
ownerIdRef.DisplayName = ownerNode.Attributes["name"].Value;
ownerIdRef.DirectoryAlias = ownerNode.Attributes["name"].Value;
owner = ownerIdRef;
}
// The automated test name should be FQDN, if we are unable to figure it out like in case of a webtest,
// set it as "name" from the parent (where it is always present)
XmlNode testResultNode = definitionNode.SelectSingleNode("./TestMethod");
string automatedTestName = null;
if (testResultNode != null && testResultNode.Attributes["className"] != null && testResultNode.Attributes["name"] != null)
{
// At times the class names are coming as
// className="MS.TF.Test.AgileX.VSTests.WiLinking.UI.WiLinkingUIQueryTests"
// at other times, they are as
// className="UnitTestProject3.UnitTest1, UnitTestProject3, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null"
string className = testResultNode.Attributes["className"].Value.Split(',')[0];
automatedTestName = className + "." + testResultNode.Attributes["name"].Value;
}
else if (definitionNode.Attributes["name"] != null)
{
automatedTestName = definitionNode.Attributes["name"].Value;
}
_definitions.Add(definitionNode.Attributes["id"].Value, new TestCaseDefinition(automatedTestName, owner, priority, storage));
}
}
// Read UnitTestResults, WebTestResults and OrderedTestResults
XmlNodeList resultsNodes = doc.SelectNodes("/TestRun/Results/UnitTestResult");
XmlNodeList webTestResultNodes = doc.SelectNodes("/TestRun/Results/WebTestResult");
XmlNodeList orderedTestResultNodes = doc.SelectNodes("/TestRun/Results/TestResultAggregation");
results.AddRange(ReadActualResults(resultsNodes, TestType.UnitTest));
results.AddRange(ReadActualResults(webTestResultNodes, TestType.WebTest));
results.AddRange(ReadActualResults(orderedTestResultNodes, TestType.OrderedTest));
testRunData.Results = results.ToArray <TestCaseResultData>();
_executionContext.Debug(string.Format(CultureInfo.InvariantCulture, "Total test results: {0}", testRunData.Results.Length));
return(testRunData);
}
/// <summary>
/// Publish separate test run for each result file that has results.
/// </summary>
private async Task PublishToNewTestRunPerTestResultFileAsync(List <string> resultFiles,
ITestRunPublisher publisher,
TestRunContext runContext,
string resultReader,
int batchSize,
CancellationToken cancellationToken)
{
try
{
var groupedFiles = resultFiles
.Select((resultFile, index) => new { Index = index, file = resultFile })
.GroupBy(pair => pair.Index / batchSize)
.Select(bucket => bucket.Select(pair => pair.file).ToList())
.ToList();
bool changeTestRunTitle = resultFiles.Count > 1;
foreach (var files in groupedFiles)
{
// Publish separate test run for each result file that has results.
var publishTasks = files.Select(async resultFile =>
{
cancellationToken.ThrowIfCancellationRequested();
string runName = _runTitle;
if (!string.IsNullOrWhiteSpace(_runTitle) && changeTestRunTitle)
{
runName = GetRunTitle();
}
_executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
TestRunData testRunData = publisher.ReadResultsFromFile(runContext, resultFile, runName);
testRunData.PipelineReference = GetTestPipelineReference(runContext);
if (_failTaskOnFailedTests)
{
_isTestRunOutcomeFailed = _isTestRunOutcomeFailed || GetTestRunOutcome(testRunData);
}
cancellationToken.ThrowIfCancellationRequested();
if (testRunData != null)
{
if (testRunData.Results != null && testRunData.Results.Length > 0)
{
testRunData.AddCustomField(_testRunSystemCustomFieldName, _testRunSystem);
AddTargetBranchInfoToRunCreateModel(testRunData, runContext.PullRequestTargetBranchName);
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, testRunData.Results, _executionContext.CancellationToken);
await publisher.EndTestRunAsync(testRunData, testRun.Id, cancellationToken: _executionContext.CancellationToken);
}
else
{
_executionContext.Output(StringUtil.Loc("NoResultFound", resultFile));
}
}
else
{
_executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
}
});
await Task.WhenAll(publishTasks);
}
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
//Do not fail the task.
LogPublishTestResultsFailureWarning(ex);
}
}
private async Task UploadTestBuildLog(Guid projectId, BuildAttachment buildAttachment, TestRunContext runContext, CancellationToken cancellationToken)
{
await _testLogStore.UploadTestBuildLogAsync(projectId, runContext.BuildId, buildAttachment.TestLogType, buildAttachment.Filename, buildAttachment.Metadata, null, buildAttachment.AllowDuplicateUploads, buildAttachment.TestLogCompressionType, cancellationToken);
}
/// <summary>
/// Reads a JUnit results file from disk, converts it into a TestRunData object.
/// </summary>
/// <param name="filePath"></param>
/// <param name="runContext"></param>
/// <returns></returns>
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
// http://windyroad.com.au/dl/Open%20Source/JUnit.xsd
XmlDocument doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Ignore
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return(null);
}
//init test run summary information - run name, host name, start time
TestSuiteSummary runSummary = new TestSuiteSummary(Name);
IdentityRef runUserIdRef = null;
string runUser = runContext != null ? runContext.Owner : string.Empty;
if (!string.IsNullOrEmpty(runUser))
{
runUserIdRef = new IdentityRef()
{
DisplayName = runUser
};
}
var presentTime = DateTime.UtcNow;
runSummary.TimeStamp = DateTime.MaxValue;
var maxCompletedTime = DateTime.MinValue;
//read data from testsuite nodes
XmlNode testSuitesNode = doc.SelectSingleNode("testsuites");
if (testSuitesNode != null)
{
//found testsuites node - some plugins generate it like karma junit plugin
XmlNodeList testSuiteNodeList = doc.SelectNodes("/testsuites/testsuite");
if (testSuiteNodeList != null)
{
foreach (XmlNode testSuiteNode in testSuiteNodeList)
{
//for each available suites get all suite details
TestSuiteSummary testSuiteSummary = ReadTestSuite(testSuiteNode, runUserIdRef);
// sum up testsuite durations and test case durations, decision on what to use will be taken later
runSummary.TotalTestCaseDuration = runSummary.TotalTestCaseDuration.Add(testSuiteSummary.TotalTestCaseDuration);
runSummary.TestSuiteDuration = runSummary.TestSuiteDuration.Add(testSuiteSummary.TestSuiteDuration);
runSummary.SuiteTimeDataAvailable = runSummary.SuiteTimeDataAvailable && testSuiteSummary.SuiteTimeDataAvailable;
runSummary.SuiteTimeStampAvailable = runSummary.SuiteTimeStampAvailable && testSuiteSummary.SuiteTimeStampAvailable;
runSummary.Host = testSuiteSummary.Host;
runSummary.Name = testSuiteSummary.Name;
//stop calculating timestamp information, if timestamp data is not avilable for even one test suite
if (testSuiteSummary.SuiteTimeStampAvailable)
{
runSummary.TimeStamp = runSummary.TimeStamp > testSuiteSummary.TimeStamp ? testSuiteSummary.TimeStamp : runSummary.TimeStamp;
DateTime completedTime = testSuiteSummary.TimeStamp.AddTicks(testSuiteSummary.TestSuiteDuration.Ticks);
maxCompletedTime = maxCompletedTime < completedTime ? completedTime : maxCompletedTime;
}
runSummary.Results.AddRange(testSuiteSummary.Results);
}
if (testSuiteNodeList.Count > 1)
{
runSummary.Name = Name + "_" + Path.GetFileName(filePath);
}
}
}
else
{
executionContext.Output("Only single test suite found, parsing its information");
XmlNode testSuiteNode = doc.SelectSingleNode("testsuite");
if (testSuiteNode != null)
{
runSummary = ReadTestSuite(testSuiteNode, runUserIdRef);
//only if start time is available then only we need to calculate completed time
if (runSummary.TimeStamp != DateTime.MaxValue)
{
DateTime completedTime = runSummary.TimeStamp.AddTicks(runSummary.TestSuiteDuration.Ticks);
maxCompletedTime = maxCompletedTime < completedTime ? completedTime : maxCompletedTime;
}
else
{
runSummary.SuiteTimeStampAvailable = false;
}
}
}
if (runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName))
{
runSummary.Name = runContext.RunName;
}
if (!runSummary.SuiteTimeStampAvailable)
{
executionContext.Output("Timestamp is not available for one or more testsuites. Total run duration is being calculated as the sum of time durations of detected testsuites");
if (!runSummary.SuiteTimeDataAvailable)
{
executionContext.Output("Time is not available for one or more testsuites. Total run duration is being calculated as the sum of time durations of detected testcases");
}
}
//if start time is not calculated then it should be initialized as present time
runSummary.TimeStamp = runSummary.TimeStamp == DateTime.MaxValue
? presentTime
: runSummary.TimeStamp;
//if suite timestamp data is not available even for single testsuite, then fallback to testsuite run time
//if testsuite run time is not available even for single testsuite, then fallback to total test case duration
maxCompletedTime = !runSummary.SuiteTimeStampAvailable || maxCompletedTime == DateTime.MinValue
? runSummary.TimeStamp.Add(runSummary.SuiteTimeDataAvailable ? runSummary.TestSuiteDuration
: runSummary.TotalTestCaseDuration) : maxCompletedTime;
//create test run data
var testRunData = new TestRunData(
name: runSummary.Name,
startedDate: runSummary.TimeStamp != DateTime.MinValue ? runSummary.TimeStamp.ToString("o") : null,
completedDate: maxCompletedTime != DateTime.MinValue ? maxCompletedTime.ToString("o") : null,
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: runContext != null ? runContext.Platform : string.Empty,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
)
{
Results = runSummary.Results.ToArray(),
Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0]
};
return(testRunData);
}
/// <summary>
/// Publish single test run
/// </summary>
private async Task PublishAllTestResultsToSingleTestRunAsync(List <string> resultFiles, ITestRunPublisher publisher, int buildId, TestRunContext runContext, string resultReader, CancellationToken cancellationToken)
{
try
{
//use local time since TestRunData defaults to local times
DateTime minStartDate = DateTime.MaxValue;
DateTime maxCompleteDate = DateTime.MinValue;
DateTime presentTime = DateTime.UtcNow;
bool dateFormatError = false;
TimeSpan totalTestCaseDuration = TimeSpan.Zero;
List <string> runAttachments = new List <string>();
List <TestCaseResultData> runResults = new List <TestCaseResultData>();
//read results from each file
foreach (string resultFile in resultFiles)
{
cancellationToken.ThrowIfCancellationRequested();
//test case results
_executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
TestRunData resultFileRunData = publisher.ReadResultsFromFile(runContext, resultFile);
if (_failTaskOnFailedTests)
{
_isTestRunOutcomeFailed = _isTestRunOutcomeFailed || GetTestRunOutcome(resultFileRunData);
}
if (resultFileRunData != null)
{
if (resultFileRunData.Results != null && resultFileRunData.Results.Length > 0)
{
try
{
if (string.IsNullOrEmpty(resultFileRunData.StartDate) || string.IsNullOrEmpty(resultFileRunData.CompleteDate))
{
dateFormatError = true;
}
//As per discussion with Manoj(refer bug 565487): Test Run duration time should be minimum Start Time to maximum Completed Time when merging
if (!string.IsNullOrEmpty(resultFileRunData.StartDate))
{
DateTime startDate = DateTime.Parse(resultFileRunData.StartDate, null, DateTimeStyles.RoundtripKind);
minStartDate = minStartDate > startDate ? startDate : minStartDate;
if (!string.IsNullOrEmpty(resultFileRunData.CompleteDate))
{
DateTime endDate = DateTime.Parse(resultFileRunData.CompleteDate, null, DateTimeStyles.RoundtripKind);
maxCompleteDate = maxCompleteDate < endDate ? endDate : maxCompleteDate;
}
}
}
catch (FormatException)
{
_executionContext.Warning(StringUtil.Loc("InvalidDateFormat", resultFile, resultFileRunData.StartDate, resultFileRunData.CompleteDate));
dateFormatError = true;
}
//continue to calculate duration as a fallback for case: if there is issue with format or dates are null or empty
foreach (TestCaseResultData tcResult in resultFileRunData.Results)
{
int durationInMs = Convert.ToInt32(tcResult.DurationInMs);
totalTestCaseDuration = totalTestCaseDuration.Add(TimeSpan.FromMilliseconds(durationInMs));
}
runResults.AddRange(resultFileRunData.Results);
//run attachments
if (resultFileRunData.Attachments != null)
{
runAttachments.AddRange(resultFileRunData.Attachments);
}
}
else
{
_executionContext.Output(StringUtil.Loc("NoResultFound", resultFile));
}
}
else
{
_executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
}
}
//publish run if there are results.
if (runResults.Count > 0)
{
string runName = string.IsNullOrWhiteSpace(_runTitle)
? StringUtil.Format("{0}_TestResults_{1}", _testRunner, buildId)
: _runTitle;
if (DateTime.Compare(minStartDate, maxCompleteDate) > 0)
{
_executionContext.Warning(StringUtil.Loc("InvalidCompletedDate", maxCompleteDate, minStartDate));
dateFormatError = true;
}
minStartDate = DateTime.Equals(minStartDate, DateTime.MaxValue) ? presentTime : minStartDate;
maxCompleteDate = dateFormatError || DateTime.Equals(maxCompleteDate, DateTime.MinValue) ? minStartDate.Add(totalTestCaseDuration) : maxCompleteDate;
//creat test run
TestRunData testRunData = new TestRunData(
name: runName,
startedDate: minStartDate.ToString("o"),
completedDate: maxCompleteDate.ToString("o"),
state: "InProgress",
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: runContext != null ? runContext.Platform : string.Empty,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.PipelineReference = GetTestPipelineReference(runContext);
testRunData.Attachments = runAttachments.ToArray();
testRunData.AddCustomField(_testRunSystemCustomFieldName, _testRunSystem);
AddTargetBranchInfoToRunCreateModel(testRunData, runContext.PullRequestTargetBranchName);
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, runResults.ToArray(), _executionContext.CancellationToken);
await publisher.EndTestRunAsync(testRunData, testRun.Id, true, _executionContext.CancellationToken);
}
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
//Do not fail the task.
LogPublishTestResultsFailureWarning(ex);
}
}
/// <summary>
/// Publish single test run
/// </summary>
private async Task PublishAllTestResultsToSingleTestRunAsync(List <string> resultFiles, ITestRunPublisher publisher, int buildId, TestRunContext runContext, string resultReader, CancellationToken cancellationToken)
{
try
{
DateTime startTime = DateTime.Now; //use local time since TestRunData defaults to local times
TimeSpan totalTestCaseDuration = TimeSpan.Zero;
List <string> runAttachments = new List <string>();
List <TestCaseResultData> runResults = new List <TestCaseResultData>();
//read results from each file
foreach (string resultFile in resultFiles)
{
cancellationToken.ThrowIfCancellationRequested();
//test case results
_executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
TestRunData resultFileRunData = publisher.ReadResultsFromFile(runContext, resultFile);
if (resultFileRunData != null && resultFileRunData.Results != null && resultFileRunData.Results.Length > 0)
{
foreach (TestCaseResultData tcResult in resultFileRunData.Results)
{
int durationInMs = 0;
int.TryParse(tcResult.DurationInMs, out durationInMs);
totalTestCaseDuration = totalTestCaseDuration.Add(TimeSpan.FromMilliseconds(durationInMs));
}
runResults.AddRange(resultFileRunData.Results);
//run attachments
if (resultFileRunData.Attachments != null)
{
runAttachments.AddRange(resultFileRunData.Attachments);
}
}
else
{
_executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
}
}
string runName = string.IsNullOrWhiteSpace(_runTitle)
? StringUtil.Format("{0}_TestResults_{1}", _testRunner, buildId)
: _runTitle;
//creat test run
TestRunData testRunData = new TestRunData(
name: runName,
startedDate: startTime.ToString("o"),
completedDate: startTime.Add(totalTestCaseDuration).ToString("o"),
state: "InProgress",
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: runContext != null ? runContext.Platform : string.Empty
);
testRunData.Attachments = runAttachments.ToArray();
//publish run if there are results.
if (runResults.Count > 0)
{
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, runResults.ToArray(), _executionContext.CancellationToken);
await publisher.EndTestRunAsync(testRunData, testRun.Id, true, _executionContext.CancellationToken);
}
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
//Do not fail the task.
LogPublishTestResultsFailureWarning(ex);
}
}
/// <summary>
/// Publish separate test run for each result file that has results.
/// </summary>
private async Task PublishToNewTestRunPerTestResultFileAsync(List <string> resultFiles, ITestRunPublisher publisher, TestRunContext runContext, string resultReader, CancellationToken cancellationToken)
{
try
{
// Publish separate test run for each result file that has results.
var publishTasks = resultFiles.Select(async resultFile =>
{
cancellationToken.ThrowIfCancellationRequested();
string runName = null;
if (!string.IsNullOrWhiteSpace(_runTitle))
{
runName = GetRunTitle();
}
_executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
TestRunData testRunData = publisher.ReadResultsFromFile(runContext, resultFile, runName);
cancellationToken.ThrowIfCancellationRequested();
if (testRunData != null && testRunData.Results != null && testRunData.Results.Length > 0)
{
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, testRunData.Results, _executionContext.CancellationToken);
await publisher.EndTestRunAsync(testRunData, testRun.Id, cancellationToken: _executionContext.CancellationToken);
}
else
{
_executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
}
});
await Task.WhenAll(publishTasks);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
//Do not fail the task.
LogPublishTestResultsFailureWarning(ex);
}
}
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext)
{
try
{
string jsonTestSummary = File.ReadAllText(filePath);
if (string.IsNullOrWhiteSpace(jsonTestSummary))
{
return(null);
}
JsonTestSummary testSummary = StringUtil.ConvertFromJson <JsonTestSummary>(jsonTestSummary);
// Adding the minimum details from the JSON.
TestRunData testRunData = new TestRunData(name: "Container Structure Test",
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: runContext != null ? runContext.Platform : string.Empty,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null);
List <TestCaseResultData> results = new List <TestCaseResultData>();
foreach (JsonTestResult result in testSummary.Results)
{
TestCaseResultData resultCreateModel = new TestCaseResultData();
resultCreateModel.TestCaseTitle = result.Name;
resultCreateModel.AutomatedTestName = result.Name;
bool outcome = result.Pass.Equals("true", StringComparison.OrdinalIgnoreCase);
if (!outcome)
{
resultCreateModel.ErrorMessage = string.Join("|", result.Errors);
}
resultCreateModel.State = "Completed";
resultCreateModel.AutomatedTestType = Name;
resultCreateModel.Outcome = outcome ? TestOutcome.Passed.ToString() : TestOutcome.Failed.ToString();
results.Add(resultCreateModel);
}
testRunData.Results = results.ToArray();
return(testRunData);
}
catch (Exception ex)
{
executionContext.Output("Error occured in reading results : " + ex);
}
return(null);
}
//Based on the XUnit V2 format: http://xunit.github.io/docs/format-xml-v2.html
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
List <TestCaseResultData> results = new List <TestCaseResultData>();
XmlDocument doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Prohibit
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return(null);
}
string runName = Name + " Test Run";
string runUser = "";
if (runContext != null)
{
if (!string.IsNullOrWhiteSpace(runContext.RunName))
{
runName = runContext.RunName;
}
else
{
runName = string.Format(CultureInfo.CurrentCulture, "{0} {1} {2}", runName, runContext.Configuration, runContext.Platform);
}
if (runContext.Owner != null)
{
runUser = runContext.Owner;
}
}
IdentityRef runUserIdRef = new IdentityRef();
runUserIdRef.DisplayName = runUser;
var runStartTime = DateTime.MinValue;
double testRunDuration = 0;
XmlNodeList assemblyNodes = doc.SelectNodes("/assemblies/assembly");
foreach (XmlNode assemblyNode in assemblyNodes)
{
var assemblyRunStartTimeStamp = DateTime.MinValue;
if (assemblyNode.Attributes["run-date"] != null && assemblyNode.Attributes["run-time"] != null)
{
string runDate = assemblyNode.Attributes["run-date"].Value;
string runTime = assemblyNode.Attributes["run-time"].Value;
var startDate = DateTime.Now;
var startTime = TimeSpan.Zero;
if (DateTime.TryParse(runDate, out startDate))
{
TimeSpan.TryParse(runTime, out startTime);
}
assemblyRunStartTimeStamp = startDate + startTime;
if (testRunDuration == 0)
{
// first assembly start time is runstart time
runStartTime = assemblyRunStartTimeStamp;
}
}
XmlNodeList testCaseNodeList = assemblyNode.SelectNodes("./collection/test");
foreach (XmlNode testCaseNode in testCaseNodeList)
{
TestCaseResultData resultCreateModel = new TestCaseResultData();
//Test storage.
if (assemblyNode.Attributes["name"] != null)
{
resultCreateModel.AutomatedTestStorage = Path.GetFileName(assemblyNode.Attributes["name"].Value);
}
//Fully Qualified Name.
if (testCaseNode.Attributes["name"] != null)
{
resultCreateModel.AutomatedTestName = testCaseNode.Attributes["name"].Value;
}
//Test Method Name.
if (testCaseNode.Attributes["method"] != null)
{
resultCreateModel.TestCaseTitle = testCaseNode.Attributes["method"].Value;
}
//Test duration.
if (testCaseNode.Attributes["time"] != null && testCaseNode.Attributes["time"].Value != null)
{
double duration = 0;
double.TryParse(testCaseNode.Attributes["time"].Value, out duration);
var durationFromSeconds = TimeSpan.FromSeconds(duration);
resultCreateModel.DurationInMs = durationFromSeconds.TotalMilliseconds.ToString(CultureInfo.InvariantCulture);
// no assemblystarttime available so dont set testcase start and completed
if (assemblyRunStartTimeStamp != DateTime.MinValue)
{
resultCreateModel.StartedDate = assemblyRunStartTimeStamp.ToString("o");
resultCreateModel.CompletedDate =
assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks).ToString("o");
assemblyRunStartTimeStamp = assemblyRunStartTimeStamp.AddTicks(1) + durationFromSeconds;
//next start time
}
//Calculate overall run duration.
testRunDuration += duration;
}
//Test outcome.
if (testCaseNode.SelectSingleNode("./failure") != null)
{
resultCreateModel.Outcome = TestOutcome.Failed.ToString();
//Error message.
XmlNode failureMessageNode = testCaseNode.SelectSingleNode("./failure/message");
if (failureMessageNode != null && !string.IsNullOrWhiteSpace(failureMessageNode.InnerText))
{
resultCreateModel.ErrorMessage = failureMessageNode.InnerText;
}
//Stack trace.
XmlNode failureStackTraceNode = testCaseNode.SelectSingleNode("./failure/stack-trace");
if (failureStackTraceNode != null && !string.IsNullOrWhiteSpace(failureStackTraceNode.InnerText))
{
resultCreateModel.StackTrace = failureStackTraceNode.InnerText;
}
}
else if (testCaseNode.Attributes["result"] != null && string.Equals(testCaseNode.Attributes["result"].Value, "pass", StringComparison.OrdinalIgnoreCase))
{
resultCreateModel.Outcome = TestOutcome.Passed.ToString();
}
else
{
resultCreateModel.Outcome = TestOutcome.NotExecuted.ToString();
}
//Test priority.
XmlNode priorityTrait = testCaseNode.SelectSingleNode("./traits/trait[@name='priority']");
if (priorityTrait != null && priorityTrait.Attributes["value"] != null)
{
resultCreateModel.TestCasePriority = priorityTrait.Attributes["value"].Value;
}
//Test owner.
XmlNode ownerNode = testCaseNode.SelectSingleNode("./traits/trait[@name='owner']");
if (ownerNode != null && ownerNode.Attributes["value"] != null && ownerNode.Attributes["value"].Value != null)
{
IdentityRef ownerIdRef = new IdentityRef();
ownerIdRef.DisplayName = ownerNode.Attributes["value"].Value;
resultCreateModel.Owner = ownerIdRef;
}
resultCreateModel.RunBy = runUserIdRef;
resultCreateModel.State = "Completed";
resultCreateModel.AutomatedTestType = Name;
if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle))
{
results.Add(resultCreateModel);
}
}
}
TestRunData testRunData = new TestRunData(
name: runName,
buildId: runContext != null ? runContext.BuildId : 0,
startedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.ToString("o"),
completedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.Add(TimeSpan.FromSeconds(testRunDuration)).ToString("o"),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildFlavor: runContext != null ? runContext.Configuration : null,
buildPlatform: runContext != null ? runContext.Platform : null,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.Results = results.ToArray();
testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];
return(testRunData);
}
private void ProcessPublishTestResultsCommand(IExecutionContext context, Dictionary <string, string> eventProperties, string data)
{
ArgUtil.NotNull(context, nameof(context));
_executionContext = context;
LoadPublishTestResultsInputs(context, eventProperties, data);
string teamProject = context.Variables.System_TeamProject;
string owner = context.Variables.Build_RequestedFor;
string buildUri = context.Variables.Build_BuildUri;
int buildId = context.Variables.Build_BuildId ?? 0;
string pullRequestTargetBranchName = context.Variables.System_PullRequest_TargetBranch;
string stageName = context.Variables.System_StageName;
string phaseName = context.Variables.System_PhaseName;
string jobName = context.Variables.System_JobName;
int stageAttempt = context.Variables.System_StageAttempt ?? 0;
int phaseAttempt = context.Variables.System_PhaseAttempt ?? 0;
int jobAttempt = context.Variables.System_JobAttempt ?? 0;
//Temporary fix to support publish in RM scenarios where there might not be a valid Build ID associated.
//TODO: Make a cleaner fix after TCM User Story 401703 is completed.
if (buildId == 0)
{
_platform = _configuration = null;
}
string releaseUri = null;
string releaseEnvironmentUri = null;
// Check to identify if we are in the Release management flow; if not, then release fields will be kept null while publishing to TCM
if (!string.IsNullOrWhiteSpace(context.Variables.Release_ReleaseUri))
{
releaseUri = context.Variables.Release_ReleaseUri;
releaseEnvironmentUri = context.Variables.Release_ReleaseEnvironmentUri;
}
IResultReader resultReader = GetTestResultReader(_testRunner);
TestRunContext runContext = new TestRunContext(owner, _platform, _configuration, buildId, buildUri, releaseUri, releaseEnvironmentUri);
runContext.StageName = stageName;
runContext.StageAttempt = stageAttempt;
runContext.PhaseName = phaseName;
runContext.PhaseAttempt = phaseAttempt;
runContext.JobName = jobName;
runContext.JobAttempt = jobAttempt;
runContext.PullRequestTargetBranchName = pullRequestTargetBranchName;
VssConnection connection = WorkerUtilities.GetVssConnection(_executionContext);
var publisher = HostContext.GetService <ITestRunPublisher>();
publisher.InitializePublisher(context, connection, teamProject, resultReader);
var commandContext = HostContext.CreateService <IAsyncCommandContext>();
commandContext.InitializeCommandContext(context, StringUtil.Loc("PublishTestResults"));
if (_mergeResults)
{
commandContext.Task = PublishAllTestResultsToSingleTestRunAsync(_testResultFiles, publisher, buildId, runContext, resultReader.Name, context.CancellationToken);
}
else
{
commandContext.Task = PublishToNewTestRunPerTestResultFileAsync(_testResultFiles, publisher, runContext, resultReader.Name, PublishBatchSize, context.CancellationToken);
}
_executionContext.AsyncCommands.Add(commandContext);
if (_isTestRunOutcomeFailed)
{
_executionContext.Result = TaskResult.Failed;
_executionContext.Error(StringUtil.Loc("FailedTestsInResults"));
}
}
private async Task UploadRunDataAttachment(TestRunContext runContext, List <TestRunData> testRunData, PublishOptions publishOptions, CancellationToken cancellationToken = default(CancellationToken))
{
await _testRunPublisher.PublishTestRunDataAsync(runContext, _projectName, testRunData, publishOptions, cancellationToken);
}
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
List<TestCaseResultData> results = new List<TestCaseResultData>();
XmlDocument doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Prohibit
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return null;
}
//read test run summary information - run name, start time
string runName = Name + " Test Run";
DateTime runStartTime = DateTime.MinValue; //Use local time instead of UTC as TestRunData uses local time for defaults. Also assuming timestamp is local is more accurate in cases where tests were run on build machine
TimeSpan totalRunDuration = TimeSpan.Zero;
TimeSpan totalTestCaseDuration = TimeSpan.Zero;
XmlNode testResultsNode = doc.SelectSingleNode("test-results");
if (testResultsNode != null)
{
//get test run summary information
if (testResultsNode.Attributes["name"] != null)
{
runName = testResultsNode.Attributes["name"].Value;
}
//run times
DateTime dateFromXml = DateTime.MinValue.Date; //Use local time instead of UTC as TestRunData uses local time for defaults.
if (testResultsNode.Attributes["date"] != null)
{
DateTime.TryParse(testResultsNode.Attributes["date"].Value, out dateFromXml);
}
TimeSpan timeFromXml = TimeSpan.Zero;
if (testResultsNode.Attributes["time"] != null)
{
TimeSpan.TryParse(testResultsNode.Attributes["time"].Value, out timeFromXml);
}
//assume runtimes from xml are current local time since timezone information is not in the xml, if xml datetime > current local time, fallback to local start time
DateTime runStartDateTimeFromXml = new DateTime(dateFromXml.Ticks).AddTicks(timeFromXml.Ticks);
if (runStartTime == DateTime.MinValue)
{
runStartTime = runStartDateTimeFromXml;
}
}
//run environment - platform, config and hostname
string platform = runContext != null ? runContext.Platform : string.Empty;
string config = runContext != null ? runContext.Configuration : string.Empty;
string runUser = runContext != null ? runContext.Owner : string.Empty;
string hostName = string.Empty;
XmlNode envNode = doc.SelectSingleNode("test-results/environment");
if (envNode != null)
{
if (envNode.Attributes["machine-name"] != null && envNode.Attributes["machine-name"].Value != null)
{
hostName = envNode.Attributes["machine-name"].Value;
}
if (envNode.Attributes["platform"] != null && envNode.Attributes["platform"].Value != null && runContext != null && runContext.BuildId > 0)
{
//We cannot publish platform information without a valid build id.
platform = envNode.Attributes["platform"].Value;
}
}
//run owner
IdentityRef runUserIdRef = null;
if (!string.IsNullOrEmpty(runUser))
{
runUserIdRef = new IdentityRef() { DisplayName = runUser };
}
//get all test assemblies
if (testResultsNode != null)
{
XmlNodeList testAssemblyNodes = testResultsNode.SelectNodes("test-suite");
if (testAssemblyNodes != null)
{
foreach (XmlNode testAssemblyNode in testAssemblyNodes)
{
var assemblyStartTime = (runStartTime == DateTime.MinValue) ? DateTime.MinValue : runStartTime + totalTestCaseDuration;
List<TestCaseResultData> testCases = FindTestCaseNodes(testAssemblyNode, hostName, runUserIdRef, assemblyStartTime);
if (testCases != null)
{
results.AddRange(testCases);
testCases.ForEach(x => totalTestCaseDuration += TimeSpan.FromMilliseconds(double.Parse(x.DurationInMs)));
}
}
}
}
if (TimeSpan.Compare(totalRunDuration, totalTestCaseDuration) < 0)
{
totalRunDuration = totalTestCaseDuration; //run duration may not be set in the xml, so use total test case duration
}
if (runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName))
{
runName = runContext.RunName;
}
//create test run data
TestRunData testRunData = new TestRunData(
name: runName,
startedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.ToString("o"),
completedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.Add(totalRunDuration).ToString("o"),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: config,
buildPlatform: platform,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.Results = results.ToArray();
testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];
return testRunData;
}
/// <summary>
/// Reads a CTest results file from disk, converts it into a TestRunData object.
/// </summary>
/// <param name="filePath"></param>
/// <param name="runContext"></param>
/// <returns></returns>
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
_ec = executionContext;
// Read Xml File
XmlDocument doc = new XmlDocument();
try
{
XmlReaderSettings settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Ignore
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return(null);
}
// init test run summary information - run name, host name, start time
List <TestCaseResultData> results = new List <TestCaseResultData>();
string runName = ParseRunName(runContext);
InitialiseRunUserIdRef(runContext);
//run environment - platform, config and hostname
string platform = runContext != null ? runContext.Platform : string.Empty;
string config = runContext != null ? runContext.Configuration : string.Empty;
string runUser = runContext != null ? runContext.Owner : string.Empty;
string hostName = string.Empty;
//Parse the run start and finish times.
XmlNode node = doc.SelectSingleNode("/Site/Testing");
_runStartDate = DateTime.MinValue;
_runFinishDate = DateTime.MinValue;
ParseRunStartAndFinishDates(node);
//create test run data object
TestRunData testRunData = new TestRunData(
name: runName,
startedDate: (_runStartDate == DateTime.MinValue) ? string.Empty : _runStartDate.ToString("o"),
completedDate: (_runFinishDate == DateTime.MinValue) ? string.Empty : _runFinishDate.ToString("o"),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: config,
buildPlatform: platform,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
)
{
Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0]
};
// Read results
XmlNodeList resultsNodes = doc.SelectNodes("/Site/Testing/Test");
results.AddRange(ReadActualResults(resultsNodes));
testRunData.Results = results.ToArray();
return(testRunData);
}
public new TestRunData GetTestRunData(string filePath, XmlDocument doc, XmlNode testResultsNode, TestRunContext runContext, bool addResultsAsAttachments)
{
this.InnerTestSuiteNodeName = "results/test-suite";
this.TestCaseNodeName = "results/test-case";
this.RootNodeName = "test-results";
return(base.GetTestRunData(filePath, doc, testResultsNode, runContext, addResultsAsAttachments));
}
//Based on the XUnit V2 format: http://xunit.github.io/docs/format-xml-v2.html
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
List<TestCaseResultData> results = new List<TestCaseResultData>();
XmlDocument doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Prohibit
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return null;
}
string runName = Name + " Test Run";
string runUser = "";
if (runContext != null)
{
if (!string.IsNullOrWhiteSpace(runContext.RunName))
{
runName = runContext.RunName;
}
else
{
runName = string.Format(CultureInfo.CurrentCulture, "{0} {1} {2}", runName, runContext.Configuration, runContext.Platform);
}
if (runContext.Owner != null)
{
runUser = runContext.Owner;
}
}
IdentityRef runUserIdRef = new IdentityRef();
runUserIdRef.DisplayName = runUser;
var runStartTime = DateTime.MinValue;
double testRunDuration = 0;
XmlNodeList assemblyNodes = doc.SelectNodes("/assemblies/assembly");
foreach (XmlNode assemblyNode in assemblyNodes)
{
var assemblyRunStartTimeStamp = DateTime.MinValue;
if (assemblyNode.Attributes["run-date"] != null && assemblyNode.Attributes["run-time"] != null)
{
string runDate = assemblyNode.Attributes["run-date"].Value;
string runTime = assemblyNode.Attributes["run-time"].Value;
var startDate = DateTime.Now;
var startTime = TimeSpan.Zero;
if (DateTime.TryParse(runDate, out startDate))
{
TimeSpan.TryParse(runTime, out startTime);
}
assemblyRunStartTimeStamp = startDate + startTime;
if (testRunDuration == 0)
{
// first assembly start time is runstart time
runStartTime = assemblyRunStartTimeStamp;
}
}
XmlNodeList testCaseNodeList = assemblyNode.SelectNodes("./collection/test");
foreach (XmlNode testCaseNode in testCaseNodeList)
{
TestCaseResultData resultCreateModel = new TestCaseResultData();
//Test storage.
if (assemblyNode.Attributes["name"] != null)
{
resultCreateModel.AutomatedTestStorage = Path.GetFileName(assemblyNode.Attributes["name"].Value);
}
//Fully Qualified Name.
if (testCaseNode.Attributes["name"] != null)
{
resultCreateModel.AutomatedTestName = testCaseNode.Attributes["name"].Value;
}
//Test Method Name.
if (testCaseNode.Attributes["method"] != null)
{
resultCreateModel.TestCaseTitle = testCaseNode.Attributes["method"].Value;
}
//Test duration.
if (testCaseNode.Attributes["time"] != null && testCaseNode.Attributes["time"].Value != null)
{
double duration = 0;
double.TryParse(testCaseNode.Attributes["time"].Value, out duration);
var durationFromSeconds = TimeSpan.FromSeconds(duration);
resultCreateModel.DurationInMs = durationFromSeconds.TotalMilliseconds.ToString(CultureInfo.InvariantCulture);
// no assemblystarttime available so dont set testcase start and completed
if (assemblyRunStartTimeStamp != DateTime.MinValue)
{
resultCreateModel.StartedDate = assemblyRunStartTimeStamp.ToString("o");
resultCreateModel.CompletedDate =
assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks).ToString("o");
assemblyRunStartTimeStamp = assemblyRunStartTimeStamp.AddTicks(1) + durationFromSeconds;
//next start time
}
//Calculate overall run duration.
testRunDuration += duration;
}
//Test outcome.
if (testCaseNode.SelectSingleNode("./failure") != null)
{
resultCreateModel.Outcome = TestOutcome.Failed.ToString();
//Error message.
XmlNode failureMessageNode = testCaseNode.SelectSingleNode("./failure/message");
if (failureMessageNode != null && !string.IsNullOrWhiteSpace(failureMessageNode.InnerText))
{
resultCreateModel.ErrorMessage = failureMessageNode.InnerText;
}
//Stack trace.
XmlNode failureStackTraceNode = testCaseNode.SelectSingleNode("./failure/stack-trace");
if (failureStackTraceNode != null && !string.IsNullOrWhiteSpace(failureStackTraceNode.InnerText))
{
resultCreateModel.StackTrace = failureStackTraceNode.InnerText;
}
}
else if (testCaseNode.Attributes["result"] != null && string.Equals(testCaseNode.Attributes["result"].Value, "pass", StringComparison.OrdinalIgnoreCase))
{
resultCreateModel.Outcome = TestOutcome.Passed.ToString();
}
else
{
resultCreateModel.Outcome = TestOutcome.NotExecuted.ToString();
}
//Test priority.
XmlNode priorityTrait = testCaseNode.SelectSingleNode("./traits/trait[@name='priority']");
if (priorityTrait != null && priorityTrait.Attributes["value"] != null)
{
resultCreateModel.TestCasePriority = priorityTrait.Attributes["value"].Value;
}
//Test owner.
XmlNode ownerNode = testCaseNode.SelectSingleNode("./traits/trait[@name='owner']");
if (ownerNode != null && ownerNode.Attributes["value"] != null && ownerNode.Attributes["value"].Value != null)
{
IdentityRef ownerIdRef = new IdentityRef();
ownerIdRef.DisplayName = ownerNode.Attributes["value"].Value;
resultCreateModel.Owner = ownerIdRef;
}
resultCreateModel.RunBy = runUserIdRef;
resultCreateModel.State = "Completed";
resultCreateModel.AutomatedTestType = Name;
if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle))
{
results.Add(resultCreateModel);
}
}
}
TestRunData testRunData = new TestRunData(
name: runName,
buildId: runContext != null ? runContext.BuildId : 0,
startedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.ToString("o"),
completedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.Add(TimeSpan.FromSeconds(testRunDuration)).ToString("o"),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildFlavor: runContext != null ? runContext.Configuration : null,
buildPlatform: runContext != null ? runContext.Platform : null,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.Results = results.ToArray();
testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];
return testRunData;
}
public TestRunData GetTestRunData(string filePath, XmlDocument doc, XmlNode testResultsNode, TestRunContext runContext, bool addResultsAsAttachments)
{
var testRunNode = doc.SelectSingleNode("test-run");
var testRunStartedOn = DateTime.MinValue;
var testRunEndedOn = DateTime.MinValue;
var testCaseResults = new List <TestCaseResultData>();
_runUserIdRef = runContext != null ? new IdentityRef()
{
DisplayName = runContext.Owner
} : null;
_platform = runContext != null ? runContext.Platform : string.Empty;
if (testRunNode.Attributes["start-time"] != null)
{
DateTime.TryParse(testRunNode.Attributes["start-time"]?.Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.None, out testRunStartedOn);
DateTime.TryParse(testRunNode.Attributes["end-time"]?.Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.None, out testRunEndedOn);
var testAssemblyNodes = testRunNode.SelectNodes("//test-suite[@type='Assembly']");
if (testAssemblyNodes != null)
{
foreach (XmlNode testAssemblyNode in testAssemblyNodes)
{
var environmentNode = testAssemblyNode.SelectSingleNode("environment");
var hostname = String.Empty;
var assemblyName = (testAssemblyNode.Attributes["name"] != null ? testAssemblyNode.Attributes["name"].Value : null);
if (environmentNode != null)
{
if (environmentNode.Attributes["machine-name"] != null)
{
hostname = environmentNode.Attributes["machine-name"].Value;
}
if (environmentNode.Attributes["platform"] != null)
{
// override platform
_platform = environmentNode.Attributes["platform"].Value;
}
}
var testCaseNodes = testAssemblyNode.SelectNodes(".//test-case");
if (testCaseNodes != null)
{
foreach (XmlNode testCaseNode in testCaseNodes)
{
testCaseResults.Add(getTestCaseResultData(testCaseNode, assemblyName, hostname));
}
}
}
}
}
TestRunData testRunData = new TestRunData(
name: runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName) ? runContext.RunName : _defaultRunName,
startedDate: (testRunStartedOn == DateTime.MinValue ? string.Empty : testRunStartedOn.ToString("o")),
completedDate: (testRunEndedOn == DateTime.MinValue ? string.Empty : testRunEndedOn.ToString("o")),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: _platform,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.Results = testCaseResults.ToArray();
testRunData.Attachments = addResultsAsAttachments ? new string[] { filePath } : new string[0];
return(testRunData);
}
private TestRunContext CreateTestRunContext()
{
string releaseUri = null;
string releaseEnvironmentUri = null;
string teamProject = _executionContext.Variables.System_TeamProject;
string owner = _executionContext.Variables.Build_RequestedFor;
string buildUri = _executionContext.Variables.Build_BuildUri;
int buildId = _executionContext.Variables.Build_BuildId ?? 0;
string pullRequestTargetBranchName = _executionContext.Variables.System_PullRequest_TargetBranch;
string stageName = _executionContext.Variables.System_StageName;
string phaseName = _executionContext.Variables.System_PhaseName;
string jobName = _executionContext.Variables.System_JobName;
int stageAttempt = _executionContext.Variables.System_StageAttempt ?? 0;
int phaseAttempt = _executionContext.Variables.System_PhaseAttempt ?? 0;
int jobAttempt = _executionContext.Variables.System_JobAttempt ?? 0;
//Temporary fix to support publish in RM scenarios where there might not be a valid Build ID associated.
//TODO: Make a cleaner fix after TCM User Story 401703 is completed.
if (buildId == 0)
{
_platform = _configuration = null;
}
if (!string.IsNullOrWhiteSpace(_executionContext.Variables.Release_ReleaseUri))
{
releaseUri = _executionContext.Variables.Release_ReleaseUri;
releaseEnvironmentUri = _executionContext.Variables.Release_ReleaseEnvironmentUri;
}
// If runName is not provided by the task, then create runName from testRunner name and buildId.
string runName = String.IsNullOrWhiteSpace(_runTitle)
? String.Format("{0}_TestResults_{1}", _testRunner, buildId)
: _runTitle;
StageReference stageReference = new StageReference()
{
StageName = stageName, Attempt = Convert.ToInt32(stageAttempt)
};
PhaseReference phaseReference = new PhaseReference()
{
PhaseName = phaseName, Attempt = Convert.ToInt32(phaseAttempt)
};
JobReference jobReference = new JobReference()
{
JobName = jobName, Attempt = Convert.ToInt32(jobAttempt)
};
PipelineReference pipelineReference = new PipelineReference()
{
PipelineId = buildId,
StageReference = stageReference,
PhaseReference = phaseReference,
JobReference = jobReference
};
TestRunContext testRunContext = new TestRunContext(
owner: owner,
platform: _platform,
configuration: _configuration,
buildId: buildId,
buildUri: buildUri,
releaseUri: releaseUri,
releaseEnvironmentUri: releaseEnvironmentUri,
runName: runName,
testRunSystem: _testRunSystem,
buildAttachmentProcessor: new CodeCoverageBuildAttachmentProcessor(),
targetBranchName: pullRequestTargetBranchName,
pipelineReference: pipelineReference
);
return(testRunContext);
}
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
var doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Ignore
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return(null);
}
INUnitResultsXmlReader nunitResultsReader = null;
var testResultsNode = doc.SelectSingleNode("test-results");
if (testResultsNode == null)
{
testResultsNode = doc.SelectSingleNode("test-run");
if (testResultsNode == null)
{
throw new NotSupportedException(StringUtil.Loc("InvalidFileFormat"));
}
nunitResultsReader = new NUnit3ResultsXmlReader();
}
else
{
nunitResultsReader = new NUnit2ResultsXmlReader();
}
return(nunitResultsReader.GetTestRunData(filePath, doc, testResultsNode, runContext, AddResultsFileToRunLevelAttachments));
}
/// <summary>
/// Publish separate test run for each result file that has results.
/// </summary>
private async Task PublishToNewTestRunPerTestResultFileAsync(List<string> resultFiles, ITestRunPublisher publisher, TestRunContext runContext, string resultReader, CancellationToken cancellationToken)
{
try
{
// Publish separate test run for each result file that has results.
var publishTasks = resultFiles.Select(async resultFile =>
{
cancellationToken.ThrowIfCancellationRequested();
string runName = null;
if (!string.IsNullOrWhiteSpace(_runTitle))
{
runName = GetRunTitle();
}
_executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
TestRunData testRunData = publisher.ReadResultsFromFile(runContext, resultFile, runName);
cancellationToken.ThrowIfCancellationRequested();
if (testRunData != null && testRunData.Results != null && testRunData.Results.Length > 0)
{
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, testRunData.Results, _executionContext.CancellationToken);
await publisher.EndTestRunAsync(testRunData, testRun.Id, cancellationToken: _executionContext.CancellationToken);
}
else
{
_executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
}
});
await Task.WhenAll(publishTasks);
}
catch (Exception ex) when (!(ex is OperationCanceledException))
{
//Do not fail the task.
LogPublishTestResultsFailureWarning(ex);
}
}
/// <summary>
/// Converts the given results file to TestRunData object
/// </summary>
/// <param name="filePath">File path</param>
/// <param name="runName">Run Name</param>
/// <returns>TestRunData</returns>
public TestRunData ReadResultsFromFile(TestRunContext runContext, string filePath, string runName)
{
Trace.Entering();
runContext.RunName = runName;
return(_resultReader.ReadResults(_executionContext, filePath, runContext));
}
/// <summary>
/// Publish single test run
/// </summary>
private async Task PublishAllTestResultsToSingleTestRunAsync(List<string> resultFiles, ITestRunPublisher publisher, int buildId, TestRunContext runContext, string resultReader, CancellationToken cancellationToken)
{
try
{
DateTime startTime = DateTime.Now; //use local time since TestRunData defaults to local times
TimeSpan totalTestCaseDuration = TimeSpan.Zero;
List<string> runAttachments = new List<string>();
List<TestCaseResultData> runResults = new List<TestCaseResultData>();
//read results from each file
foreach (string resultFile in resultFiles)
{
cancellationToken.ThrowIfCancellationRequested();
//test case results
_executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
TestRunData resultFileRunData = publisher.ReadResultsFromFile(runContext, resultFile);
if (resultFileRunData != null && resultFileRunData.Results != null && resultFileRunData.Results.Length > 0)
{
foreach (TestCaseResultData tcResult in resultFileRunData.Results)
{
int durationInMs = 0;
int.TryParse(tcResult.DurationInMs, out durationInMs);
totalTestCaseDuration = totalTestCaseDuration.Add(TimeSpan.FromMilliseconds(durationInMs));
}
runResults.AddRange(resultFileRunData.Results);
//run attachments
if (resultFileRunData.Attachments != null)
{
runAttachments.AddRange(resultFileRunData.Attachments);
}
}
else
{
_executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
}
}
string runName = string.IsNullOrWhiteSpace(_runTitle)
? StringUtil.Format("{0}_TestResults_{1}", _testRunner, buildId)
: _runTitle;
//creat test run
TestRunData testRunData = new TestRunData(
name: runName,
startedDate: startTime.ToString("o"),
completedDate: startTime.Add(totalTestCaseDuration).ToString("o"),
state: "InProgress",
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: runContext != null ? runContext.Platform : string.Empty
);
testRunData.Attachments = runAttachments.ToArray();
//publish run if there are results.
if (runResults.Count > 0)
{
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, runResults.ToArray(), _executionContext.CancellationToken);
await publisher.EndTestRunAsync(testRunData, testRun.Id, true, _executionContext.CancellationToken);
}
}
catch (Exception ex) when (!(ex is OperationCanceledException))
{
//Do not fail the task.
LogPublishTestResultsFailureWarning(ex);
}
}
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
{
List <TestCaseResultData> results = new List <TestCaseResultData>();
XmlDocument doc = new XmlDocument();
try
{
var settings = new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Ignore
};
using (XmlReader reader = XmlReader.Create(filePath, settings))
{
doc.Load(reader);
}
}
catch (XmlException ex)
{
executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
return(null);
}
//read test run summary information - run name, start time
string runName = Name + " Test Run";
DateTime runStartTime = DateTime.MinValue; //Use local time instead of UTC as TestRunData uses local time for defaults. Also assuming timestamp is local is more accurate in cases where tests were run on build machine
TimeSpan totalRunDuration = TimeSpan.Zero;
TimeSpan totalTestCaseDuration = TimeSpan.Zero;
XmlNode testResultsNode = doc.SelectSingleNode("test-results");
if (testResultsNode != null)
{
//get test run summary information
if (testResultsNode.Attributes["name"] != null)
{
runName = testResultsNode.Attributes["name"].Value;
}
//run times
DateTime dateFromXml = DateTime.MinValue.Date; //Use local time instead of UTC as TestRunData uses local time for defaults.
if (testResultsNode.Attributes["date"] != null)
{
DateTime.TryParse(testResultsNode.Attributes["date"].Value, out dateFromXml);
}
TimeSpan timeFromXml = TimeSpan.Zero;
if (testResultsNode.Attributes["time"] != null)
{
TimeSpan.TryParse(testResultsNode.Attributes["time"].Value, out timeFromXml);
}
//assume runtimes from xml are current local time since timezone information is not in the xml, if xml datetime > current local time, fallback to local start time
DateTime runStartDateTimeFromXml = new DateTime(dateFromXml.Ticks).AddTicks(timeFromXml.Ticks);
if (runStartTime == DateTime.MinValue)
{
runStartTime = runStartDateTimeFromXml;
}
}
//run environment - platform, config and hostname
string platform = runContext != null ? runContext.Platform : string.Empty;
string config = runContext != null ? runContext.Configuration : string.Empty;
string runUser = runContext != null ? runContext.Owner : string.Empty;
string hostName = string.Empty;
XmlNode envNode = doc.SelectSingleNode("test-results/environment");
if (envNode != null)
{
if (envNode.Attributes["machine-name"] != null && envNode.Attributes["machine-name"].Value != null)
{
hostName = envNode.Attributes["machine-name"].Value;
}
if (envNode.Attributes["platform"] != null && envNode.Attributes["platform"].Value != null && runContext != null && runContext.BuildId > 0)
{
//We cannot publish platform information without a valid build id.
platform = envNode.Attributes["platform"].Value;
}
}
//run owner
IdentityRef runUserIdRef = null;
if (!string.IsNullOrEmpty(runUser))
{
runUserIdRef = new IdentityRef()
{
DisplayName = runUser
};
}
//get all test assemblies
if (testResultsNode != null)
{
XmlNodeList testAssemblyNodes = testResultsNode.SelectNodes("test-suite");
if (testAssemblyNodes != null)
{
foreach (XmlNode testAssemblyNode in testAssemblyNodes)
{
var assemblyStartTime = (runStartTime == DateTime.MinValue) ? DateTime.MinValue : runStartTime + totalTestCaseDuration;
List <TestCaseResultData> testCases = FindTestCaseNodes(testAssemblyNode, hostName, runUserIdRef, assemblyStartTime);
if (testCases != null)
{
results.AddRange(testCases);
testCases.ForEach(x => totalTestCaseDuration += TimeSpan.FromMilliseconds(x.DurationInMs));
}
}
}
}
if (TimeSpan.Compare(totalRunDuration, totalTestCaseDuration) < 0)
{
totalRunDuration = totalTestCaseDuration; //run duration may not be set in the xml, so use total test case duration
}
if (runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName))
{
runName = runContext.RunName;
}
//create test run data
TestRunData testRunData = new TestRunData(
name: runName,
startedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.ToString("o"),
completedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.Add(totalRunDuration).ToString("o"),
state: TestRunState.InProgress.ToString(),
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: config,
buildPlatform: platform,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.Results = results.ToArray();
testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];
return(testRunData);
}
private void ReadResults(TestRunContext runContext = null, bool attachRunLevelAttachments = true)
{
_fileName = Path.Combine(Path.GetTempPath(), Path.GetTempFileName());
File.WriteAllText(_fileName, _junitResultsToBeRead);
_jUnitReader = new JUnitResultReader();
_jUnitReader.AddResultsFileToRunLevelAttachments = attachRunLevelAttachments;
_testRunData = _jUnitReader.ReadResults(_ec.Object, _fileName, runContext);
}
public TestDataProvider ParseTestResultFiles(IExecutionContext executionContext, TestRunContext testRunContext, List <string> testResultsFiles)
{
if (string.IsNullOrEmpty(Name))
{
executionContext.Warning("Test runner name is null or empty");
return(null);
}
// Create test result parser object based on the test Runner provided
var testResultParser = GetTestResultParser(executionContext);
if (testResultParser == null)
{
return(null);
}
// Parse with the corresponding testResultParser object
return(ParseFiles(executionContext, testRunContext, testResultsFiles, testResultParser));
}
/// <summary>
/// Converts the given results file to TestRunData object
/// </summary>
/// <param name="filePath">File path</param>
/// <param name="runName">Run Name</param>
/// <returns>TestRunData</returns>
public TestRunData ReadResultsFromFile(TestRunContext runContext, string filePath, string runName)
{
Trace.Entering();
runContext.RunName = runName;
return _resultReader.ReadResults(_executionContext, filePath, runContext);
}
private async Task PublishTestResultsAsync(VssConnection connection, String teamProject, int buildId, TestRunContext runContext, IResultReader resultReader)
{
var publisher = HostContext.GetService <ITestRunPublisher>();
publisher.InitializePublisher(_executionContext, connection, teamProject, resultReader);
if (_mergeResults)
{
await PublishAllTestResultsToSingleTestRunAsync(_testResultFiles, publisher, buildId, runContext, resultReader.Name, _executionContext.CancellationToken);
}
else
{
await PublishToNewTestRunPerTestResultFileAsync(_testResultFiles, publisher, runContext, resultReader.Name, PublishBatchSize, _executionContext.CancellationToken);
}
await PublishEventsAsync(connection);
}
