private void ProcessPublishTestResultsCommand(IExecutionContext context, Dictionary<string, string> eventProperties, string data)
        {
            ArgUtil.NotNull(context, nameof(context));
            _executionContext = context;

            LoadPublishTestResultsInputs(eventProperties, data);

            string teamProject = context.Variables.System_TeamProject;
            string owner = context.Variables.Build_RequestedFor;
            string buildUri = context.Variables.Build_BuildUri;
            int buildId = context.Variables.Build_BuildId ?? 0;

            //Temporary fix to support publish in RM scenarios where there might not be a valid Build ID associated.
            //TODO: Make a cleaner fix after TCM User Story 401703 is completed.
            if (buildId == 0)
            {
                _platform = _configuration = null;
            }

            string releaseUri = null;
            string releaseEnvironmentUri = null;

            // Check to identify if we are in the Release management flow; if not, then release fields will be kept null while publishing to TCM 
            if (!string.IsNullOrWhiteSpace(context.Variables.Release_ReleaseUri))
            {
                releaseUri = context.Variables.Release_ReleaseUri;
                releaseEnvironmentUri = context.Variables.Release_ReleaseEnvironmentUri;
            }

            IResultReader resultReader = GetTestResultReader(_testRunner);
            TestRunContext runContext = new TestRunContext(owner, _platform, _configuration, buildId, buildUri, releaseUri, releaseEnvironmentUri);
            Client.VssConnection connection = WorkerUtilies.GetVssConnection(_executionContext);

            var publisher = HostContext.GetService<ITestRunPublisher>();
            publisher.InitializePublisher(context, connection, teamProject, resultReader);

            var commandContext = HostContext.CreateService<IAsyncCommandContext>();
            commandContext.InitializeCommandContext(context, StringUtil.Loc("PublishTestResults"));

            if (_mergeResults)
            {
                commandContext.Task = PublishAllTestResultsToSingleTestRunAsync(_testResultFiles, publisher, buildId, runContext, resultReader.Name, context.CancellationToken);
            }
            else
            {
                commandContext.Task = PublishToNewTestRunPerTestResultFileAsync(_testResultFiles, publisher, runContext, resultReader.Name, context.CancellationToken);
            }
            _executionContext.AsyncCommands.Add(commandContext);
        }
        public TestRunPublisherTests()
        {
            _attachmentFilePath = "attachment.txt";

            File.WriteAllText(_attachmentFilePath, "asdf");
            _testRunContext = new TestRunContext("owner", "platform", "config", 1, "builduri", "releaseuri", "releaseenvuri");

            _reader = new Mock<IResultReader>();
            _reader.Setup(x => x.ReadResults(It.IsAny<IExecutionContext>(), It.IsAny<string>(), It.IsAny<TestRunContext>()))
                        .Callback<IExecutionContext, string, TestRunContext>
                        ((executionContext, filePath, runContext) =>
                        {
                            _runContext = runContext;
                            _resultsFilepath = filePath;
                        })
                        .Returns((IExecutionContext executionContext, string filePath, TestRunContext runContext) =>
                        {
                            TestRunData trd = new TestRunData(
                                name: "xyz",
                                buildId: runContext.BuildId,
                                completedDate: "",
                                state: "InProgress",
                                isAutomated: true,
                                dueDate: "",
                                type: "",
                                buildFlavor: runContext.Configuration,
                                buildPlatform: runContext.Platform,
                                releaseUri: runContext.ReleaseUri,
                                releaseEnvironmentUri: runContext.ReleaseEnvironmentUri
                            );
                            trd.Attachments = new string[] { "attachment.txt" };
                            return trd;
                        });

            _testResultServer = new Mock<ITestResultsServer>();
            _testResultServer.Setup(x => x.InitializeServer(It.IsAny<Client.VssConnection>()));
            _testResultServer.Setup(x => x.AddTestResultsToTestRunAsync(It.IsAny<TestResultCreateModel[]>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
                        .Callback<TestResultCreateModel[], string, int, CancellationToken>
                        ((currentBatch, projectName, testRunId, cancellationToken) =>
                        {
                            _batchSizes.Add(currentBatch.Length);
                            _resultCreateModels = currentBatch;
                        })
                        .Returns(() =>
                        {
                            List<TestCaseResult> resultsList = new List<TestCaseResult>();
                            int i = 0;
                            foreach (TestResultCreateModel resultCreateModel in _resultCreateModels)
                            {
                                resultsList.Add(new TestCaseResult() { Id = ++i });
                            }
                            return Task.FromResult(resultsList);
                        });

            _testResultServer.Setup(x => x.CreateTestRunAsync(It.IsAny<string>(), It.IsAny<RunCreateModel>(), It.IsAny<CancellationToken>()))
                        .Callback<string, RunCreateModel, CancellationToken>
                        ((projectName, testRunData, cancellationToken) =>
                        {
                            _projectId = projectName;
                            _testRun = (TestRunData)testRunData;
                        })
                        .Returns(Task.FromResult(new TestRun() { Name = "TestRun", Id = 1 }));

            _testResultServer.Setup(x => x.UpdateTestRunAsync(It.IsAny<string>(), It.IsAny<int>(), It.IsAny<RunUpdateModel>(), It.IsAny<CancellationToken>()))
                        .Callback<string, int, RunUpdateModel, CancellationToken>
                        ((projectName, testRunId, updateModel, cancellationToken) =>
                        {
                            _runId = testRunId;
                            _projectId = projectName;
                            _updateProperties = updateModel;
                        })
                        .Returns(Task.FromResult(new TestRun() { Name = "TestRun", Id = 1 }));

            _testResultServer.Setup(x => x.CreateTestRunAttachmentAsync(
                        It.IsAny<TestAttachmentRequestModel>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
                        .Callback<TestAttachmentRequestModel, string, int, CancellationToken>
                        ((reqModel, projectName, testRunId, cancellationToken) =>
                        {
                            _attachmentRequestModel = reqModel;
                            _projectId = projectName;
                            _runId = testRunId;
                        })
                        .Returns(Task.FromResult(new TestAttachmentReference()));

            _testResultServer.Setup(x => x.CreateTestResultAttachmentAsync(It.IsAny<TestAttachmentRequestModel>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
                        .Callback<TestAttachmentRequestModel, string, int, int, CancellationToken>
                        ((reqModel, projectName, testRunId, testCaseResultId, cancellationToken) =>
                        {
                            if (_resultsLevelAttachments.ContainsKey(testCaseResultId))
                            {
                                _resultsLevelAttachments[testCaseResultId].Add(reqModel);
                            }
                            else
                            {
                                _resultsLevelAttachments.Add(testCaseResultId, new List<TestAttachmentRequestModel>() { reqModel });
                            }
                        })
                        .Returns(Task.FromResult(new TestAttachmentReference()));
        }
        /// <summary>
        /// Reads a JUnit results file from disk, converts it into a TestRunData object.        
        /// </summary>
        /// <param name="filePath"></param>
        /// <param name="runContext"></param>
        /// <returns></returns>
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
        {
            // http://windyroad.com.au/dl/Open%20Source/JUnit.xsd

            XmlDocument doc = new XmlDocument();
            try
            {
                var settings = new XmlReaderSettings
                {
                    DtdProcessing = DtdProcessing.Prohibit
                };

                using (XmlReader reader = XmlReader.Create(filePath, settings))
                {
                    doc.Load(reader);
                }
            }
            catch (XmlException ex)
            {
                executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
                return null;
            }

            //init test run summary information - run name, host name, start time
            TestSuiteSummary runSummary = new TestSuiteSummary(Name);

            IdentityRef runUserIdRef = null;
            string runUser = runContext != null ? runContext.Owner : string.Empty;
            if (!string.IsNullOrEmpty(runUser))
            {
                runUserIdRef = new IdentityRef() { DisplayName = runUser };
            }

            //read data from testsuite nodes

            XmlNode testSuitesNode = doc.SelectSingleNode("testsuites");
            if (testSuitesNode != null)
            {
                //found testsuites node - some plugins generate it like karma junit plugin
                XmlNodeList testSuiteNodeList = doc.SelectNodes("/testsuites/testsuite");
                if (testSuiteNodeList != null)
                {
                    foreach (XmlNode testSuiteNode in testSuiteNodeList)
                    {
                        TestSuiteSummary testSuiteSummary = ReadTestSuite(testSuiteNode, runUserIdRef);
                        runSummary.Duration = runSummary.Duration.Add(testSuiteSummary.Duration);
                        runSummary.Results.AddRange(testSuiteSummary.Results);
                        runSummary.Host = testSuiteSummary.Host;
                        runSummary.Name = testSuiteSummary.Name;
                    }

                    if (testSuiteNodeList.Count > 1)
                    {
                        runSummary.Name = Name + "_" + Path.GetFileName(filePath);
                    }
                }
            }
            else
            {
                XmlNode testSuiteNode = doc.SelectSingleNode("testsuite");
                if (testSuiteNode != null)
                {
                    runSummary = ReadTestSuite(testSuiteNode, runUserIdRef);
                }
            }

            if (runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName))
            {
                runSummary.Name = runContext.RunName;
            }

            if (runSummary.Results.Count > 0)
            {
                //first testsuite starteddate is the starteddate of the run
                runSummary.TimeStamp = DateTime.Parse(runSummary.Results[0].StartedDate);
            }

            //create test run data
            var testRunData = new TestRunData(
                name: runSummary.Name,
                startedDate: runSummary.TimeStamp.ToString("o"),
                completedDate: runSummary.TimeStamp.Add(runSummary.Duration).ToString("o"),
                state: TestRunState.InProgress.ToString(),
                isAutomated: true,
                buildId: runContext != null ? runContext.BuildId : 0,
                buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
                buildPlatform: runContext != null ? runContext.Platform : string.Empty,
                releaseUri: runContext != null ? runContext.ReleaseUri : null,
                releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
                )
            {
                Results = runSummary.Results.ToArray(),
                Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0]

            };

            return testRunData;
        }
Exemple #4
0
        private async Task PublishTestRunDataAsync(VssConnection connection, String teamProject, TestRunContext testRunContext)
        {
            bool isTestRunOutcomeFailed = false;

            var featureFlagService = HostContext.GetService <IFeatureFlagService>();

            featureFlagService.InitializeFeatureService(_executionContext, connection);
            var publishTestResultsLibFeatureState = featureFlagService.GetFeatureFlagState(TestResultsConstants.UsePublishTestResultsLibFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);

            _telemetryProperties.Add("UsePublishTestResultsLib", publishTestResultsLibFeatureState);

            //This check is to determine to use "Microsoft.TeamFoundation.PublishTestResults" Library or the agent code to parse and publish the test results.
            if (publishTestResultsLibFeatureState)
            {
                var publisher = HostContext.GetService <ITestDataPublisher>();
                publisher.InitializePublisher(_executionContext, teamProject, connection, _testRunner);

                isTestRunOutcomeFailed = await publisher.PublishAsync(testRunContext, _testResultFiles, GetPublishOptions(), _executionContext.CancellationToken);
            }
            else
            {
                var publisher = HostContext.GetService <ILegacyTestRunDataPublisher>();
                publisher.InitializePublisher(_executionContext, teamProject, connection, _testRunner, _publishRunLevelAttachments);

                isTestRunOutcomeFailed = await publisher.PublishAsync(testRunContext, _testResultFiles, _runTitle, _executionContext.Variables.Build_BuildId, _mergeResults);
            }

            if (isTestRunOutcomeFailed && _failTaskOnFailedTests)
            {
                _executionContext.Result = TaskResult.Failed;
                _executionContext.Error(StringUtil.Loc("FailedTestsInResults"));
            }

            await PublishEventsAsync(connection);
        }
        //Based on the XUnit V2 format: http://xunit.github.io/docs/format-xml-v2.html
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
        {
            List <TestCaseResultData> results = new List <TestCaseResultData>();

            XmlDocument doc = new XmlDocument();

            try
            {
                var settings = new XmlReaderSettings
                {
                    DtdProcessing = DtdProcessing.Ignore
                };

                using (XmlReader reader = XmlReader.Create(filePath, settings))
                {
                    doc.Load(reader);
                }
            }
            catch (XmlException ex)
            {
                executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
                return(null);
            }

            string runName = Name + " Test Run";
            string runUser = "";

            if (runContext != null)
            {
                if (!string.IsNullOrWhiteSpace(runContext.RunName))
                {
                    runName = runContext.RunName;
                }
                else
                {
                    runName = string.Format(CultureInfo.CurrentCulture, "{0} {1} {2}", runName, runContext.Configuration, runContext.Platform);
                }

                if (runContext.Owner != null)
                {
                    runUser = runContext.Owner;
                }
            }

            IdentityRef runUserIdRef = new IdentityRef();

            runUserIdRef.DisplayName = runUser;

            var    minStartTime       = DateTime.MaxValue;
            var    maxCompletedTime   = DateTime.MinValue;
            bool   dateTimeParseError = true;
            bool   assemblyRunDateTimeAttributesNotPresent = false;
            bool   assemblyTimeAttributeNotPresent         = false;
            double assemblyRunDuration = 0;
            double testRunDuration     = 0;

            XmlNodeList assemblyNodes = doc.SelectNodes("/assemblies/assembly");

            foreach (XmlNode assemblyNode in assemblyNodes)
            {
                var assemblyRunStartTimeStamp = DateTime.MinValue;
                if (assemblyNode.Attributes["run-date"] != null && assemblyNode.Attributes["run-time"] != null)
                {
                    string runDate = assemblyNode.Attributes["run-date"].Value;
                    string runTime = assemblyNode.Attributes["run-time"].Value;

                    var startDate = DateTime.Now;
                    var startTime = TimeSpan.Zero;
                    if (DateTime.TryParse(runDate, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.None, out startDate) &&
                        TimeSpan.TryParse(runTime, CultureInfo.InvariantCulture, out startTime))
                    {
                        dateTimeParseError = false;
                    }
                    assemblyRunStartTimeStamp = startDate + startTime;
                    if (minStartTime > assemblyRunStartTimeStamp)
                    {
                        minStartTime = assemblyRunStartTimeStamp;
                    }
                }
                else
                {
                    assemblyRunDateTimeAttributesNotPresent = true;
                }
                if (!assemblyTimeAttributeNotPresent && assemblyNode.Attributes["time"] != null)
                {
                    double assemblyDuration = 0;
                    Double.TryParse(assemblyNode.Attributes["time"].Value, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out assemblyDuration);
                    assemblyRunDuration = assemblyRunDuration + assemblyDuration;
                    var durationFromSeconds = TimeSpan.FromSeconds(assemblyDuration);

                    // no assemblystarttime available so dont calculate assemblycompletedtime
                    if (assemblyRunStartTimeStamp != DateTime.MinValue)
                    {
                        DateTime assemblyRunCompleteTimeStamp =
                            assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks);

                        //finding maximum comleted time
                        if (maxCompletedTime < assemblyRunCompleteTimeStamp)
                        {
                            maxCompletedTime = assemblyRunCompleteTimeStamp;
                        }
                    }
                }
                else
                {
                    assemblyTimeAttributeNotPresent = true;
                }
                XmlNodeList testCaseNodeList = assemblyNode.SelectNodes("./collection/test");
                foreach (XmlNode testCaseNode in testCaseNodeList)
                {
                    TestCaseResultData resultCreateModel = new TestCaseResultData()
                    {
                        Priority = TestManagementConstants.UnspecifiedPriority,  //Priority is int type so if no priority set then its 255.
                    };

                    //Test storage.
                    if (assemblyNode.Attributes["name"] != null)
                    {
                        resultCreateModel.AutomatedTestStorage = Path.GetFileName(assemblyNode.Attributes["name"].Value);
                    }

                    //Fully Qualified Name.
                    if (testCaseNode.Attributes["name"] != null)
                    {
                        resultCreateModel.AutomatedTestName = testCaseNode.Attributes["name"].Value;
                    }

                    //Test Method Name.
                    if (testCaseNode.Attributes["method"] != null)
                    {
                        resultCreateModel.TestCaseTitle = testCaseNode.Attributes["method"].Value;
                    }

                    //Test duration.
                    if (testCaseNode.Attributes["time"] != null && testCaseNode.Attributes["time"].Value != null)
                    {
                        double duration = 0;
                        double.TryParse(testCaseNode.Attributes["time"].Value, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out duration);
                        var durationFromSeconds = TimeSpan.FromSeconds(duration);
                        resultCreateModel.DurationInMs = durationFromSeconds.TotalMilliseconds;

                        // no assemblystarttime available so dont set testcase start and completed
                        if (assemblyRunStartTimeStamp != DateTime.MinValue)
                        {
                            resultCreateModel.StartedDate   = assemblyRunStartTimeStamp;
                            resultCreateModel.CompletedDate = assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks);
                            assemblyRunStartTimeStamp       = assemblyRunStartTimeStamp.AddTicks(1) + durationFromSeconds;
                            //next start time
                        }

                        //Calculate overall run duration.
                        testRunDuration += duration;
                    }


                    //Test outcome.
                    if (testCaseNode.SelectSingleNode("./failure") != null)
                    {
                        resultCreateModel.Outcome = TestOutcome.Failed.ToString();

                        //Error message.
                        XmlNode failureMessageNode = testCaseNode.SelectSingleNode("./failure/message");
                        if (failureMessageNode != null && !string.IsNullOrWhiteSpace(failureMessageNode.InnerText))
                        {
                            resultCreateModel.ErrorMessage = failureMessageNode.InnerText;
                        }

                        //Stack trace.
                        XmlNode failureStackTraceNode = testCaseNode.SelectSingleNode("./failure/stack-trace");
                        if (failureStackTraceNode != null && !string.IsNullOrWhiteSpace(failureStackTraceNode.InnerText))
                        {
                            resultCreateModel.StackTrace = failureStackTraceNode.InnerText;
                        }
                    }
                    else if (testCaseNode.Attributes["result"] != null && string.Equals(testCaseNode.Attributes["result"].Value, "pass", StringComparison.OrdinalIgnoreCase))
                    {
                        resultCreateModel.Outcome = TestOutcome.Passed.ToString();
                    }
                    else
                    {
                        resultCreateModel.Outcome = TestOutcome.NotExecuted.ToString();
                    }

                    //Test priority.
                    XmlNode priorityTrait = testCaseNode.SelectSingleNode("./traits/trait[@name='priority']");
                    if (priorityTrait != null && priorityTrait.Attributes["value"] != null)
                    {
                        var priorityValue = priorityTrait.Attributes["value"].Value;
                        resultCreateModel.Priority = !string.IsNullOrEmpty(priorityValue) ? Convert.ToInt32(priorityValue)
                                                        : TestManagementConstants.UnspecifiedPriority;
                    }

                    //Test owner.
                    XmlNode ownerNode = testCaseNode.SelectSingleNode("./traits/trait[@name='owner']");
                    if (ownerNode != null && ownerNode.Attributes["value"] != null && ownerNode.Attributes["value"].Value != null)
                    {
                        IdentityRef ownerIdRef = new IdentityRef();
                        ownerIdRef.DisplayName    = ownerNode.Attributes["value"].Value;
                        ownerIdRef.DirectoryAlias = ownerNode.Attributes["value"].Value;
                        resultCreateModel.Owner   = ownerIdRef;
                    }

                    resultCreateModel.RunBy = runUserIdRef;

                    resultCreateModel.State = "Completed";

                    resultCreateModel.AutomatedTestType = Name;

                    if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle))
                    {
                        results.Add(resultCreateModel);
                    }
                }
            }
            if (dateTimeParseError || assemblyRunDateTimeAttributesNotPresent)
            {
                executionContext.Warning("Atleast for one assembly start time was not obtained due to tag not present or parsing issue, total run duration will now be summation of time taken by each assembly");

                if (assemblyTimeAttributeNotPresent)
                {
                    executionContext.Warning("Atleast for one assembly time tag is not present, total run duration will now be summation of time from all test runs");
                }
            }

            //if minimum start time is not available then set it to present time
            minStartTime = minStartTime == DateTime.MaxValue ? DateTime.UtcNow : minStartTime;

            //if start time cannot be obtained even for one assembly then fallback duration to sum of assembly run time
            //if assembly run time cannot be obtained even for one assembly then fallback duration to total test run
            maxCompletedTime = dateTimeParseError || assemblyRunDateTimeAttributesNotPresent || maxCompletedTime == DateTime.MinValue ? minStartTime.Add(assemblyTimeAttributeNotPresent ? TimeSpan.FromSeconds(testRunDuration) : TimeSpan.FromSeconds(assemblyRunDuration)) : maxCompletedTime;

            executionContext.Output(string.Format("Obtained XUnit Test Run Start Date: {0} and Completed Date: {1}", minStartTime.ToString("o"), maxCompletedTime.ToString("o")));
            TestRunData testRunData = new TestRunData(
                name: runName,
                buildId: runContext != null ? runContext.BuildId : 0,
                startedDate: minStartTime.ToString("o"),
                completedDate: maxCompletedTime.ToString("o"),
                state: TestRunState.InProgress.ToString(),
                isAutomated: true,
                buildFlavor: runContext != null ? runContext.Configuration : null,
                buildPlatform: runContext != null ? runContext.Platform : null,
                releaseUri: runContext != null ? runContext.ReleaseUri : null,
                releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
                );

            testRunData.Results     = results.ToArray();
            testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];

            return(testRunData);
        }
        /// <summary>
        /// Reads a trx file from disk, converts it into a TestRunData object.
        /// </summary>
        /// <param name="filePath">File path</param>
        /// <returns>TestRunData</returns>
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext)
        {
            _executionContext = executionContext;

            _definitions = new Dictionary <string, TestCaseDefinition>();

            List <TestCaseResultData> results = new List <TestCaseResultData>();

            string xmlContents = File.ReadAllText(filePath);

            xmlContents = xmlContents.Replace("xmlns", "ns");

            XmlDocument doc = new XmlDocument();

            doc.LoadXml(xmlContents);

            string runName = Name + " Test Run";

            if (runContext != null)
            {
                if (!String.IsNullOrWhiteSpace(runContext.RunName))
                {
                    runName = runContext.RunName;
                }
                else
                {
                    runName = StringUtil.Format("{0} {1} {2}", runName, runContext.Configuration, runContext.Platform);
                }
            }


            string runUser = "";

            if (runContext.Owner != null)
            {
                runUser = runContext.Owner;
            }

            _runUserIdRef             = new IdentityRef();
            _runUserIdRef.DisplayName = runUser;

            //Parse the run start and finish times. If either of it is not available, send neither.
            XmlNode  node          = doc.SelectSingleNode("/TestRun/Times");
            DateTime runStartDate  = DateTime.MinValue;
            DateTime runFinishDate = DateTime.MinValue;

            if (node != null && node.Attributes["start"] != null && node.Attributes["finish"] != null)
            {
                if (DateTime.TryParse(node.Attributes["start"].Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out runStartDate))
                {
                    _executionContext.Debug(string.Format(CultureInfo.InvariantCulture, "Setting run start and finish times."));
                    //Only if there is a valid start date.
                    DateTime.TryParse(node.Attributes["finish"].Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out runFinishDate);
                    if (runFinishDate < runStartDate)
                    {
                        runFinishDate = runStartDate = DateTime.MinValue;
                        _executionContext.Debug("Run finish date is less than start date. Resetting to min value.");
                    }
                }
            }

            TestRunData testRunData = new TestRunData(
                name: runName,
                buildId: runContext.BuildId,
                startedDate: runStartDate != DateTime.MinValue ? runStartDate.ToString("o") : null,
                completedDate: runFinishDate != DateTime.MinValue ? runFinishDate.ToString("o") : null,
                state: TestRunState.InProgress.ToString(),
                isAutomated: true,
                dueDate: string.Empty,
                type: string.Empty,
                buildFlavor: runContext.Configuration,
                buildPlatform: runContext.Platform,
                releaseUri: runContext.ReleaseUri,
                releaseEnvironmentUri: runContext.ReleaseEnvironmentUri
                );

            //Parse the Deployment node for the runDeploymentRoot - this is the attachment root. Required for .NET Core
            XmlNode deploymentNode  = doc.SelectSingleNode("/TestRun/TestSettings/Deployment");
            var     _attachmentRoot = string.Empty;

            if (deploymentNode != null && deploymentNode.Attributes["runDeploymentRoot"] != null)
            {
                _attachmentRoot = deploymentNode.Attributes["runDeploymentRoot"].Value;
            }
            else
            {
                _attachmentRoot = Path.GetFileNameWithoutExtension(filePath); // This for platform v1
            }
            _attachmentLocation = Path.Combine(Path.GetDirectoryName(filePath), _attachmentRoot, "In");
            _executionContext.Debug(string.Format(CultureInfo.InvariantCulture, "Attachment location: {0}", _attachmentLocation));

            AddRunLevelAttachments(filePath, doc, testRunData);

            // Create a dictionary of testcase definitions to be used when iterating over the results
            Dictionary <string, TestCaseDefinition> definitions = new Dictionary <string, TestCaseDefinition>();

            if (doc.SelectNodes("/TestRun/TestDefinitions").Count > 0)
            {
                foreach (XmlNode definitionNode in doc.SelectNodes("/TestRun/TestDefinitions")[0])
                {
                    IdentityRef owner = null;
                    string      priority = null, storage = null;
                    if (definitionNode.Attributes["storage"] != null && definitionNode.Attributes["storage"].Value != null)
                    {
                        storage = Path.GetFileName(definitionNode.Attributes["storage"].Value);
                    }

                    XmlAttribute priorityAttribute = definitionNode.Attributes["priority"];
                    if (priorityAttribute != null)
                    {
                        priority = priorityAttribute.Value;
                    }

                    XmlNode ownerNode = definitionNode.SelectSingleNode("./Owners/Owner");
                    if (ownerNode != null)
                    {
                        IdentityRef ownerIdRef = new IdentityRef();
                        ownerIdRef.DisplayName    = ownerNode.Attributes["name"].Value;
                        ownerIdRef.DirectoryAlias = ownerNode.Attributes["name"].Value;
                        owner = ownerIdRef;
                    }

                    // The automated test name should be FQDN, if we are unable to figure it out like in case of a webtest,
                    // set it as "name" from the parent (where it is always present)
                    XmlNode testResultNode    = definitionNode.SelectSingleNode("./TestMethod");
                    string  automatedTestName = null;
                    if (testResultNode != null && testResultNode.Attributes["className"] != null && testResultNode.Attributes["name"] != null)
                    {
                        // At times the class names are coming as
                        // className="MS.TF.Test.AgileX.VSTests.WiLinking.UI.WiLinkingUIQueryTests"
                        // at other times, they are as
                        // className="UnitTestProject3.UnitTest1, UnitTestProject3, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null"
                        string className = testResultNode.Attributes["className"].Value.Split(',')[0];
                        automatedTestName = className + "." + testResultNode.Attributes["name"].Value;
                    }
                    else if (definitionNode.Attributes["name"] != null)
                    {
                        automatedTestName = definitionNode.Attributes["name"].Value;
                    }

                    _definitions.Add(definitionNode.Attributes["id"].Value, new TestCaseDefinition(automatedTestName, owner, priority, storage));
                }
            }

            // Read UnitTestResults, WebTestResults and OrderedTestResults
            XmlNodeList resultsNodes           = doc.SelectNodes("/TestRun/Results/UnitTestResult");
            XmlNodeList webTestResultNodes     = doc.SelectNodes("/TestRun/Results/WebTestResult");
            XmlNodeList orderedTestResultNodes = doc.SelectNodes("/TestRun/Results/TestResultAggregation");

            results.AddRange(ReadActualResults(resultsNodes, TestType.UnitTest));
            results.AddRange(ReadActualResults(webTestResultNodes, TestType.WebTest));
            results.AddRange(ReadActualResults(orderedTestResultNodes, TestType.OrderedTest));


            testRunData.Results = results.ToArray <TestCaseResultData>();
            _executionContext.Debug(string.Format(CultureInfo.InvariantCulture, "Total test results: {0}", testRunData.Results.Length));

            return(testRunData);
        }
        /// <summary>
        /// Publish separate test run for each result file that has results.
        /// </summary>
        private async Task PublishToNewTestRunPerTestResultFileAsync(List <string> resultFiles,
                                                                     ITestRunPublisher publisher,
                                                                     TestRunContext runContext,
                                                                     string resultReader,
                                                                     int batchSize,
                                                                     CancellationToken cancellationToken)
        {
            try
            {
                var groupedFiles = resultFiles
                                   .Select((resultFile, index) => new { Index = index, file = resultFile })
                                   .GroupBy(pair => pair.Index / batchSize)
                                   .Select(bucket => bucket.Select(pair => pair.file).ToList())
                                   .ToList();

                bool changeTestRunTitle = resultFiles.Count > 1;

                foreach (var files in groupedFiles)
                {
                    // Publish separate test run for each result file that has results.
                    var publishTasks = files.Select(async resultFile =>
                    {
                        cancellationToken.ThrowIfCancellationRequested();
                        string runName = _runTitle;
                        if (!string.IsNullOrWhiteSpace(_runTitle) && changeTestRunTitle)
                        {
                            runName = GetRunTitle();
                        }

                        _executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
                        TestRunData testRunData       = publisher.ReadResultsFromFile(runContext, resultFile, runName);
                        testRunData.PipelineReference = GetTestPipelineReference(runContext);
                        if (_failTaskOnFailedTests)
                        {
                            _isTestRunOutcomeFailed = _isTestRunOutcomeFailed || GetTestRunOutcome(testRunData);
                        }

                        cancellationToken.ThrowIfCancellationRequested();

                        if (testRunData != null)
                        {
                            if (testRunData.Results != null && testRunData.Results.Length > 0)
                            {
                                testRunData.AddCustomField(_testRunSystemCustomFieldName, _testRunSystem);
                                AddTargetBranchInfoToRunCreateModel(testRunData, runContext.PullRequestTargetBranchName);
                                TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
                                await publisher.AddResultsAsync(testRun, testRunData.Results, _executionContext.CancellationToken);
                                await publisher.EndTestRunAsync(testRunData, testRun.Id, cancellationToken: _executionContext.CancellationToken);
                            }
                            else
                            {
                                _executionContext.Output(StringUtil.Loc("NoResultFound", resultFile));
                            }
                        }
                        else
                        {
                            _executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
                        }
                    });
                    await Task.WhenAll(publishTasks);
                }
            }
            catch (Exception ex) when(!(ex is OperationCanceledException))
            {
                //Do not fail the task.
                LogPublishTestResultsFailureWarning(ex);
            }
        }
 private async Task UploadTestBuildLog(Guid projectId, BuildAttachment buildAttachment, TestRunContext runContext, CancellationToken cancellationToken)
 {
     await _testLogStore.UploadTestBuildLogAsync(projectId, runContext.BuildId, buildAttachment.TestLogType, buildAttachment.Filename, buildAttachment.Metadata, null, buildAttachment.AllowDuplicateUploads, buildAttachment.TestLogCompressionType, cancellationToken);
 }
        /// <summary>
        /// Reads a JUnit results file from disk, converts it into a TestRunData object.
        /// </summary>
        /// <param name="filePath"></param>
        /// <param name="runContext"></param>
        /// <returns></returns>
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
        {
            // http://windyroad.com.au/dl/Open%20Source/JUnit.xsd

            XmlDocument doc = new XmlDocument();

            try
            {
                var settings = new XmlReaderSettings
                {
                    DtdProcessing = DtdProcessing.Ignore
                };

                using (XmlReader reader = XmlReader.Create(filePath, settings))
                {
                    doc.Load(reader);
                }
            }
            catch (XmlException ex)
            {
                executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
                return(null);
            }

            //init test run summary information - run name, host name, start time
            TestSuiteSummary runSummary = new TestSuiteSummary(Name);

            IdentityRef runUserIdRef = null;
            string      runUser      = runContext != null ? runContext.Owner : string.Empty;

            if (!string.IsNullOrEmpty(runUser))
            {
                runUserIdRef = new IdentityRef()
                {
                    DisplayName = runUser
                };
            }

            var presentTime = DateTime.UtcNow;

            runSummary.TimeStamp = DateTime.MaxValue;
            var maxCompletedTime = DateTime.MinValue;

            //read data from testsuite nodes

            XmlNode testSuitesNode = doc.SelectSingleNode("testsuites");

            if (testSuitesNode != null)
            {
                //found testsuites node - some plugins generate it like karma junit plugin
                XmlNodeList testSuiteNodeList = doc.SelectNodes("/testsuites/testsuite");
                if (testSuiteNodeList != null)
                {
                    foreach (XmlNode testSuiteNode in testSuiteNodeList)
                    {
                        //for each available suites get all suite details
                        TestSuiteSummary testSuiteSummary = ReadTestSuite(testSuiteNode, runUserIdRef);

                        // sum up testsuite durations and test case durations, decision on what to use will be taken later
                        runSummary.TotalTestCaseDuration   = runSummary.TotalTestCaseDuration.Add(testSuiteSummary.TotalTestCaseDuration);
                        runSummary.TestSuiteDuration       = runSummary.TestSuiteDuration.Add(testSuiteSummary.TestSuiteDuration);
                        runSummary.SuiteTimeDataAvailable  = runSummary.SuiteTimeDataAvailable && testSuiteSummary.SuiteTimeDataAvailable;
                        runSummary.SuiteTimeStampAvailable = runSummary.SuiteTimeStampAvailable && testSuiteSummary.SuiteTimeStampAvailable;
                        runSummary.Host = testSuiteSummary.Host;
                        runSummary.Name = testSuiteSummary.Name;
                        //stop calculating timestamp information, if timestamp data is not avilable for even one test suite
                        if (testSuiteSummary.SuiteTimeStampAvailable)
                        {
                            runSummary.TimeStamp = runSummary.TimeStamp > testSuiteSummary.TimeStamp ? testSuiteSummary.TimeStamp : runSummary.TimeStamp;
                            DateTime completedTime = testSuiteSummary.TimeStamp.AddTicks(testSuiteSummary.TestSuiteDuration.Ticks);
                            maxCompletedTime = maxCompletedTime < completedTime ? completedTime : maxCompletedTime;
                        }
                        runSummary.Results.AddRange(testSuiteSummary.Results);
                    }

                    if (testSuiteNodeList.Count > 1)
                    {
                        runSummary.Name = Name + "_" + Path.GetFileName(filePath);
                    }
                }
            }
            else
            {
                executionContext.Output("Only single test suite found, parsing its information");
                XmlNode testSuiteNode = doc.SelectSingleNode("testsuite");
                if (testSuiteNode != null)
                {
                    runSummary = ReadTestSuite(testSuiteNode, runUserIdRef);
                    //only if start time is available then only we need to calculate completed time
                    if (runSummary.TimeStamp != DateTime.MaxValue)
                    {
                        DateTime completedTime = runSummary.TimeStamp.AddTicks(runSummary.TestSuiteDuration.Ticks);
                        maxCompletedTime = maxCompletedTime < completedTime ? completedTime : maxCompletedTime;
                    }
                    else
                    {
                        runSummary.SuiteTimeStampAvailable = false;
                    }
                }
            }

            if (runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName))
            {
                runSummary.Name = runContext.RunName;
            }

            if (!runSummary.SuiteTimeStampAvailable)
            {
                executionContext.Output("Timestamp is not available for one or more testsuites. Total run duration is being calculated as the sum of time durations of detected testsuites");

                if (!runSummary.SuiteTimeDataAvailable)
                {
                    executionContext.Output("Time is not available for one or more testsuites. Total run duration is being calculated as the sum of time durations of detected testcases");
                }
            }
            //if start time is not calculated then it should be initialized as present time
            runSummary.TimeStamp = runSummary.TimeStamp == DateTime.MaxValue
                ? presentTime
                : runSummary.TimeStamp;
            //if suite timestamp data is not available even for single testsuite, then fallback to testsuite run time
            //if testsuite run time is not available even for single testsuite, then fallback to total test case duration
            maxCompletedTime = !runSummary.SuiteTimeStampAvailable || maxCompletedTime == DateTime.MinValue
                ? runSummary.TimeStamp.Add(runSummary.SuiteTimeDataAvailable ? runSummary.TestSuiteDuration
                : runSummary.TotalTestCaseDuration) : maxCompletedTime;
            //create test run data
            var testRunData = new TestRunData(
                name: runSummary.Name,
                startedDate: runSummary.TimeStamp != DateTime.MinValue ? runSummary.TimeStamp.ToString("o") : null,
                completedDate: maxCompletedTime != DateTime.MinValue ? maxCompletedTime.ToString("o") : null,
                state: TestRunState.InProgress.ToString(),
                isAutomated: true,
                buildId: runContext != null ? runContext.BuildId : 0,
                buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
                buildPlatform: runContext != null ? runContext.Platform : string.Empty,
                releaseUri: runContext != null ? runContext.ReleaseUri : null,
                releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
                )
            {
                Results     = runSummary.Results.ToArray(),
                Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0]
            };

            return(testRunData);
        }
        /// <summary>
        /// Publish single test run
        /// </summary>
        private async Task PublishAllTestResultsToSingleTestRunAsync(List <string> resultFiles, ITestRunPublisher publisher, int buildId, TestRunContext runContext, string resultReader, CancellationToken cancellationToken)
        {
            try
            {
                //use local time since TestRunData defaults to local times
                DateTime                  minStartDate          = DateTime.MaxValue;
                DateTime                  maxCompleteDate       = DateTime.MinValue;
                DateTime                  presentTime           = DateTime.UtcNow;
                bool                      dateFormatError       = false;
                TimeSpan                  totalTestCaseDuration = TimeSpan.Zero;
                List <string>             runAttachments        = new List <string>();
                List <TestCaseResultData> runResults            = new List <TestCaseResultData>();

                //read results from each file
                foreach (string resultFile in resultFiles)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    //test case results
                    _executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
                    TestRunData resultFileRunData = publisher.ReadResultsFromFile(runContext, resultFile);

                    if (_failTaskOnFailedTests)
                    {
                        _isTestRunOutcomeFailed = _isTestRunOutcomeFailed || GetTestRunOutcome(resultFileRunData);
                    }

                    if (resultFileRunData != null)
                    {
                        if (resultFileRunData.Results != null && resultFileRunData.Results.Length > 0)
                        {
                            try
                            {
                                if (string.IsNullOrEmpty(resultFileRunData.StartDate) || string.IsNullOrEmpty(resultFileRunData.CompleteDate))
                                {
                                    dateFormatError = true;
                                }

                                //As per discussion with Manoj(refer bug 565487): Test Run duration time should be minimum Start Time to maximum Completed Time when merging
                                if (!string.IsNullOrEmpty(resultFileRunData.StartDate))
                                {
                                    DateTime startDate = DateTime.Parse(resultFileRunData.StartDate, null, DateTimeStyles.RoundtripKind);
                                    minStartDate = minStartDate > startDate ? startDate : minStartDate;

                                    if (!string.IsNullOrEmpty(resultFileRunData.CompleteDate))
                                    {
                                        DateTime endDate = DateTime.Parse(resultFileRunData.CompleteDate, null, DateTimeStyles.RoundtripKind);
                                        maxCompleteDate = maxCompleteDate < endDate ? endDate : maxCompleteDate;
                                    }
                                }
                            }
                            catch (FormatException)
                            {
                                _executionContext.Warning(StringUtil.Loc("InvalidDateFormat", resultFile, resultFileRunData.StartDate, resultFileRunData.CompleteDate));
                                dateFormatError = true;
                            }

                            //continue to calculate duration as a fallback for case: if there is issue with format or dates are null or empty
                            foreach (TestCaseResultData tcResult in resultFileRunData.Results)
                            {
                                int durationInMs = Convert.ToInt32(tcResult.DurationInMs);
                                totalTestCaseDuration = totalTestCaseDuration.Add(TimeSpan.FromMilliseconds(durationInMs));
                            }

                            runResults.AddRange(resultFileRunData.Results);

                            //run attachments
                            if (resultFileRunData.Attachments != null)
                            {
                                runAttachments.AddRange(resultFileRunData.Attachments);
                            }
                        }
                        else
                        {
                            _executionContext.Output(StringUtil.Loc("NoResultFound", resultFile));
                        }
                    }
                    else
                    {
                        _executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
                    }
                }

                //publish run if there are results.
                if (runResults.Count > 0)
                {
                    string runName = string.IsNullOrWhiteSpace(_runTitle)
                    ? StringUtil.Format("{0}_TestResults_{1}", _testRunner, buildId)
                    : _runTitle;

                    if (DateTime.Compare(minStartDate, maxCompleteDate) > 0)
                    {
                        _executionContext.Warning(StringUtil.Loc("InvalidCompletedDate", maxCompleteDate, minStartDate));
                        dateFormatError = true;
                    }

                    minStartDate    = DateTime.Equals(minStartDate, DateTime.MaxValue) ? presentTime : minStartDate;
                    maxCompleteDate = dateFormatError || DateTime.Equals(maxCompleteDate, DateTime.MinValue) ? minStartDate.Add(totalTestCaseDuration) : maxCompleteDate;

                    //creat test run
                    TestRunData testRunData = new TestRunData(
                        name: runName,
                        startedDate: minStartDate.ToString("o"),
                        completedDate: maxCompleteDate.ToString("o"),
                        state: "InProgress",
                        isAutomated: true,
                        buildId: runContext != null ? runContext.BuildId : 0,
                        buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
                        buildPlatform: runContext != null ? runContext.Platform : string.Empty,
                        releaseUri: runContext != null ? runContext.ReleaseUri : null,
                        releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
                        );
                    testRunData.PipelineReference = GetTestPipelineReference(runContext);
                    testRunData.Attachments       = runAttachments.ToArray();
                    testRunData.AddCustomField(_testRunSystemCustomFieldName, _testRunSystem);
                    AddTargetBranchInfoToRunCreateModel(testRunData, runContext.PullRequestTargetBranchName);

                    TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);

                    await publisher.AddResultsAsync(testRun, runResults.ToArray(), _executionContext.CancellationToken);

                    await publisher.EndTestRunAsync(testRunData, testRun.Id, true, _executionContext.CancellationToken);
                }
            }
            catch (Exception ex) when(!(ex is OperationCanceledException))
            {
                //Do not fail the task.
                LogPublishTestResultsFailureWarning(ex);
            }
        }
        /// <summary>
        /// Publish single test run
        /// </summary>
        private async Task PublishAllTestResultsToSingleTestRunAsync(List <string> resultFiles, ITestRunPublisher publisher, int buildId, TestRunContext runContext, string resultReader, CancellationToken cancellationToken)
        {
            try
            {
                DateTime                  startTime             = DateTime.Now; //use local time since TestRunData defaults to local times
                TimeSpan                  totalTestCaseDuration = TimeSpan.Zero;
                List <string>             runAttachments        = new List <string>();
                List <TestCaseResultData> runResults            = new List <TestCaseResultData>();

                //read results from each file
                foreach (string resultFile in resultFiles)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    //test case results
                    _executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
                    TestRunData resultFileRunData = publisher.ReadResultsFromFile(runContext, resultFile);

                    if (resultFileRunData != null && resultFileRunData.Results != null && resultFileRunData.Results.Length > 0)
                    {
                        foreach (TestCaseResultData tcResult in resultFileRunData.Results)
                        {
                            int durationInMs = 0;
                            int.TryParse(tcResult.DurationInMs, out durationInMs);
                            totalTestCaseDuration = totalTestCaseDuration.Add(TimeSpan.FromMilliseconds(durationInMs));
                        }
                        runResults.AddRange(resultFileRunData.Results);

                        //run attachments
                        if (resultFileRunData.Attachments != null)
                        {
                            runAttachments.AddRange(resultFileRunData.Attachments);
                        }
                    }
                    else
                    {
                        _executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
                    }
                }

                string runName = string.IsNullOrWhiteSpace(_runTitle)
                    ? StringUtil.Format("{0}_TestResults_{1}", _testRunner, buildId)
                    : _runTitle;

                //creat test run
                TestRunData testRunData = new TestRunData(
                    name: runName,
                    startedDate: startTime.ToString("o"),
                    completedDate: startTime.Add(totalTestCaseDuration).ToString("o"),
                    state: "InProgress",
                    isAutomated: true,
                    buildId: runContext != null ? runContext.BuildId : 0,
                    buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
                    buildPlatform: runContext != null ? runContext.Platform : string.Empty
                    );

                testRunData.Attachments = runAttachments.ToArray();

                //publish run if there are results.
                if (runResults.Count > 0)
                {
                    TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);

                    await publisher.AddResultsAsync(testRun, runResults.ToArray(), _executionContext.CancellationToken);

                    await publisher.EndTestRunAsync(testRunData, testRun.Id, true, _executionContext.CancellationToken);
                }
            }
            catch (Exception ex) when(!(ex is OperationCanceledException))
            {
                //Do not fail the task.
                LogPublishTestResultsFailureWarning(ex);
            }
        }
        /// <summary>
        /// Publish separate test run for each result file that has results.
        /// </summary>
        private async Task PublishToNewTestRunPerTestResultFileAsync(List <string> resultFiles, ITestRunPublisher publisher, TestRunContext runContext, string resultReader, CancellationToken cancellationToken)
        {
            try
            {
                // Publish separate test run for each result file that has results.
                var publishTasks = resultFiles.Select(async resultFile =>
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    string runName = null;
                    if (!string.IsNullOrWhiteSpace(_runTitle))
                    {
                        runName = GetRunTitle();
                    }

                    _executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
                    TestRunData testRunData = publisher.ReadResultsFromFile(runContext, resultFile, runName);

                    cancellationToken.ThrowIfCancellationRequested();

                    if (testRunData != null && testRunData.Results != null && testRunData.Results.Length > 0)
                    {
                        TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
                        await publisher.AddResultsAsync(testRun, testRunData.Results, _executionContext.CancellationToken);
                        await publisher.EndTestRunAsync(testRunData, testRun.Id, cancellationToken: _executionContext.CancellationToken);
                    }
                    else
                    {
                        _executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
                    }
                });
                await Task.WhenAll(publishTasks);
            }
            catch (Exception ex) when(!(ex is OperationCanceledException))
            {
                //Do not fail the task.
                LogPublishTestResultsFailureWarning(ex);
            }
        }
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext)
        {
            try
            {
                string jsonTestSummary = File.ReadAllText(filePath);
                if (string.IsNullOrWhiteSpace(jsonTestSummary))
                {
                    return(null);
                }

                JsonTestSummary testSummary = StringUtil.ConvertFromJson <JsonTestSummary>(jsonTestSummary);

                // Adding the minimum details from the JSON.
                TestRunData testRunData = new TestRunData(name: "Container Structure Test",
                                                          isAutomated: true,
                                                          buildId: runContext != null ? runContext.BuildId : 0,
                                                          buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
                                                          buildPlatform: runContext != null ? runContext.Platform : string.Empty,
                                                          releaseUri: runContext != null ? runContext.ReleaseUri : null,
                                                          releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null);

                List <TestCaseResultData> results = new List <TestCaseResultData>();

                foreach (JsonTestResult result in testSummary.Results)
                {
                    TestCaseResultData resultCreateModel = new TestCaseResultData();
                    resultCreateModel.TestCaseTitle     = result.Name;
                    resultCreateModel.AutomatedTestName = result.Name;
                    bool outcome = result.Pass.Equals("true", StringComparison.OrdinalIgnoreCase);

                    if (!outcome)
                    {
                        resultCreateModel.ErrorMessage = string.Join("|", result.Errors);
                    }

                    resultCreateModel.State             = "Completed";
                    resultCreateModel.AutomatedTestType = Name;
                    resultCreateModel.Outcome           = outcome ? TestOutcome.Passed.ToString() : TestOutcome.Failed.ToString();
                    results.Add(resultCreateModel);
                }

                testRunData.Results = results.ToArray();

                return(testRunData);
            }
            catch (Exception ex)
            {
                executionContext.Output("Error occured in reading results : " + ex);
            }
            return(null);
        }
Exemple #14
0
        //Based on the XUnit V2 format: http://xunit.github.io/docs/format-xml-v2.html
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
        {
            List <TestCaseResultData> results = new List <TestCaseResultData>();

            XmlDocument doc = new XmlDocument();

            try
            {
                var settings = new XmlReaderSettings
                {
                    DtdProcessing = DtdProcessing.Prohibit
                };

                using (XmlReader reader = XmlReader.Create(filePath, settings))
                {
                    doc.Load(reader);
                }
            }
            catch (XmlException ex)
            {
                executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
                return(null);
            }

            string runName = Name + " Test Run";
            string runUser = "";

            if (runContext != null)
            {
                if (!string.IsNullOrWhiteSpace(runContext.RunName))
                {
                    runName = runContext.RunName;
                }
                else
                {
                    runName = string.Format(CultureInfo.CurrentCulture, "{0} {1} {2}", runName, runContext.Configuration, runContext.Platform);
                }

                if (runContext.Owner != null)
                {
                    runUser = runContext.Owner;
                }
            }

            IdentityRef runUserIdRef = new IdentityRef();

            runUserIdRef.DisplayName = runUser;

            var    runStartTime    = DateTime.MinValue;
            double testRunDuration = 0;

            XmlNodeList assemblyNodes = doc.SelectNodes("/assemblies/assembly");

            foreach (XmlNode assemblyNode in assemblyNodes)
            {
                var assemblyRunStartTimeStamp = DateTime.MinValue;
                if (assemblyNode.Attributes["run-date"] != null && assemblyNode.Attributes["run-time"] != null)
                {
                    string runDate = assemblyNode.Attributes["run-date"].Value;
                    string runTime = assemblyNode.Attributes["run-time"].Value;

                    var startDate = DateTime.Now;
                    var startTime = TimeSpan.Zero;
                    if (DateTime.TryParse(runDate, out startDate))
                    {
                        TimeSpan.TryParse(runTime, out startTime);
                    }
                    assemblyRunStartTimeStamp = startDate + startTime;
                    if (testRunDuration == 0)
                    {
                        // first assembly start time is runstart time
                        runStartTime = assemblyRunStartTimeStamp;
                    }
                }

                XmlNodeList testCaseNodeList = assemblyNode.SelectNodes("./collection/test");
                foreach (XmlNode testCaseNode in testCaseNodeList)
                {
                    TestCaseResultData resultCreateModel = new TestCaseResultData();

                    //Test storage.
                    if (assemblyNode.Attributes["name"] != null)
                    {
                        resultCreateModel.AutomatedTestStorage = Path.GetFileName(assemblyNode.Attributes["name"].Value);
                    }

                    //Fully Qualified Name.
                    if (testCaseNode.Attributes["name"] != null)
                    {
                        resultCreateModel.AutomatedTestName = testCaseNode.Attributes["name"].Value;
                    }

                    //Test Method Name.
                    if (testCaseNode.Attributes["method"] != null)
                    {
                        resultCreateModel.TestCaseTitle = testCaseNode.Attributes["method"].Value;
                    }

                    //Test duration.
                    if (testCaseNode.Attributes["time"] != null && testCaseNode.Attributes["time"].Value != null)
                    {
                        double duration = 0;
                        double.TryParse(testCaseNode.Attributes["time"].Value, out duration);
                        var durationFromSeconds = TimeSpan.FromSeconds(duration);
                        resultCreateModel.DurationInMs = durationFromSeconds.TotalMilliseconds.ToString(CultureInfo.InvariantCulture);

                        // no assemblystarttime available so dont set testcase start and completed
                        if (assemblyRunStartTimeStamp != DateTime.MinValue)
                        {
                            resultCreateModel.StartedDate   = assemblyRunStartTimeStamp.ToString("o");
                            resultCreateModel.CompletedDate =
                                assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks).ToString("o");
                            assemblyRunStartTimeStamp = assemblyRunStartTimeStamp.AddTicks(1) + durationFromSeconds;
                            //next start time
                        }

                        //Calculate overall run duration.
                        testRunDuration += duration;
                    }


                    //Test outcome.
                    if (testCaseNode.SelectSingleNode("./failure") != null)
                    {
                        resultCreateModel.Outcome = TestOutcome.Failed.ToString();

                        //Error message.
                        XmlNode failureMessageNode = testCaseNode.SelectSingleNode("./failure/message");
                        if (failureMessageNode != null && !string.IsNullOrWhiteSpace(failureMessageNode.InnerText))
                        {
                            resultCreateModel.ErrorMessage = failureMessageNode.InnerText;
                        }

                        //Stack trace.
                        XmlNode failureStackTraceNode = testCaseNode.SelectSingleNode("./failure/stack-trace");
                        if (failureStackTraceNode != null && !string.IsNullOrWhiteSpace(failureStackTraceNode.InnerText))
                        {
                            resultCreateModel.StackTrace = failureStackTraceNode.InnerText;
                        }
                    }
                    else if (testCaseNode.Attributes["result"] != null && string.Equals(testCaseNode.Attributes["result"].Value, "pass", StringComparison.OrdinalIgnoreCase))
                    {
                        resultCreateModel.Outcome = TestOutcome.Passed.ToString();
                    }
                    else
                    {
                        resultCreateModel.Outcome = TestOutcome.NotExecuted.ToString();
                    }

                    //Test priority.
                    XmlNode priorityTrait = testCaseNode.SelectSingleNode("./traits/trait[@name='priority']");
                    if (priorityTrait != null && priorityTrait.Attributes["value"] != null)
                    {
                        resultCreateModel.TestCasePriority = priorityTrait.Attributes["value"].Value;
                    }

                    //Test owner.
                    XmlNode ownerNode = testCaseNode.SelectSingleNode("./traits/trait[@name='owner']");
                    if (ownerNode != null && ownerNode.Attributes["value"] != null && ownerNode.Attributes["value"].Value != null)
                    {
                        IdentityRef ownerIdRef = new IdentityRef();
                        ownerIdRef.DisplayName  = ownerNode.Attributes["value"].Value;
                        resultCreateModel.Owner = ownerIdRef;
                    }

                    resultCreateModel.RunBy = runUserIdRef;

                    resultCreateModel.State = "Completed";

                    resultCreateModel.AutomatedTestType = Name;

                    if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle))
                    {
                        results.Add(resultCreateModel);
                    }
                }
            }

            TestRunData testRunData = new TestRunData(
                name: runName,
                buildId: runContext != null ? runContext.BuildId : 0,
                startedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.ToString("o"),
                completedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.Add(TimeSpan.FromSeconds(testRunDuration)).ToString("o"),
                state: TestRunState.InProgress.ToString(),
                isAutomated: true,
                buildFlavor: runContext != null ? runContext.Configuration : null,
                buildPlatform: runContext != null ? runContext.Platform : null,
                releaseUri: runContext != null ? runContext.ReleaseUri : null,
                releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
                );

            testRunData.Results     = results.ToArray();
            testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];

            return(testRunData);
        }
        private void ProcessPublishTestResultsCommand(IExecutionContext context, Dictionary <string, string> eventProperties, string data)
        {
            ArgUtil.NotNull(context, nameof(context));
            _executionContext = context;

            LoadPublishTestResultsInputs(context, eventProperties, data);

            string teamProject = context.Variables.System_TeamProject;
            string owner       = context.Variables.Build_RequestedFor;
            string buildUri    = context.Variables.Build_BuildUri;
            int    buildId     = context.Variables.Build_BuildId ?? 0;
            string pullRequestTargetBranchName = context.Variables.System_PullRequest_TargetBranch;
            string stageName    = context.Variables.System_StageName;
            string phaseName    = context.Variables.System_PhaseName;
            string jobName      = context.Variables.System_JobName;
            int    stageAttempt = context.Variables.System_StageAttempt ?? 0;
            int    phaseAttempt = context.Variables.System_PhaseAttempt ?? 0;
            int    jobAttempt   = context.Variables.System_JobAttempt ?? 0;


            //Temporary fix to support publish in RM scenarios where there might not be a valid Build ID associated.
            //TODO: Make a cleaner fix after TCM User Story 401703 is completed.
            if (buildId == 0)
            {
                _platform = _configuration = null;
            }

            string releaseUri            = null;
            string releaseEnvironmentUri = null;

            // Check to identify if we are in the Release management flow; if not, then release fields will be kept null while publishing to TCM
            if (!string.IsNullOrWhiteSpace(context.Variables.Release_ReleaseUri))
            {
                releaseUri            = context.Variables.Release_ReleaseUri;
                releaseEnvironmentUri = context.Variables.Release_ReleaseEnvironmentUri;
            }

            IResultReader  resultReader = GetTestResultReader(_testRunner);
            TestRunContext runContext   = new TestRunContext(owner, _platform, _configuration, buildId, buildUri, releaseUri, releaseEnvironmentUri);

            runContext.StageName    = stageName;
            runContext.StageAttempt = stageAttempt;
            runContext.PhaseName    = phaseName;
            runContext.PhaseAttempt = phaseAttempt;
            runContext.JobName      = jobName;
            runContext.JobAttempt   = jobAttempt;

            runContext.PullRequestTargetBranchName = pullRequestTargetBranchName;

            VssConnection connection = WorkerUtilities.GetVssConnection(_executionContext);

            var publisher = HostContext.GetService <ITestRunPublisher>();

            publisher.InitializePublisher(context, connection, teamProject, resultReader);

            var commandContext = HostContext.CreateService <IAsyncCommandContext>();

            commandContext.InitializeCommandContext(context, StringUtil.Loc("PublishTestResults"));
            if (_mergeResults)
            {
                commandContext.Task = PublishAllTestResultsToSingleTestRunAsync(_testResultFiles, publisher, buildId, runContext, resultReader.Name, context.CancellationToken);
            }
            else
            {
                commandContext.Task = PublishToNewTestRunPerTestResultFileAsync(_testResultFiles, publisher, runContext, resultReader.Name, PublishBatchSize, context.CancellationToken);
            }
            _executionContext.AsyncCommands.Add(commandContext);

            if (_isTestRunOutcomeFailed)
            {
                _executionContext.Result = TaskResult.Failed;
                _executionContext.Error(StringUtil.Loc("FailedTestsInResults"));
            }
        }
 private async Task UploadRunDataAttachment(TestRunContext runContext, List <TestRunData> testRunData, PublishOptions publishOptions, CancellationToken cancellationToken = default(CancellationToken))
 {
     await _testRunPublisher.PublishTestRunDataAsync(runContext, _projectName, testRunData, publishOptions, cancellationToken);
 }
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
        {
            List<TestCaseResultData> results = new List<TestCaseResultData>();

            XmlDocument doc = new XmlDocument();
            try
            {
                var settings = new XmlReaderSettings
                {
                    DtdProcessing = DtdProcessing.Prohibit
                };

                using (XmlReader reader = XmlReader.Create(filePath, settings))
                {
                    doc.Load(reader);
                }
            }
            catch (XmlException ex)
            {
                executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
                return null;
            }

            //read test run summary information - run name, start time
            string runName = Name + " Test Run";
            DateTime runStartTime = DateTime.MinValue; //Use local time instead of UTC as TestRunData uses local time for defaults. Also assuming timestamp is local is more accurate in cases where tests were run on build machine
            TimeSpan totalRunDuration = TimeSpan.Zero;
            TimeSpan totalTestCaseDuration = TimeSpan.Zero;

            XmlNode testResultsNode = doc.SelectSingleNode("test-results");
            if (testResultsNode != null)
            {
                //get test run summary information
                if (testResultsNode.Attributes["name"] != null)
                {
                    runName = testResultsNode.Attributes["name"].Value;
                }

                //run times
                DateTime dateFromXml = DateTime.MinValue.Date; //Use local time instead of UTC as TestRunData uses local time for defaults.
                if (testResultsNode.Attributes["date"] != null)
                {
                    DateTime.TryParse(testResultsNode.Attributes["date"].Value, out dateFromXml);
                }

                TimeSpan timeFromXml = TimeSpan.Zero;
                if (testResultsNode.Attributes["time"] != null)
                {
                    TimeSpan.TryParse(testResultsNode.Attributes["time"].Value, out timeFromXml);
                }

                //assume runtimes from xml are current local time since timezone information is not in the xml, if xml datetime > current local time, fallback to local start time
                DateTime runStartDateTimeFromXml = new DateTime(dateFromXml.Ticks).AddTicks(timeFromXml.Ticks);
                if (runStartTime == DateTime.MinValue)
                {
                    runStartTime = runStartDateTimeFromXml;
                }
            }

            //run environment - platform, config and hostname
            string platform = runContext != null ? runContext.Platform : string.Empty;
            string config = runContext != null ? runContext.Configuration : string.Empty;
            string runUser = runContext != null ? runContext.Owner : string.Empty;
            string hostName = string.Empty;

            XmlNode envNode = doc.SelectSingleNode("test-results/environment");
            if (envNode != null)
            {
                if (envNode.Attributes["machine-name"] != null && envNode.Attributes["machine-name"].Value != null)
                {
                    hostName = envNode.Attributes["machine-name"].Value;
                }

                if (envNode.Attributes["platform"] != null && envNode.Attributes["platform"].Value != null && runContext != null && runContext.BuildId > 0)
                {
                    //We cannot publish platform information without a valid build id.
                    platform = envNode.Attributes["platform"].Value;
                }
            }

            //run owner
            IdentityRef runUserIdRef = null;
            if (!string.IsNullOrEmpty(runUser))
            {
                runUserIdRef = new IdentityRef() { DisplayName = runUser };
            }

            //get all test assemblies
            if (testResultsNode != null)
            {
                XmlNodeList testAssemblyNodes = testResultsNode.SelectNodes("test-suite");
                if (testAssemblyNodes != null)
                {
                    foreach (XmlNode testAssemblyNode in testAssemblyNodes)
                    {
                        var assemblyStartTime = (runStartTime == DateTime.MinValue) ? DateTime.MinValue : runStartTime + totalTestCaseDuration;
                        List<TestCaseResultData> testCases = FindTestCaseNodes(testAssemblyNode, hostName, runUserIdRef, assemblyStartTime);
                        if (testCases != null)
                        {
                            results.AddRange(testCases);
                            testCases.ForEach(x => totalTestCaseDuration += TimeSpan.FromMilliseconds(double.Parse(x.DurationInMs)));
                        }
                    }
                }
            }

            if (TimeSpan.Compare(totalRunDuration, totalTestCaseDuration) < 0)
            {
                totalRunDuration = totalTestCaseDuration; //run duration may not be set in the xml, so use total test case duration 
            }

            if (runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName))
            {
                runName = runContext.RunName;
            }

            //create test run data
            TestRunData testRunData = new TestRunData(
                name: runName,
                startedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.ToString("o"),
                completedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.Add(totalRunDuration).ToString("o"),
                state: TestRunState.InProgress.ToString(),
                isAutomated: true,
                buildId: runContext != null ? runContext.BuildId : 0,
                buildFlavor: config,
                buildPlatform: platform,
                releaseUri: runContext != null ? runContext.ReleaseUri : null,
                releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
                );

            testRunData.Results = results.ToArray();
            testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];

            return testRunData;
        }
        /// <summary>
        /// Reads a CTest results file from disk, converts it into a TestRunData object.
        /// </summary>
        /// <param name="filePath"></param>
        /// <param name="runContext"></param>
        /// <returns></returns>
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
        {
            _ec = executionContext;

            // Read Xml File
            XmlDocument doc = new XmlDocument();

            try
            {
                XmlReaderSettings settings = new XmlReaderSettings
                {
                    DtdProcessing = DtdProcessing.Ignore
                };

                using (XmlReader reader = XmlReader.Create(filePath, settings))
                {
                    doc.Load(reader);
                }
            }
            catch (XmlException ex)
            {
                executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
                return(null);
            }

            // init test run summary information - run name, host name, start time
            List <TestCaseResultData> results = new List <TestCaseResultData>();
            string runName = ParseRunName(runContext);

            InitialiseRunUserIdRef(runContext);

            //run environment - platform, config and hostname
            string platform = runContext != null ? runContext.Platform : string.Empty;
            string config   = runContext != null ? runContext.Configuration : string.Empty;
            string runUser  = runContext != null ? runContext.Owner : string.Empty;
            string hostName = string.Empty;

            //Parse the run start and finish times.
            XmlNode node = doc.SelectSingleNode("/Site/Testing");

            _runStartDate  = DateTime.MinValue;
            _runFinishDate = DateTime.MinValue;
            ParseRunStartAndFinishDates(node);

            //create test run data object
            TestRunData testRunData = new TestRunData(
                name: runName,
                startedDate: (_runStartDate == DateTime.MinValue) ? string.Empty : _runStartDate.ToString("o"),
                completedDate: (_runFinishDate == DateTime.MinValue) ? string.Empty : _runFinishDate.ToString("o"),
                state: TestRunState.InProgress.ToString(),
                isAutomated: true,
                buildId: runContext != null ? runContext.BuildId : 0,
                buildFlavor: config,
                buildPlatform: platform,
                releaseUri: runContext != null ? runContext.ReleaseUri : null,
                releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
                )
            {
                Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0]
            };
            // Read results
            XmlNodeList resultsNodes = doc.SelectNodes("/Site/Testing/Test");

            results.AddRange(ReadActualResults(resultsNodes));
            testRunData.Results = results.ToArray();

            return(testRunData);
        }
Exemple #19
0
        public new TestRunData GetTestRunData(string filePath, XmlDocument doc, XmlNode testResultsNode, TestRunContext runContext, bool addResultsAsAttachments)
        {
            this.InnerTestSuiteNodeName = "results/test-suite";
            this.TestCaseNodeName       = "results/test-case";
            this.RootNodeName           = "test-results";

            return(base.GetTestRunData(filePath, doc, testResultsNode, runContext, addResultsAsAttachments));
        }
        //Based on the XUnit V2 format: http://xunit.github.io/docs/format-xml-v2.html
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
        {
            List<TestCaseResultData> results = new List<TestCaseResultData>();

            XmlDocument doc = new XmlDocument();
            try
            {
                var settings = new XmlReaderSettings
                {
                    DtdProcessing = DtdProcessing.Prohibit
                };

                using (XmlReader reader = XmlReader.Create(filePath, settings))
                {
                    doc.Load(reader);
                }
            }
            catch (XmlException ex)
            {
                executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
                return null;
            }

            string runName = Name + " Test Run";
            string runUser = "";
            if (runContext != null)
            {
                if (!string.IsNullOrWhiteSpace(runContext.RunName))
                {
                    runName = runContext.RunName;
                }
                else
                {
                    runName = string.Format(CultureInfo.CurrentCulture, "{0} {1} {2}", runName, runContext.Configuration, runContext.Platform);
                }

                if (runContext.Owner != null)
                {
                    runUser = runContext.Owner;
                }
            }

            IdentityRef runUserIdRef = new IdentityRef();
            runUserIdRef.DisplayName = runUser;

            var runStartTime = DateTime.MinValue;
            double testRunDuration = 0;

            XmlNodeList assemblyNodes = doc.SelectNodes("/assemblies/assembly");
            foreach (XmlNode assemblyNode in assemblyNodes)
            {
                var assemblyRunStartTimeStamp = DateTime.MinValue;
                if (assemblyNode.Attributes["run-date"] != null && assemblyNode.Attributes["run-time"] != null)
                {
                    string runDate = assemblyNode.Attributes["run-date"].Value;
                    string runTime = assemblyNode.Attributes["run-time"].Value;

                    var startDate = DateTime.Now;
                    var startTime = TimeSpan.Zero;
                    if (DateTime.TryParse(runDate, out startDate))
                    {
                        TimeSpan.TryParse(runTime, out startTime);
                    }
                    assemblyRunStartTimeStamp = startDate + startTime;
                    if (testRunDuration == 0)
                    {
                        // first assembly start time is runstart time
                        runStartTime = assemblyRunStartTimeStamp;
                    }
                }

                XmlNodeList testCaseNodeList = assemblyNode.SelectNodes("./collection/test");
                foreach (XmlNode testCaseNode in testCaseNodeList)
                {
                    TestCaseResultData resultCreateModel = new TestCaseResultData();

                    //Test storage.
                    if (assemblyNode.Attributes["name"] != null)
                    {
                        resultCreateModel.AutomatedTestStorage = Path.GetFileName(assemblyNode.Attributes["name"].Value);
                    }

                    //Fully Qualified Name.
                    if (testCaseNode.Attributes["name"] != null)
                    {
                        resultCreateModel.AutomatedTestName = testCaseNode.Attributes["name"].Value;
                    }

                    //Test Method Name.
                    if (testCaseNode.Attributes["method"] != null)
                    {
                        resultCreateModel.TestCaseTitle = testCaseNode.Attributes["method"].Value;
                    }

                    //Test duration.
                    if (testCaseNode.Attributes["time"] != null && testCaseNode.Attributes["time"].Value != null)
                    {
                        double duration = 0;
                        double.TryParse(testCaseNode.Attributes["time"].Value, out duration);
                        var durationFromSeconds = TimeSpan.FromSeconds(duration);
                        resultCreateModel.DurationInMs = durationFromSeconds.TotalMilliseconds.ToString(CultureInfo.InvariantCulture);

                        // no assemblystarttime available so dont set testcase start and completed
                        if (assemblyRunStartTimeStamp != DateTime.MinValue)
                        {
                            resultCreateModel.StartedDate = assemblyRunStartTimeStamp.ToString("o");
                            resultCreateModel.CompletedDate =
                                assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks).ToString("o");
                            assemblyRunStartTimeStamp = assemblyRunStartTimeStamp.AddTicks(1) + durationFromSeconds;
                            //next start time
                        }

                        //Calculate overall run duration.
                        testRunDuration += duration;
                    }


                    //Test outcome.
                    if (testCaseNode.SelectSingleNode("./failure") != null)
                    {
                        resultCreateModel.Outcome = TestOutcome.Failed.ToString();

                        //Error message.
                        XmlNode failureMessageNode = testCaseNode.SelectSingleNode("./failure/message");
                        if (failureMessageNode != null && !string.IsNullOrWhiteSpace(failureMessageNode.InnerText))
                        {
                            resultCreateModel.ErrorMessage = failureMessageNode.InnerText;
                        }

                        //Stack trace.
                        XmlNode failureStackTraceNode = testCaseNode.SelectSingleNode("./failure/stack-trace");
                        if (failureStackTraceNode != null && !string.IsNullOrWhiteSpace(failureStackTraceNode.InnerText))
                        {
                            resultCreateModel.StackTrace = failureStackTraceNode.InnerText;
                        }
                    }
                    else if (testCaseNode.Attributes["result"] != null && string.Equals(testCaseNode.Attributes["result"].Value, "pass", StringComparison.OrdinalIgnoreCase))
                    {
                        resultCreateModel.Outcome = TestOutcome.Passed.ToString();
                    }
                    else
                    {
                        resultCreateModel.Outcome = TestOutcome.NotExecuted.ToString();
                    }

                    //Test priority.
                    XmlNode priorityTrait = testCaseNode.SelectSingleNode("./traits/trait[@name='priority']");
                    if (priorityTrait != null && priorityTrait.Attributes["value"] != null)
                    {
                        resultCreateModel.TestCasePriority = priorityTrait.Attributes["value"].Value;
                    }

                    //Test owner.
                    XmlNode ownerNode = testCaseNode.SelectSingleNode("./traits/trait[@name='owner']");
                    if (ownerNode != null && ownerNode.Attributes["value"] != null && ownerNode.Attributes["value"].Value != null)
                    {
                        IdentityRef ownerIdRef = new IdentityRef();
                        ownerIdRef.DisplayName = ownerNode.Attributes["value"].Value;
                        resultCreateModel.Owner = ownerIdRef;
                    }

                    resultCreateModel.RunBy = runUserIdRef;

                    resultCreateModel.State = "Completed";

                    resultCreateModel.AutomatedTestType = Name;

                    if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle))
                    {
                        results.Add(resultCreateModel);
                    }
                }
            }

            TestRunData testRunData = new TestRunData(
                name: runName,
                buildId: runContext != null ? runContext.BuildId : 0,
                startedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.ToString("o"),
                completedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.Add(TimeSpan.FromSeconds(testRunDuration)).ToString("o"),
                state: TestRunState.InProgress.ToString(),
                isAutomated: true,
                buildFlavor: runContext != null ? runContext.Configuration : null,
                buildPlatform: runContext != null ? runContext.Platform : null,
                releaseUri: runContext != null ? runContext.ReleaseUri : null,
                releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
                );

            testRunData.Results = results.ToArray();
            testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];

            return testRunData;
        }
Exemple #21
0
        public TestRunData GetTestRunData(string filePath, XmlDocument doc, XmlNode testResultsNode, TestRunContext runContext, bool addResultsAsAttachments)
        {
            var testRunNode      = doc.SelectSingleNode("test-run");
            var testRunStartedOn = DateTime.MinValue;
            var testRunEndedOn   = DateTime.MinValue;
            var testCaseResults  = new List <TestCaseResultData>();

            _runUserIdRef = runContext != null ? new IdentityRef()
            {
                DisplayName = runContext.Owner
            } : null;
            _platform = runContext != null ? runContext.Platform : string.Empty;
            if (testRunNode.Attributes["start-time"] != null)
            {
                DateTime.TryParse(testRunNode.Attributes["start-time"]?.Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.None, out testRunStartedOn);
                DateTime.TryParse(testRunNode.Attributes["end-time"]?.Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.None, out testRunEndedOn);
                var testAssemblyNodes = testRunNode.SelectNodes("//test-suite[@type='Assembly']");
                if (testAssemblyNodes != null)
                {
                    foreach (XmlNode testAssemblyNode in testAssemblyNodes)
                    {
                        var environmentNode = testAssemblyNode.SelectSingleNode("environment");
                        var hostname        = String.Empty;
                        var assemblyName    = (testAssemblyNode.Attributes["name"] != null ? testAssemblyNode.Attributes["name"].Value : null);
                        if (environmentNode != null)
                        {
                            if (environmentNode.Attributes["machine-name"] != null)
                            {
                                hostname = environmentNode.Attributes["machine-name"].Value;
                            }
                            if (environmentNode.Attributes["platform"] != null)
                            {
                                // override platform
                                _platform = environmentNode.Attributes["platform"].Value;
                            }
                        }
                        var testCaseNodes = testAssemblyNode.SelectNodes(".//test-case");
                        if (testCaseNodes != null)
                        {
                            foreach (XmlNode testCaseNode in testCaseNodes)
                            {
                                testCaseResults.Add(getTestCaseResultData(testCaseNode, assemblyName, hostname));
                            }
                        }
                    }
                }
            }
            TestRunData testRunData = new TestRunData(
                name: runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName) ? runContext.RunName : _defaultRunName,
                startedDate: (testRunStartedOn == DateTime.MinValue ? string.Empty : testRunStartedOn.ToString("o")),
                completedDate: (testRunEndedOn == DateTime.MinValue ? string.Empty : testRunEndedOn.ToString("o")),
                state: TestRunState.InProgress.ToString(),
                isAutomated: true,
                buildId: runContext != null ? runContext.BuildId : 0,
                buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
                buildPlatform: _platform,
                releaseUri: runContext != null ? runContext.ReleaseUri : null,
                releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
                );

            testRunData.Results     = testCaseResults.ToArray();
            testRunData.Attachments = addResultsAsAttachments ? new string[] { filePath } : new string[0];
            return(testRunData);
        }
Exemple #22
0
        private TestRunContext CreateTestRunContext()
        {
            string releaseUri            = null;
            string releaseEnvironmentUri = null;

            string teamProject = _executionContext.Variables.System_TeamProject;
            string owner       = _executionContext.Variables.Build_RequestedFor;
            string buildUri    = _executionContext.Variables.Build_BuildUri;
            int    buildId     = _executionContext.Variables.Build_BuildId ?? 0;
            string pullRequestTargetBranchName = _executionContext.Variables.System_PullRequest_TargetBranch;
            string stageName    = _executionContext.Variables.System_StageName;
            string phaseName    = _executionContext.Variables.System_PhaseName;
            string jobName      = _executionContext.Variables.System_JobName;
            int    stageAttempt = _executionContext.Variables.System_StageAttempt ?? 0;
            int    phaseAttempt = _executionContext.Variables.System_PhaseAttempt ?? 0;
            int    jobAttempt   = _executionContext.Variables.System_JobAttempt ?? 0;

            //Temporary fix to support publish in RM scenarios where there might not be a valid Build ID associated.
            //TODO: Make a cleaner fix after TCM User Story 401703 is completed.
            if (buildId == 0)
            {
                _platform = _configuration = null;
            }

            if (!string.IsNullOrWhiteSpace(_executionContext.Variables.Release_ReleaseUri))
            {
                releaseUri            = _executionContext.Variables.Release_ReleaseUri;
                releaseEnvironmentUri = _executionContext.Variables.Release_ReleaseEnvironmentUri;
            }

            // If runName is not provided by the task, then create runName from testRunner name and buildId.
            string runName = String.IsNullOrWhiteSpace(_runTitle)
                ? String.Format("{0}_TestResults_{1}", _testRunner, buildId)
                : _runTitle;

            StageReference stageReference = new StageReference()
            {
                StageName = stageName, Attempt = Convert.ToInt32(stageAttempt)
            };
            PhaseReference phaseReference = new PhaseReference()
            {
                PhaseName = phaseName, Attempt = Convert.ToInt32(phaseAttempt)
            };
            JobReference jobReference = new JobReference()
            {
                JobName = jobName, Attempt = Convert.ToInt32(jobAttempt)
            };
            PipelineReference pipelineReference = new PipelineReference()
            {
                PipelineId     = buildId,
                StageReference = stageReference,
                PhaseReference = phaseReference,
                JobReference   = jobReference
            };

            TestRunContext testRunContext = new TestRunContext(
                owner: owner,
                platform: _platform,
                configuration: _configuration,
                buildId: buildId,
                buildUri: buildUri,
                releaseUri: releaseUri,
                releaseEnvironmentUri: releaseEnvironmentUri,
                runName: runName,
                testRunSystem: _testRunSystem,
                buildAttachmentProcessor: new CodeCoverageBuildAttachmentProcessor(),
                targetBranchName: pullRequestTargetBranchName,
                pipelineReference: pipelineReference
                );

            return(testRunContext);
        }
Exemple #23
0
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
        {
            var doc = new XmlDocument();

            try
            {
                var settings = new XmlReaderSettings
                {
                    DtdProcessing = DtdProcessing.Ignore
                };

                using (XmlReader reader = XmlReader.Create(filePath, settings))
                {
                    doc.Load(reader);
                }
            }
            catch (XmlException ex)
            {
                executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
                return(null);
            }

            INUnitResultsXmlReader nunitResultsReader = null;
            var testResultsNode = doc.SelectSingleNode("test-results");

            if (testResultsNode == null)
            {
                testResultsNode = doc.SelectSingleNode("test-run");
                if (testResultsNode == null)
                {
                    throw new NotSupportedException(StringUtil.Loc("InvalidFileFormat"));
                }
                nunitResultsReader = new NUnit3ResultsXmlReader();
            }
            else
            {
                nunitResultsReader = new NUnit2ResultsXmlReader();
            }

            return(nunitResultsReader.GetTestRunData(filePath, doc, testResultsNode, runContext, AddResultsFileToRunLevelAttachments));
        }
        /// <summary>
        /// Publish separate test run for each result file that has results.
        /// </summary>
        private async Task PublishToNewTestRunPerTestResultFileAsync(List<string> resultFiles, ITestRunPublisher publisher, TestRunContext runContext, string resultReader, CancellationToken cancellationToken)
        {
            try
            {
                // Publish separate test run for each result file that has results.
                var publishTasks = resultFiles.Select(async resultFile =>
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    string runName = null;
                    if (!string.IsNullOrWhiteSpace(_runTitle))
                    {
                        runName = GetRunTitle();
                    }

                    _executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
                    TestRunData testRunData = publisher.ReadResultsFromFile(runContext, resultFile, runName);

                    cancellationToken.ThrowIfCancellationRequested();

                    if (testRunData != null && testRunData.Results != null && testRunData.Results.Length > 0)
                    {
                        TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
                        await publisher.AddResultsAsync(testRun, testRunData.Results, _executionContext.CancellationToken);
                        await publisher.EndTestRunAsync(testRunData, testRun.Id, cancellationToken: _executionContext.CancellationToken);
                    }
                    else
                    {
                        _executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
                    }
                });
                await Task.WhenAll(publishTasks);
            }
            catch (Exception ex) when (!(ex is OperationCanceledException))
            {
                //Do not fail the task.
                LogPublishTestResultsFailureWarning(ex);
            }
        }
Exemple #25
0
 /// <summary>
 /// Converts the given results file to TestRunData object
 /// </summary>
 /// <param name="filePath">File path</param>
 /// <param name="runName">Run Name</param>
 /// <returns>TestRunData</returns>
 public TestRunData ReadResultsFromFile(TestRunContext runContext, string filePath, string runName)
 {
     Trace.Entering();
     runContext.RunName = runName;
     return(_resultReader.ReadResults(_executionContext, filePath, runContext));
 }
        /// <summary>
        /// Publish single test run
        /// </summary>
        private async Task PublishAllTestResultsToSingleTestRunAsync(List<string> resultFiles, ITestRunPublisher publisher, int buildId, TestRunContext runContext, string resultReader, CancellationToken cancellationToken)
        {
            try
            {
                DateTime startTime = DateTime.Now; //use local time since TestRunData defaults to local times
                TimeSpan totalTestCaseDuration = TimeSpan.Zero;
                List<string> runAttachments = new List<string>();
                List<TestCaseResultData> runResults = new List<TestCaseResultData>();

                //read results from each file
                foreach (string resultFile in resultFiles)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    //test case results
                    _executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
                    TestRunData resultFileRunData = publisher.ReadResultsFromFile(runContext, resultFile);

                    if (resultFileRunData != null && resultFileRunData.Results != null && resultFileRunData.Results.Length > 0)
                    {
                        foreach (TestCaseResultData tcResult in resultFileRunData.Results)
                        {
                            int durationInMs = 0;
                            int.TryParse(tcResult.DurationInMs, out durationInMs);
                            totalTestCaseDuration = totalTestCaseDuration.Add(TimeSpan.FromMilliseconds(durationInMs));
                        }
                        runResults.AddRange(resultFileRunData.Results);

                        //run attachments
                        if (resultFileRunData.Attachments != null)
                        {
                            runAttachments.AddRange(resultFileRunData.Attachments);
                        }
                    }
                    else
                    {
                        _executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
                    }
                }

                string runName = string.IsNullOrWhiteSpace(_runTitle)
                    ? StringUtil.Format("{0}_TestResults_{1}", _testRunner, buildId)
                    : _runTitle;

                //creat test run
                TestRunData testRunData = new TestRunData(
                    name: runName,
                    startedDate: startTime.ToString("o"),
                    completedDate: startTime.Add(totalTestCaseDuration).ToString("o"),
                    state: "InProgress",
                    isAutomated: true,
                    buildId: runContext != null ? runContext.BuildId : 0,
                    buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
                    buildPlatform: runContext != null ? runContext.Platform : string.Empty
                    );

                testRunData.Attachments = runAttachments.ToArray();

                //publish run if there are results.
                if (runResults.Count > 0)
                {
                    TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
                    await publisher.AddResultsAsync(testRun, runResults.ToArray(), _executionContext.CancellationToken);
                    await publisher.EndTestRunAsync(testRunData, testRun.Id, true, _executionContext.CancellationToken);
                }
            }
            catch (Exception ex) when (!(ex is OperationCanceledException))
            {
                //Do not fail the task.
                LogPublishTestResultsFailureWarning(ex);
            }
        }
Exemple #27
0
        public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null)
        {
            List <TestCaseResultData> results = new List <TestCaseResultData>();

            XmlDocument doc = new XmlDocument();

            try
            {
                var settings = new XmlReaderSettings
                {
                    DtdProcessing = DtdProcessing.Ignore
                };

                using (XmlReader reader = XmlReader.Create(filePath, settings))
                {
                    doc.Load(reader);
                }
            }
            catch (XmlException ex)
            {
                executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message));
                return(null);
            }

            //read test run summary information - run name, start time
            string   runName               = Name + " Test Run";
            DateTime runStartTime          = DateTime.MinValue; //Use local time instead of UTC as TestRunData uses local time for defaults. Also assuming timestamp is local is more accurate in cases where tests were run on build machine
            TimeSpan totalRunDuration      = TimeSpan.Zero;
            TimeSpan totalTestCaseDuration = TimeSpan.Zero;

            XmlNode testResultsNode = doc.SelectSingleNode("test-results");

            if (testResultsNode != null)
            {
                //get test run summary information
                if (testResultsNode.Attributes["name"] != null)
                {
                    runName = testResultsNode.Attributes["name"].Value;
                }

                //run times
                DateTime dateFromXml = DateTime.MinValue.Date; //Use local time instead of UTC as TestRunData uses local time for defaults.
                if (testResultsNode.Attributes["date"] != null)
                {
                    DateTime.TryParse(testResultsNode.Attributes["date"].Value, out dateFromXml);
                }

                TimeSpan timeFromXml = TimeSpan.Zero;
                if (testResultsNode.Attributes["time"] != null)
                {
                    TimeSpan.TryParse(testResultsNode.Attributes["time"].Value, out timeFromXml);
                }

                //assume runtimes from xml are current local time since timezone information is not in the xml, if xml datetime > current local time, fallback to local start time
                DateTime runStartDateTimeFromXml = new DateTime(dateFromXml.Ticks).AddTicks(timeFromXml.Ticks);
                if (runStartTime == DateTime.MinValue)
                {
                    runStartTime = runStartDateTimeFromXml;
                }
            }

            //run environment - platform, config and hostname
            string platform = runContext != null ? runContext.Platform : string.Empty;
            string config   = runContext != null ? runContext.Configuration : string.Empty;
            string runUser  = runContext != null ? runContext.Owner : string.Empty;
            string hostName = string.Empty;

            XmlNode envNode = doc.SelectSingleNode("test-results/environment");

            if (envNode != null)
            {
                if (envNode.Attributes["machine-name"] != null && envNode.Attributes["machine-name"].Value != null)
                {
                    hostName = envNode.Attributes["machine-name"].Value;
                }

                if (envNode.Attributes["platform"] != null && envNode.Attributes["platform"].Value != null && runContext != null && runContext.BuildId > 0)
                {
                    //We cannot publish platform information without a valid build id.
                    platform = envNode.Attributes["platform"].Value;
                }
            }

            //run owner
            IdentityRef runUserIdRef = null;

            if (!string.IsNullOrEmpty(runUser))
            {
                runUserIdRef = new IdentityRef()
                {
                    DisplayName = runUser
                };
            }

            //get all test assemblies
            if (testResultsNode != null)
            {
                XmlNodeList testAssemblyNodes = testResultsNode.SelectNodes("test-suite");
                if (testAssemblyNodes != null)
                {
                    foreach (XmlNode testAssemblyNode in testAssemblyNodes)
                    {
                        var assemblyStartTime = (runStartTime == DateTime.MinValue) ? DateTime.MinValue : runStartTime + totalTestCaseDuration;
                        List <TestCaseResultData> testCases = FindTestCaseNodes(testAssemblyNode, hostName, runUserIdRef, assemblyStartTime);
                        if (testCases != null)
                        {
                            results.AddRange(testCases);
                            testCases.ForEach(x => totalTestCaseDuration += TimeSpan.FromMilliseconds(x.DurationInMs));
                        }
                    }
                }
            }

            if (TimeSpan.Compare(totalRunDuration, totalTestCaseDuration) < 0)
            {
                totalRunDuration = totalTestCaseDuration; //run duration may not be set in the xml, so use total test case duration
            }

            if (runContext != null && !string.IsNullOrWhiteSpace(runContext.RunName))
            {
                runName = runContext.RunName;
            }

            //create test run data
            TestRunData testRunData = new TestRunData(
                name: runName,
                startedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.ToString("o"),
                completedDate: (runStartTime == DateTime.MinValue) ? string.Empty : runStartTime.Add(totalRunDuration).ToString("o"),
                state: TestRunState.InProgress.ToString(),
                isAutomated: true,
                buildId: runContext != null ? runContext.BuildId : 0,
                buildFlavor: config,
                buildPlatform: platform,
                releaseUri: runContext != null ? runContext.ReleaseUri : null,
                releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
                );

            testRunData.Results     = results.ToArray();
            testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0];

            return(testRunData);
        }
        private void ReadResults(TestRunContext runContext = null, bool attachRunLevelAttachments = true)
        {
            _fileName = Path.Combine(Path.GetTempPath(), Path.GetTempFileName());
            File.WriteAllText(_fileName, _junitResultsToBeRead);

            _jUnitReader = new JUnitResultReader();
            _jUnitReader.AddResultsFileToRunLevelAttachments = attachRunLevelAttachments;
            _testRunData = _jUnitReader.ReadResults(_ec.Object, _fileName, runContext);
        }
        public TestDataProvider ParseTestResultFiles(IExecutionContext executionContext, TestRunContext testRunContext, List <string> testResultsFiles)
        {
            if (string.IsNullOrEmpty(Name))
            {
                executionContext.Warning("Test runner name is null or empty");
                return(null);
            }
            // Create test result parser object based on the test Runner provided
            var testResultParser = GetTestResultParser(executionContext);

            if (testResultParser == null)
            {
                return(null);
            }

            // Parse with the corresponding testResultParser object
            return(ParseFiles(executionContext, testRunContext, testResultsFiles, testResultParser));
        }
 /// <summary>
 /// Converts the given results file to TestRunData object
 /// </summary>
 /// <param name="filePath">File path</param>
 /// <param name="runName">Run Name</param>
 /// <returns>TestRunData</returns>
 public TestRunData ReadResultsFromFile(TestRunContext runContext, string filePath, string runName)
 {
     Trace.Entering();
     runContext.RunName = runName;
     return _resultReader.ReadResults(_executionContext, filePath, runContext);
 }
Exemple #31
0
        private async Task PublishTestResultsAsync(VssConnection connection, String teamProject, int buildId, TestRunContext runContext, IResultReader resultReader)
        {
            var publisher = HostContext.GetService <ITestRunPublisher>();

            publisher.InitializePublisher(_executionContext, connection, teamProject, resultReader);

            if (_mergeResults)
            {
                await PublishAllTestResultsToSingleTestRunAsync(_testResultFiles, publisher, buildId, runContext, resultReader.Name, _executionContext.CancellationToken);
            }
            else
            {
                await PublishToNewTestRunPerTestResultFileAsync(_testResultFiles, publisher, runContext, resultReader.Name, PublishBatchSize, _executionContext.CancellationToken);
            }

            await PublishEventsAsync(connection);
        }