private void AddVariousLogsForResultIfOutcomeIsFailed(TestCaseResultData resultCreateModel, XmlNode resultNode) { if (!resultCreateModel.Outcome.Equals("Failed")) { return; } XmlNode errorMessage, errorStackTrace, consoleLog, standardError; if ((errorMessage = resultNode.SelectSingleNode("./Output/ErrorInfo/Message")) != null && !string.IsNullOrWhiteSpace(errorMessage.InnerText)) { resultCreateModel.ErrorMessage = errorMessage.InnerText; } // stack trace if ((errorStackTrace = resultNode.SelectSingleNode("./Output/ErrorInfo/StackTrace")) != null && !string.IsNullOrWhiteSpace(errorStackTrace.InnerText)) { resultCreateModel.StackTrace = errorStackTrace.InnerText; } // console log if ((consoleLog = resultNode.SelectSingleNode("./Output/StdOut")) != null && !string.IsNullOrWhiteSpace(consoleLog.InnerText)) { resultCreateModel.AttachmentData.ConsoleLog = consoleLog.InnerText; } // standard error if ((standardError = resultNode.SelectSingleNode("./Output/StdErr")) != null && !string.IsNullOrWhiteSpace(standardError.InnerText)) { resultCreateModel.AttachmentData.StandardError = standardError.InnerText; } }
private async Task UploadTestResultsAttachmentAsync(int testRunId, TestCaseResultData testCaseResultData, TestCaseResult testCaseResult, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); if (testCaseResult == null || testCaseResultData == null) { return; } if (testCaseResultData.AttachmentData != null) { // Remove duplicate entries string[] attachments = testCaseResultData.AttachmentData.AttachmentsFilePathList?.ToArray(); HashSet <string> attachedFiles = GetUniqueTestRunFiles(attachments); if (attachedFiles != null && attachedFiles.Any()) { var createAttachmentsTasks = attachedFiles.Select(async attachment => { TestAttachmentRequestModel reqModel = GetAttachmentRequestModel(attachment); if (reqModel != null) { await _testResultsServer.CreateTestResultAttachmentAsync(reqModel, _projectName, testRunId, testCaseResult.Id, cancellationToken); } }); await Task.WhenAll(createAttachmentsTasks); } // Upload console log as attachment string consoleLog = testCaseResultData?.AttachmentData.ConsoleLog; TestAttachmentRequestModel attachmentRequestModel = GetConsoleLogAttachmentRequestModel(consoleLog); if (attachmentRequestModel != null) { await _testResultsServer.CreateTestResultAttachmentAsync(attachmentRequestModel, _projectName, testRunId, testCaseResult.Id, cancellationToken); } // Upload standard error as attachment string standardError = testCaseResultData.AttachmentData.StandardError; TestAttachmentRequestModel stdErrAttachmentRequestModel = GetStandardErrorAttachmentRequestModel(standardError); if (stdErrAttachmentRequestModel != null) { await _testResultsServer.CreateTestResultAttachmentAsync(stdErrAttachmentRequestModel, _projectName, testRunId, testCaseResult.Id, cancellationToken); } } if (testCaseResult.SubResults != null && testCaseResult.SubResults.Any() && testCaseResultData.TestCaseSubResultData != null) { for (int i = 0; i < testCaseResultData.TestCaseSubResultData.Count; i++) { await UploadTestSubResultsAttachmentAsync(testRunId, testCaseResult.Id, testCaseResultData.TestCaseSubResultData[i], testCaseResult.SubResults[i], 1, cancellationToken); } } }
public static void Convert(TestCaseResultData testCaseResultData, TestCaseResult testCaseResultWebApi) { ArgUtil.NotNull(testCaseResultData, nameof(testCaseResultData)); ArgUtil.NotNull(testCaseResultWebApi, nameof(testCaseResultWebApi)); testCaseResultWebApi.Area = testCaseResultData.Area; testCaseResultWebApi.AssociatedBugs = testCaseResultData.AssociatedBugs; testCaseResultWebApi.AutomatedTestId = testCaseResultData.AutomatedTestId; testCaseResultWebApi.AutomatedTestName = testCaseResultData.AutomatedTestName; testCaseResultWebApi.AutomatedTestStorage = testCaseResultData.AutomatedTestStorage; testCaseResultWebApi.AutomatedTestType = testCaseResultData.AutomatedTestType; testCaseResultWebApi.AutomatedTestTypeId = testCaseResultData.AutomatedTestTypeId; testCaseResultWebApi.Build = testCaseResultData.Build; testCaseResultWebApi.BuildReference = testCaseResultData.BuildReference; testCaseResultWebApi.Comment = testCaseResultData.Comment; testCaseResultWebApi.CompletedDate = testCaseResultData.CompletedDate; testCaseResultWebApi.ComputerName = testCaseResultData.ComputerName; testCaseResultWebApi.Configuration = testCaseResultData.Configuration; testCaseResultWebApi.CreatedDate = testCaseResultData.CreatedDate; testCaseResultWebApi.CustomFields = testCaseResultData.CustomFields; testCaseResultWebApi.DurationInMs = testCaseResultData.DurationInMs; testCaseResultWebApi.ErrorMessage = testCaseResultData.ErrorMessage; testCaseResultWebApi.FailureType = testCaseResultData.FailureType; testCaseResultWebApi.Id = testCaseResultData.Id; testCaseResultWebApi.IterationDetails = testCaseResultData.IterationDetails; testCaseResultWebApi.LastUpdatedBy = testCaseResultData.LastUpdatedBy; testCaseResultWebApi.LastUpdatedDate = testCaseResultData.LastUpdatedDate; testCaseResultWebApi.Outcome = testCaseResultData.Outcome; testCaseResultWebApi.Owner = testCaseResultData.Owner; testCaseResultWebApi.Priority = testCaseResultData.Priority; testCaseResultWebApi.Project = testCaseResultData.Project; testCaseResultWebApi.Release = testCaseResultData.Release; testCaseResultWebApi.ReleaseReference = testCaseResultData.ReleaseReference; testCaseResultWebApi.ResetCount = testCaseResultData.ResetCount; testCaseResultWebApi.ResolutionState = testCaseResultData.ResolutionState; testCaseResultWebApi.ResolutionStateId = testCaseResultData.ResolutionStateId; testCaseResultWebApi.ResultGroupType = testCaseResultData.ResultGroupType; testCaseResultWebApi.Revision = testCaseResultData.Revision; testCaseResultWebApi.RunBy = testCaseResultData.RunBy; testCaseResultWebApi.StackTrace = testCaseResultData.StackTrace; testCaseResultWebApi.StartedDate = testCaseResultData.StartedDate; testCaseResultWebApi.State = testCaseResultData.State; testCaseResultWebApi.TestCase = testCaseResultData.TestCase; testCaseResultWebApi.TestCaseReferenceId = testCaseResultData.TestCaseReferenceId; testCaseResultWebApi.TestCaseTitle = testCaseResultData.TestCaseTitle; testCaseResultWebApi.TestPlan = testCaseResultData.TestPlan; testCaseResultWebApi.TestPoint = testCaseResultData.TestPoint; testCaseResultWebApi.TestRun = testCaseResultData.TestRun; testCaseResultWebApi.TestSuite = testCaseResultData.TestSuite; testCaseResultWebApi.Url = testCaseResultData.Url; ConvertSubResults(testCaseResultData, testCaseResultWebApi); }
private void ProcessFailureNode(XmlNode failure, TestCaseResultData resultCreateModel) { resultCreateModel.Outcome = TestOutcome.Failed.ToString(); if (failure.Attributes["message"] != null && !string.IsNullOrWhiteSpace(failure.Attributes["message"].Value)) { resultCreateModel.ErrorMessage = failure.Attributes["message"].Value; } if (!string.IsNullOrWhiteSpace(failure.InnerText)) { resultCreateModel.StackTrace = failure.InnerText; } }
public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext) { try { string jsonTestSummary = File.ReadAllText(filePath); if (string.IsNullOrWhiteSpace(jsonTestSummary)) { return(null); } JsonTestSummary testSummary = StringUtil.ConvertFromJson <JsonTestSummary>(jsonTestSummary); // Adding the minimum details from the JSON. TestRunData testRunData = new TestRunData(name: "Container Structure Test", isAutomated: true, buildId: runContext != null ? runContext.BuildId : 0, buildFlavor: runContext != null ? runContext.Configuration : string.Empty, buildPlatform: runContext != null ? runContext.Platform : string.Empty, releaseUri: runContext != null ? runContext.ReleaseUri : null, releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null); List <TestCaseResultData> results = new List <TestCaseResultData>(); foreach (JsonTestResult result in testSummary.Results) { TestCaseResultData resultCreateModel = new TestCaseResultData(); resultCreateModel.TestCaseTitle = result.Name; resultCreateModel.AutomatedTestName = result.Name; bool outcome = result.Pass.Equals("true", StringComparison.OrdinalIgnoreCase); if (!outcome) { resultCreateModel.ErrorMessage = string.Join("|", result.Errors); } resultCreateModel.State = "Completed"; resultCreateModel.AutomatedTestType = Name; resultCreateModel.Outcome = outcome ? TestOutcome.Passed.ToString() : TestOutcome.Failed.ToString(); results.Add(resultCreateModel); } testRunData.Results = results.ToArray(); return(testRunData); } catch (Exception ex) { executionContext.Output("Error occured in reading results : " + ex); } return(null); }
private static void ConvertSubResults(TestCaseResultData testCaseResultData, TestCaseResult testCaseResultWebApi) { if (testCaseResultData.TestCaseSubResultData == null || !testCaseResultData.TestCaseSubResultData.Any()) { return; } testCaseResultWebApi.SubResults = new List <TestSubResult>(); foreach (var subResultData in testCaseResultData.TestCaseSubResultData) { var subResultWebApi = new TestSubResult(); TestCaseSubResultDataConverter.Convert(subResultData, subResultWebApi); testCaseResultWebApi.SubResults.Add(subResultWebApi); } }
/// <summary> /// Publishes the given results to the test run. /// </summary> /// <param name="testResults">Results to be published.</param> public async Task AddResultsAsync(TestRun testRun, TestCaseResultData[] testResults, CancellationToken cancellationToken) { ArgUtil.NotNull(testRun, nameof(testRun)); ArgUtil.NotNull(testResults, nameof(testResults)); Trace.Entering(); int noOfResultsToBePublished = BATCH_SIZE; _executionContext.Output(StringUtil.Loc("PublishingTestResults", testRun.Id)); for (int i = 0; i < testResults.Length; i += BATCH_SIZE) { cancellationToken.ThrowIfCancellationRequested(); if (i + BATCH_SIZE >= testResults.Length) { noOfResultsToBePublished = testResults.Length - i; } _executionContext.Output(StringUtil.Loc("TestResultsRemaining", (testResults.Length - i), testRun.Id)); var currentBatch = new TestCaseResultData[noOfResultsToBePublished]; var testResultsBatch = new TestCaseResult[noOfResultsToBePublished]; Array.Copy(testResults, i, currentBatch, 0, noOfResultsToBePublished); for (int testResultsIndex = 0; testResultsIndex < noOfResultsToBePublished; testResultsIndex++) { if (IsMaxLimitReachedForSubresultPreProcessing(currentBatch[testResultsIndex].AutomatedTestName, currentBatch[testResultsIndex].TestCaseSubResultData) == false) { _executionContext.Warning(StringUtil.Loc("MaxHierarchyLevelReached", TestManagementConstants.maxHierarchyLevelForSubresults)); currentBatch[testResultsIndex].TestCaseSubResultData = null; } testResultsBatch[testResultsIndex] = new TestCaseResult(); TestCaseResultDataConverter.Convert(currentBatch[testResultsIndex], testResultsBatch[testResultsIndex]); } List <TestCaseResult> uploadedTestResults = await _testResultsServer.AddTestResultsToTestRunAsync(testResultsBatch, _projectName, testRun.Id, cancellationToken); for (int j = 0; j < noOfResultsToBePublished; j++) { await this.UploadTestResultsAttachmentAsync(testRun.Id, testResults[i + j], uploadedTestResults[j], cancellationToken); } } Trace.Leaving(); }
private void AddSystemLogsToResult(XmlNode testCaseNode, TestCaseResultData resultCreateModel) { XmlNode stdout, stderr; // Standard output logs stdout = testCaseNode.SelectSingleNode("./system-out"); resultCreateModel.AttachmentData = new AttachmentData(); if (stdout != null && !string.IsNullOrWhiteSpace(stdout.InnerText)) { resultCreateModel.AttachmentData.ConsoleLog = stdout.InnerText; } // Standard error logs stderr = testCaseNode.SelectSingleNode("./system-err"); if (stderr != null && !string.IsNullOrWhiteSpace(stderr.InnerText)) { resultCreateModel.AttachmentData.StandardError = stderr.InnerText; } }
private List <TestCaseResultData> ReadActualResults(XmlNodeList resultsNodes, TestType testType) { List <TestCaseResultData> results = new List <TestCaseResultData>(); object sync = new object(); Parallel.ForEach <XmlNode>(resultsNodes.Cast <XmlNode>(), resultNode => { TestCaseResultData resultCreateModel = new TestCaseResultData() { Priority = TestManagementConstants.UnspecifiedPriority, //Priority is int type so if no priority set then its 255. }; //Find and format dates as per TCM requirement. TimeSpan duration; if (resultNode.Attributes["duration"] != null && resultNode.Attributes["duration"].Value != null) { TimeSpan.TryParse(resultNode.Attributes["duration"].Value, CultureInfo.InvariantCulture, out duration); } else { duration = TimeSpan.Zero; } resultCreateModel.DurationInMs = duration.TotalMilliseconds; DateTime startedDate; if (resultNode.Attributes["startTime"] != null && resultNode.Attributes["startTime"].Value != null) { DateTime.TryParse(resultNode.Attributes["startTime"].Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out startedDate); } else { startedDate = DateTime.UtcNow; } resultCreateModel.StartedDate = startedDate; DateTime completedDate = startedDate.AddTicks(duration.Ticks); resultCreateModel.CompletedDate = completedDate; if ((DateTime.Compare(default(DateTime), startedDate) < 0 && DateTime.Compare(startedDate, (DateTime)SqlDateTime.MinValue) <= 0) || (DateTime.Compare(default(DateTime), completedDate) < 0 && DateTime.Compare(completedDate, (DateTime)SqlDateTime.MinValue) <= 0)) { DateTime utcNow = DateTime.UtcNow; resultCreateModel.StartedDate = utcNow; resultCreateModel.CompletedDate = utcNow.AddTicks(duration.Ticks); } if (resultNode.Attributes["outcome"] == null || resultNode.Attributes["outcome"].Value == null || string.Equals(resultNode.Attributes["outcome"].Value, "failed", StringComparison.OrdinalIgnoreCase)) { resultCreateModel.Outcome = TestOutcome.Failed.ToString();; } else if (string.Equals(resultNode.Attributes["outcome"].Value, "passed", StringComparison.OrdinalIgnoreCase)) { resultCreateModel.Outcome = TestOutcome.Passed.ToString(); } else if (string.Equals(resultNode.Attributes["outcome"].Value, "inconclusive", StringComparison.OrdinalIgnoreCase)) { resultCreateModel.Outcome = TestOutcome.Inconclusive.ToString(); } else { resultCreateModel.Outcome = TestOutcome.NotExecuted.ToString(); } if (resultNode.Attributes["testName"] != null && resultNode.Attributes["testName"].Value != null) { resultCreateModel.TestCaseTitle = resultNode.Attributes["testName"].Value; } resultCreateModel.State = "Completed"; resultCreateModel.AutomatedTestType = testType.ToString(); if (resultNode.Attributes["computerName"] != null && resultNode.Attributes["computerName"].Value != null) { resultCreateModel.ComputerName = resultNode.Attributes["computerName"].Value; } if (resultNode.Attributes["testId"] != null && resultNode.Attributes["testId"].Value != null) { resultCreateModel.AutomatedTestId = resultNode.Attributes["testId"].Value; } string executionId = null; if (resultNode.Attributes["executionId"] != null && resultNode.Attributes["executionId"].Value != null) { executionId = resultNode.Attributes["executionId"].Value; } lock (sync) { if (resultCreateModel.AutomatedTestId != null && _definitions.ContainsKey(resultCreateModel.AutomatedTestId)) { TestCaseDefinition definition = _definitions[resultCreateModel.AutomatedTestId]; if (definition != null) { if (definition.Storage != null) { resultCreateModel.AutomatedTestStorage = definition.Storage; } if (definition.Priority != null) { resultCreateModel.Priority = !string.IsNullOrEmpty(definition.Priority) ? Convert.ToInt32(definition.Priority) : TestManagementConstants.UnspecifiedPriority; } if (definition.Owner != null) { resultCreateModel.Owner = definition.Owner; } if (definition.AutomatedTestName != null) { resultCreateModel.AutomatedTestName = definition.AutomatedTestName; } } } //AutomatedTestId should be a valid guid. Delaying the check to here since we use it as dictionary key above. Guid automatedTestId; if (!Guid.TryParse(resultCreateModel.AutomatedTestId, out automatedTestId)) { resultCreateModel.AutomatedTestId = null; } } if (resultNode.Attributes["testType"] != null && resultNode.Attributes["testType"].Value != null) { Guid automatedTestType; if (Guid.TryParse(resultNode.Attributes["testType"].Value, out automatedTestType)) { resultCreateModel.AutomatedTestTypeId = resultNode.Attributes["testType"].Value; } } resultCreateModel.RunBy = _runUserIdRef; List <string> resulLeveltAttachments = new List <string>() { }; XmlNodeList resultAttachmentNodes = resultNode.SelectNodes("CollectorDataEntries/Collector/UriAttachments/UriAttachment/A"); if (resultAttachmentNodes.Count > 0 && executionId != null) { foreach (XmlNode resultAttachmentNode in resultAttachmentNodes) { if (resultAttachmentNode.Attributes["href"]?.Value != null) { resulLeveltAttachments.Add(Path.Combine(_attachmentLocation, executionId, resultAttachmentNode.Attributes["href"].Value)); } } } XmlNodeList resultFileNodes = resultNode.SelectNodes("ResultFiles/ResultFile"); if (resultFileNodes.Count > 0 && executionId != null) { foreach (XmlNode resultFileNode in resultFileNodes) { if (resultFileNode.Attributes["path"]?.Value != null) { resulLeveltAttachments.Add(Path.Combine(_attachmentLocation, executionId, resultFileNode.Attributes["path"].Value)); } } } resultCreateModel.AttachmentData = new AttachmentData() { AttachmentsFilePathList = resulLeveltAttachments.ToArray() }; AddVariousLogsForResultIfOutcomeIsFailed(resultCreateModel, resultNode); XmlNode innerResults = resultNode.SelectSingleNode("InnerResults"); if (innerResults != null) { resultCreateModel.ResultGroupType = GetResultGroupType(resultNode, testType); XmlNodeList resNodes = innerResults.SelectNodes("UnitTestResult"); XmlNodeList webTestResultNodes = innerResults.SelectNodes("WebTestResult"); XmlNodeList orderedTestResultNodes = innerResults.SelectNodes("TestResultAggregation"); resultCreateModel.TestCaseSubResultData = new List <TestCaseSubResultData>(); resultCreateModel.TestCaseSubResultData.AddRange(ReadActualSubResults(resNodes, TestType.UnitTest, 1)); resultCreateModel.TestCaseSubResultData.AddRange(ReadActualSubResults(webTestResultNodes, TestType.WebTest, 1)); resultCreateModel.TestCaseSubResultData.AddRange(ReadActualSubResults(orderedTestResultNodes, TestType.OrderedTest, 1)); } lock (sync) { //Mandatory fields. Skip if they are not available. if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle)) { results.Add(resultCreateModel); } } }); return(results); }
private TestCaseResultData getTestCaseResultData(XmlNode testCaseResultNode, string assemblyName, string hostname) { var testCaseResultData = new TestCaseResultData(); if (!string.IsNullOrEmpty(assemblyName)) { testCaseResultData.AutomatedTestStorage = assemblyName; } testCaseResultData.ComputerName = hostname; testCaseResultData.TestCaseTitle = testCaseResultNode.Attributes["name"]?.Value; testCaseResultData.AutomatedTestName = testCaseResultNode.Attributes["fullname"]?.Value; double duration = 0; double.TryParse(testCaseResultNode.Attributes["duration"]?.Value, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out duration); // Ensure Duration cannot be negative duration = (duration < 0) ? 0 : duration; testCaseResultData.DurationInMs = TimeSpan.FromSeconds(duration).TotalMilliseconds; var testExecutionStartedOn = DateTime.MinValue; DateTime.TryParse(testCaseResultNode.Attributes["start-time"]?.Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out testExecutionStartedOn); testCaseResultData.StartedDate = testExecutionStartedOn; var testExecutionEndedOn = DateTime.MinValue; DateTime.TryParse(testCaseResultNode.Attributes["end-time"]?.Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out testExecutionEndedOn); testCaseResultData.CompletedDate = testExecutionEndedOn; testCaseResultData.AttachmentData = new AttachmentData(); if (testCaseResultNode.Attributes["result"] != null) { if (string.Equals(testCaseResultNode.Attributes["result"].Value, "Passed", StringComparison.OrdinalIgnoreCase)) { testCaseResultData.Outcome = TestOutcome.Passed.ToString(); } else if (string.Equals(testCaseResultNode.Attributes["result"].Value, "Failed", StringComparison.OrdinalIgnoreCase)) { testCaseResultData.Outcome = TestOutcome.Failed.ToString(); } else if (string.Equals(testCaseResultNode.Attributes["result"].Value, "Skipped", StringComparison.OrdinalIgnoreCase)) { testCaseResultData.Outcome = TestOutcome.NotExecuted.ToString(); } else { testCaseResultData.Outcome = TestOutcome.Inconclusive.ToString(); } var failureNode = testCaseResultNode.SelectSingleNode("failure"); if (failureNode != null) { var failureMessageNode = failureNode.SelectSingleNode("message"); var failureStackTraceNode = failureNode.SelectSingleNode("stack-trace"); testCaseResultData.ErrorMessage = failureMessageNode?.InnerText; testCaseResultData.StackTrace = failureStackTraceNode?.InnerText; // console log XmlNode consoleLog = testCaseResultNode.SelectSingleNode("output"); if (consoleLog != null && !string.IsNullOrWhiteSpace(consoleLog.InnerText)) { testCaseResultData.AttachmentData.ConsoleLog = consoleLog.InnerText; } } } testCaseResultData.State = "Completed"; testCaseResultData.AutomatedTestType = "NUnit"; if (_runUserIdRef != null) { testCaseResultData.RunBy = _runUserIdRef; testCaseResultData.Owner = _runUserIdRef; } // Adding test-case result level attachments testCaseResultData.AttachmentData.AttachmentsFilePathList = this.GetTestCaseResultLevelAttachments(testCaseResultNode).ToArray(); return(testCaseResultData); }
private List <TestCaseResultData> FindTestCaseNodes(XmlNode startNode, string hostName, IdentityRef runUserIdRef, DateTime assemblyStartTime, string assemblyName = null) { List <TestCaseResultData> results = new List <TestCaseResultData>(); string testStorage = assemblyName; if (startNode.Attributes["type"] != null && startNode.Attributes["type"].Value != null && startNode.Attributes["type"].Value.Equals("assembly", StringComparison.OrdinalIgnoreCase)) { if (startNode.Attributes["name"] != null && startNode.Attributes["name"].Value != null) { testStorage = startNode.Attributes["name"].Value; } } //get each test case result information XmlNodeList testCaseNodes = startNode.SelectNodes(TestCaseNodeName); //all test-case nodes under testAssemblyNode if (testCaseNodes != null) { DateTime testCaseStartTime = assemblyStartTime; foreach (XmlNode testCaseNode in testCaseNodes) { TestCaseResultData resultCreateModel = new TestCaseResultData(); //test case name and type if (testCaseNode.Attributes["name"] != null && testCaseNode.Attributes["name"].Value != null) { resultCreateModel.TestCaseTitle = testCaseNode.Attributes["name"].Value; resultCreateModel.AutomatedTestName = testCaseNode.Attributes["name"].Value; } if (!string.IsNullOrEmpty(testStorage)) { resultCreateModel.AutomatedTestStorage = testStorage; } //test case duration, starttime and endtime TimeSpan testCaseDuration = TimeSpan.Zero; if (testCaseNode.Attributes["time"] != null && testCaseNode.Attributes["time"].Value != null) { double duration = 0; double.TryParse(testCaseNode.Attributes["time"].Value, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out duration); // Duration of a test case cannot be less than zero testCaseDuration = (duration < 0) ? testCaseDuration : TimeSpan.FromSeconds(duration); } resultCreateModel.DurationInMs = testCaseDuration.TotalMilliseconds; if (assemblyStartTime != DateTime.MinValue) { resultCreateModel.StartedDate = testCaseStartTime; resultCreateModel.CompletedDate = testCaseStartTime.AddTicks(testCaseDuration.Ticks); testCaseStartTime = testCaseStartTime.AddTicks(1) + testCaseDuration; //next start time } //test run outcome if (testCaseNode.SelectSingleNode("./failure") != null) { resultCreateModel.Outcome = "Failed"; XmlNode failureMessageNode, failureStackTraceNode; if ((failureMessageNode = testCaseNode.SelectSingleNode("./failure/message")) != null && !string.IsNullOrWhiteSpace(failureMessageNode.InnerText)) { resultCreateModel.ErrorMessage = failureMessageNode.InnerText; } // stack trace if ((failureStackTraceNode = testCaseNode.SelectSingleNode("./failure/stack-trace")) != null && !string.IsNullOrWhiteSpace(failureStackTraceNode.InnerText)) { resultCreateModel.StackTrace = failureStackTraceNode.InnerText; } // console log resultCreateModel.AttachmentData = new AttachmentData(); XmlNode consoleLog = testCaseNode.SelectSingleNode("./output"); if (consoleLog != null && !string.IsNullOrWhiteSpace(consoleLog.InnerText)) { resultCreateModel.AttachmentData.ConsoleLog = consoleLog.InnerText; } } else { if (testCaseNode.Attributes["result"] != null && string.Equals(testCaseNode.Attributes["result"].Value, "Ignored", StringComparison.OrdinalIgnoreCase)) { resultCreateModel.Outcome = "NotExecuted"; } else { resultCreateModel.Outcome = "Passed"; } } resultCreateModel.State = "Completed"; resultCreateModel.AutomatedTestType = "NUnit"; //other properties if (runUserIdRef != null) { resultCreateModel.RunBy = runUserIdRef; resultCreateModel.Owner = runUserIdRef; } resultCreateModel.ComputerName = hostName; if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle)) { results.Add(resultCreateModel); } } } XmlNodeList testSuiteNodes = startNode.SelectNodes(InnerTestSuiteNodeName); if (testSuiteNodes != null) { foreach (XmlNode testSuiteNode in testSuiteNodes) { results.AddRange(FindTestCaseNodes(testSuiteNode, hostName, runUserIdRef, assemblyStartTime, testStorage)); } } return(results); }
/// <summary> /// Reads all test result nodes /// </summary> /// <param name="resultsNodes"></param> /// <returns>List of test case results</returns> private List <TestCaseResultData> ReadActualResults(XmlNodeList resultsNodes) { List <TestCaseResultData> results = new List <TestCaseResultData>(); var resultXmlNodes = resultsNodes.Cast <XmlNode>(); foreach (var testCaseNode in resultXmlNodes) { TestCaseResultData resultCreateModel = new TestCaseResultData(); // find test case title and other information resultCreateModel.TestCaseTitle = testCaseNode.SelectSingleNode("./Name")?.InnerText; resultCreateModel.AutomatedTestName = testCaseNode.SelectSingleNode("./FullName")?.InnerText; resultCreateModel.AutomatedTestStorage = testCaseNode.SelectSingleNode("./Path")?.InnerText; resultCreateModel.AutomatedTestType = ParserName; // find duration of test case, starttime and endtime resultCreateModel.DurationInMs = (long)GetTestCaseResultDuration(testCaseNode).TotalMilliseconds; // start time of test case is kept as run start time sice test case start is not available resultCreateModel.StartedDate = _runStartDate; resultCreateModel.CompletedDate = resultCreateModel.StartedDate.AddMilliseconds(resultCreateModel.DurationInMs); // find test case outcome resultCreateModel.Outcome = GetTestCaseOutcome(testCaseNode).ToString(); // If test outcome is failed, fill stacktrace and error message if (resultCreateModel.Outcome.ToString().Equals(TestOutcome.Failed.ToString())) { XmlNode failure; // Stacktrace if ((failure = testCaseNode.SelectSingleNode("./Results/Measurement/Value")) != null) { if (!string.IsNullOrEmpty(failure.InnerText)) { resultCreateModel.StackTrace = failure.InnerText; } } } else { // fill console logs resultCreateModel.AttachmentData = new AttachmentData(); XmlNode stdOutputLog = testCaseNode.SelectSingleNode("./Results/Measurement/Value"); if (!string.IsNullOrEmpty(stdOutputLog?.InnerText)) { resultCreateModel.AttachmentData.ConsoleLog = stdOutputLog.InnerText; } } resultCreateModel.State = "Completed"; //other properties if (_runUserIdRef != null) { resultCreateModel.RunBy = _runUserIdRef; resultCreateModel.Owner = _runUserIdRef; } //Mandatory fields. Skip if they are not available. if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle)) { results.Add(resultCreateModel); } } return(results); }
/// <summary> /// Read testcases under testsuite node in xml /// </summary> /// <param name="rootNode"></param> private TestSuiteSummary ReadTestSuite(XmlNode rootNode, IdentityRef runUserIdRef) { TestSuiteSummary testSuiteSummary = new TestSuiteSummary(Name); XmlNodeList innerTestSuiteNodeList = rootNode.SelectNodes("./testsuite"); if (innerTestSuiteNodeList != null) { foreach (XmlNode innerTestSuiteNode in innerTestSuiteNodeList) { TestSuiteSummary innerTestSuiteSummary = ReadTestSuite(innerTestSuiteNode, runUserIdRef); testSuiteSummary.Results.AddRange(innerTestSuiteSummary.Results); } } TimeSpan totalTestSuiteDuration = TimeSpan.Zero; TimeSpan totalTestCaseDuration = TimeSpan.Zero; if (rootNode.Attributes["name"] != null && rootNode.Attributes["name"].Value != null && rootNode.Attributes["name"].Value != string.Empty) { testSuiteSummary.Name = rootNode.Attributes["name"].Value; } if (rootNode.Attributes["hostname"] != null && rootNode.Attributes["hostname"].Value != null) { testSuiteSummary.Host = rootNode.Attributes["hostname"].Value; } //assume runtimes from xml are current local time since timezone information is not in the xml, if xml datetime > current local time, fallback to local start time DateTime timestampFromXml = DateTime.MinValue; XmlAttribute timestampNode = rootNode.Attributes["timestamp"]; if (timestampNode != null && timestampNode.Value != null) { if (DateTime.TryParse(timestampNode.Value, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out timestampFromXml)) { testSuiteSummary.TimeStamp = timestampFromXml; } } if (timestampFromXml == DateTime.MinValue) { testSuiteSummary.SuiteTimeStampAvailable = false; } bool SuiteTimeDataAvailable = false; totalTestSuiteDuration = GetTimeSpan(rootNode, out SuiteTimeDataAvailable); testSuiteSummary.SuiteTimeDataAvailable = SuiteTimeDataAvailable; var testSuiteStartTime = testSuiteSummary.TimeStamp; //find test case nodes in JUnit result xml XmlNodeList testCaseNodes = rootNode.SelectNodes("./testcase"); if (testCaseNodes != null) { DateTime testCaseStartTime = testSuiteStartTime; //Add test case results to the test run foreach (XmlNode testCaseNode in testCaseNodes) { TestCaseResultData resultCreateModel = new TestCaseResultData(); //test case name and type if (testCaseNode.Attributes["name"] != null && testCaseNode.Attributes["name"].Value != null) { resultCreateModel.TestCaseTitle = testCaseNode.Attributes["name"].Value; resultCreateModel.AutomatedTestName = testCaseNode.Attributes["name"].Value; } if (testCaseNode.Attributes["classname"] != null && testCaseNode.Attributes["classname"].Value != null) { resultCreateModel.AutomatedTestStorage = testCaseNode.Attributes["classname"].Value; } if (testCaseNode.Attributes["owner"]?.Value != null) { var ownerName = testCaseNode.Attributes["owner"].Value; resultCreateModel.Owner = new IdentityRef { DisplayName = ownerName, DirectoryAlias = ownerName }; } //test case duration bool TestCaseTimeDataAvailable = false; var testCaseDuration = GetTimeSpan(testCaseNode, out TestCaseTimeDataAvailable); totalTestCaseDuration = totalTestCaseDuration + testCaseDuration; resultCreateModel.DurationInMs = testCaseDuration.TotalMilliseconds; resultCreateModel.StartedDate = testCaseStartTime; resultCreateModel.CompletedDate = testCaseStartTime.AddTicks(testCaseDuration.Ticks); testCaseStartTime = testCaseStartTime.AddTicks(1) + testCaseDuration; //next start time //test case outcome XmlNode failure, error, skipped; if ((failure = testCaseNode.SelectSingleNode("./failure")) != null) { ProcessFailureNode(failure, resultCreateModel); AddSystemLogsToResult(testCaseNode, resultCreateModel); } else if ((error = testCaseNode.SelectSingleNode("./error")) != null) { ProcessFailureNode(error, resultCreateModel); AddSystemLogsToResult(testCaseNode, resultCreateModel); } else if ((skipped = testCaseNode.SelectSingleNode("./skipped")) != null) { resultCreateModel.Outcome = TestOutcome.NotExecuted.ToString(); if (skipped.Attributes["message"] != null && !string.IsNullOrWhiteSpace(skipped.Attributes["message"].Value)) { resultCreateModel.ErrorMessage = skipped.Attributes["message"].Value; } } else { resultCreateModel.Outcome = TestOutcome.Passed.ToString(); } resultCreateModel.State = "Completed"; resultCreateModel.AutomatedTestType = Name; //other properties - host name and user resultCreateModel.ComputerName = testSuiteSummary.Host; if (runUserIdRef != null) { resultCreateModel.RunBy = runUserIdRef; } if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle)) { testSuiteSummary.Results.Add(resultCreateModel); } } } testSuiteSummary.TestSuiteDuration = totalTestSuiteDuration; testSuiteSummary.TotalTestCaseDuration = totalTestCaseDuration; return(testSuiteSummary); }
//Based on the XUnit V2 format: http://xunit.github.io/docs/format-xml-v2.html public TestRunData ReadResults(IExecutionContext executionContext, string filePath, TestRunContext runContext = null) { List <TestCaseResultData> results = new List <TestCaseResultData>(); XmlDocument doc = new XmlDocument(); try { var settings = new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore }; using (XmlReader reader = XmlReader.Create(filePath, settings)) { doc.Load(reader); } } catch (XmlException ex) { executionContext.Warning(StringUtil.Loc("FailedToReadFile", filePath, ex.Message)); return(null); } string runName = Name + " Test Run"; string runUser = ""; if (runContext != null) { if (!string.IsNullOrWhiteSpace(runContext.RunName)) { runName = runContext.RunName; } else { runName = string.Format(CultureInfo.CurrentCulture, "{0} {1} {2}", runName, runContext.Configuration, runContext.Platform); } if (runContext.Owner != null) { runUser = runContext.Owner; } } IdentityRef runUserIdRef = new IdentityRef(); runUserIdRef.DisplayName = runUser; var minStartTime = DateTime.MaxValue; var maxCompletedTime = DateTime.MinValue; bool dateTimeParseError = true; bool assemblyRunDateTimeAttributesNotPresent = false; bool assemblyTimeAttributeNotPresent = false; double assemblyRunDuration = 0; double testRunDuration = 0; XmlNodeList assemblyNodes = doc.SelectNodes("/assemblies/assembly"); foreach (XmlNode assemblyNode in assemblyNodes) { var assemblyRunStartTimeStamp = DateTime.MinValue; if (assemblyNode.Attributes["run-date"] != null && assemblyNode.Attributes["run-time"] != null) { string runDate = assemblyNode.Attributes["run-date"].Value; string runTime = assemblyNode.Attributes["run-time"].Value; var startDate = DateTime.Now; var startTime = TimeSpan.Zero; if (DateTime.TryParse(runDate, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out startDate) && TimeSpan.TryParse(runTime, CultureInfo.InvariantCulture, out startTime)) { dateTimeParseError = false; } assemblyRunStartTimeStamp = startDate + startTime; if (minStartTime > assemblyRunStartTimeStamp) { minStartTime = assemblyRunStartTimeStamp; } } else { assemblyRunDateTimeAttributesNotPresent = true; } if (!assemblyTimeAttributeNotPresent && assemblyNode.Attributes["time"] != null) { double assemblyDuration = 0; Double.TryParse(assemblyNode.Attributes["time"].Value, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out assemblyDuration); assemblyRunDuration = assemblyRunDuration + assemblyDuration; var durationFromSeconds = TimeSpan.FromSeconds(assemblyDuration); // no assemblystarttime available so dont calculate assemblycompletedtime if (assemblyRunStartTimeStamp != DateTime.MinValue) { DateTime assemblyRunCompleteTimeStamp = assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks); //finding maximum comleted time if (maxCompletedTime < assemblyRunCompleteTimeStamp) { maxCompletedTime = assemblyRunCompleteTimeStamp; } } } else { assemblyTimeAttributeNotPresent = true; } XmlNodeList testCaseNodeList = assemblyNode.SelectNodes("./collection/test"); foreach (XmlNode testCaseNode in testCaseNodeList) { TestCaseResultData resultCreateModel = new TestCaseResultData() { Priority = TestManagementConstants.UnspecifiedPriority, //Priority is int type so if no priority set then its 255. }; //Test storage. if (assemblyNode.Attributes["name"] != null) { resultCreateModel.AutomatedTestStorage = Path.GetFileName(assemblyNode.Attributes["name"].Value); } //Fully Qualified Name. if (testCaseNode.Attributes["name"] != null) { resultCreateModel.AutomatedTestName = testCaseNode.Attributes["name"].Value; } //Test Method Name. if (testCaseNode.Attributes["method"] != null) { resultCreateModel.TestCaseTitle = testCaseNode.Attributes["method"].Value; } //Test duration. if (testCaseNode.Attributes["time"] != null && testCaseNode.Attributes["time"].Value != null) { double duration = 0; double.TryParse(testCaseNode.Attributes["time"].Value, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out duration); var durationFromSeconds = TimeSpan.FromSeconds(duration); resultCreateModel.DurationInMs = durationFromSeconds.TotalMilliseconds; // no assemblystarttime available so dont set testcase start and completed if (assemblyRunStartTimeStamp != DateTime.MinValue) { resultCreateModel.StartedDate = assemblyRunStartTimeStamp; resultCreateModel.CompletedDate = assemblyRunStartTimeStamp.AddTicks(durationFromSeconds.Ticks); assemblyRunStartTimeStamp = assemblyRunStartTimeStamp.AddTicks(1) + durationFromSeconds; //next start time } //Calculate overall run duration. testRunDuration += duration; } //Test outcome. if (testCaseNode.SelectSingleNode("./failure") != null) { resultCreateModel.Outcome = TestOutcome.Failed.ToString(); //Error message. XmlNode failureMessageNode = testCaseNode.SelectSingleNode("./failure/message"); if (failureMessageNode != null && !string.IsNullOrWhiteSpace(failureMessageNode.InnerText)) { resultCreateModel.ErrorMessage = failureMessageNode.InnerText; } //Stack trace. XmlNode failureStackTraceNode = testCaseNode.SelectSingleNode("./failure/stack-trace"); if (failureStackTraceNode != null && !string.IsNullOrWhiteSpace(failureStackTraceNode.InnerText)) { resultCreateModel.StackTrace = failureStackTraceNode.InnerText; } // Console log resultCreateModel.AttachmentData = new AttachmentData(); XmlNode consoleLog = testCaseNode.SelectSingleNode("./output"); if (consoleLog != null && !string.IsNullOrWhiteSpace(consoleLog.InnerText)) { resultCreateModel.AttachmentData.ConsoleLog = consoleLog.InnerText; } } else if (testCaseNode.Attributes["result"] != null && string.Equals(testCaseNode.Attributes["result"].Value, "pass", StringComparison.OrdinalIgnoreCase)) { resultCreateModel.Outcome = TestOutcome.Passed.ToString(); } else { resultCreateModel.Outcome = TestOutcome.NotExecuted.ToString(); } //Test priority. XmlNode priorityTrait = testCaseNode.SelectSingleNode("./traits/trait[@name='priority']"); if (priorityTrait != null && priorityTrait.Attributes["value"] != null) { var priorityValue = priorityTrait.Attributes["value"].Value; resultCreateModel.Priority = !string.IsNullOrEmpty(priorityValue) ? Convert.ToInt32(priorityValue) : TestManagementConstants.UnspecifiedPriority; } //Test owner. XmlNode ownerNode = testCaseNode.SelectSingleNode("./traits/trait[@name='owner']"); if (ownerNode != null && ownerNode.Attributes["value"] != null && ownerNode.Attributes["value"].Value != null) { IdentityRef ownerIdRef = new IdentityRef(); ownerIdRef.DisplayName = ownerNode.Attributes["value"].Value; ownerIdRef.DirectoryAlias = ownerNode.Attributes["value"].Value; resultCreateModel.Owner = ownerIdRef; } resultCreateModel.RunBy = runUserIdRef; resultCreateModel.State = "Completed"; resultCreateModel.AutomatedTestType = Name; if (!string.IsNullOrEmpty(resultCreateModel.AutomatedTestName) && !string.IsNullOrEmpty(resultCreateModel.TestCaseTitle)) { results.Add(resultCreateModel); } } } if (dateTimeParseError || assemblyRunDateTimeAttributesNotPresent) { executionContext.Warning("Atleast for one assembly start time was not obtained due to tag not present or parsing issue, total run duration will now be summation of time taken by each assembly"); if (assemblyTimeAttributeNotPresent) { executionContext.Warning("Atleast for one assembly time tag is not present, total run duration will now be summation of time from all test runs"); } } //if minimum start time is not available then set it to present time minStartTime = minStartTime == DateTime.MaxValue ? DateTime.UtcNow : minStartTime; //if start time cannot be obtained even for one assembly then fallback duration to sum of assembly run time //if assembly run time cannot be obtained even for one assembly then fallback duration to total test run maxCompletedTime = dateTimeParseError || assemblyRunDateTimeAttributesNotPresent || maxCompletedTime == DateTime.MinValue ? minStartTime.Add(assemblyTimeAttributeNotPresent ? TimeSpan.FromSeconds(testRunDuration) : TimeSpan.FromSeconds(assemblyRunDuration)) : maxCompletedTime; executionContext.Output(string.Format("Obtained XUnit Test Run Start Date: {0} and Completed Date: {1}", minStartTime.ToString("o"), maxCompletedTime.ToString("o"))); TestRunData testRunData = new TestRunData( name: runName, buildId: runContext != null ? runContext.BuildId : 0, startedDate: minStartTime != DateTime.MinValue ? minStartTime.ToString("o") : null, completedDate: maxCompletedTime != DateTime.MinValue ? maxCompletedTime.ToString("o") : null, state: TestRunState.InProgress.ToString(), isAutomated: true, buildFlavor: runContext != null ? runContext.Configuration : null, buildPlatform: runContext != null ? runContext.Platform : null, releaseUri: runContext != null ? runContext.ReleaseUri : null, releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null ); testRunData.Results = results.ToArray(); testRunData.Attachments = AddResultsFileToRunLevelAttachments ? new string[] { filePath } : new string[0]; return(testRunData); }