public TrxObjectModel.TestOutcome ToOutcome(ObjectModel.TestOutcome rockSteadyOutcome) { TrxObjectModel.TestOutcome outcome = TrxObjectModel.TestOutcome.Failed; switch (rockSteadyOutcome) { case ObjectModel.TestOutcome.Failed: outcome = TrxObjectModel.TestOutcome.Failed; break; case ObjectModel.TestOutcome.Passed: outcome = TrxObjectModel.TestOutcome.Passed; break; case ObjectModel.TestOutcome.Skipped: case ObjectModel.TestOutcome.None: case ObjectModel.TestOutcome.NotFound: outcome = TrxObjectModel.TestOutcome.NotExecuted; break; default: Debug.Fail("Unexpected Outcome."); break; } return(outcome); }
/// <summary> /// Creates test result /// </summary> /// <param name="executionId"></param> /// <param name="parentExecutionId"></param> /// <param name="testType"></param> /// <param name="testElement"></param> /// <param name="parentTestElement"></param> /// <param name="parentTestResult"></param> /// <param name="rocksteadyTestResult"></param> /// <returns>Trx test result</returns> private ITestResult CreateTestResult(Guid executionId, Guid parentExecutionId, TestType testType, ITestElement testElement, ITestElement parentTestElement, ITestResult parentTestResult, ObjectModel.TestResult rocksteadyTestResult) { // Create test result TrxLoggerObjectModel.TestOutcome testOutcome = this.converter.ToOutcome(rocksteadyTestResult.Outcome); var testResult = this.converter.ToTestResult(testElement.Id.Id, executionId, parentExecutionId, testElement.Name, this.testResultsDirPath, testType, testElement.CategoryId, testOutcome, this.testRun, rocksteadyTestResult); // Normal result scenario if (parentTestResult == null) { this.results.TryAdd(executionId, testResult); return(testResult); } // Ordered test inner result scenario if (parentTestElement != null && parentTestElement.TestType.Equals(TrxLoggerConstants.OrderedTestType)) { (parentTestResult as TestResultAggregation).InnerResults.Add(testResult); this.innerResults.TryAdd(executionId, testResult); return(testResult); } // Data driven inner result scenario if (parentTestElement != null && parentTestElement.TestType.Equals(TrxLoggerConstants.UnitTestType)) { (parentTestResult as TestResultAggregation).InnerResults.Add(testResult); testResult.DataRowInfo = (parentTestResult as TestResultAggregation).InnerResults.Count; testResult.ResultType = TrxLoggerConstants.InnerDataDrivenResultType; parentTestResult.ResultType = TrxLoggerConstants.ParentDataDrivenResultType; return(testResult); } return(testResult); }
/// <summary> /// Called when a test message is received. /// </summary> /// <param name="sender"> /// The sender. /// </param> /// <param name="e"> /// Event args /// </param> internal void TestMessageHandler(object sender, TestRunMessageEventArgs e) { ValidateArg.NotNull(sender, "sender"); ValidateArg.NotNull(e, "e"); RunInfo runMessage; switch (e.Level) { case TestMessageLevel.Informational: this.AddRunLevelInformationalMessage(e.Message); break; case TestMessageLevel.Warning: runMessage = new RunInfo(e.Message, null, Environment.MachineName, TrxLoggerObjectModel.TestOutcome.Warning); this.runLevelErrorsAndWarnings.Add(runMessage); break; case TestMessageLevel.Error: this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Failed; runMessage = new RunInfo(e.Message, null, Environment.MachineName, TrxLoggerObjectModel.TestOutcome.Error); this.runLevelErrorsAndWarnings.Add(runMessage); break; default: Debug.Fail("TrxLogger.TestMessageHandler: The test message level is unrecognized: {0}", e.Level.ToString()); break; } }
/// <summary> /// Returns the QToolsCommon.TestResult object created from rockSteady TestResult. /// </summary> /// <param name="rockSteadyTestResult"> rock steady test result</param> /// <param name="testElement"> testElement of that test</param> /// <param name="testOutcome"> Test outcome </param> /// <param name="testRun"> test run object </param> /// <returns> TestResult object </returns> private static TrxObjectModel.UnitTestResult GetQToolsTestResultFromTestResult( ObjectModel.TestResult rockSteadyTestResult, TrxObjectModel.UnitTestElement testElement, TrxObjectModel.TestOutcome testOutcome, TrxObjectModel.TestRun testRun) { UnitTestResult testResult = new UnitTestResult(Environment.MachineName, testRun.Id, testElement, testOutcome); if (rockSteadyTestResult.ErrorMessage != null) { testResult.ErrorMessage = rockSteadyTestResult.ErrorMessage; } if (rockSteadyTestResult.ErrorStackTrace != null) { testResult.ErrorStackTrace = rockSteadyTestResult.ErrorStackTrace; } // set start and end times if (rockSteadyTestResult.EndTime != null) { testResult.EndTime = rockSteadyTestResult.EndTime.UtcDateTime; } if (rockSteadyTestResult.StartTime != null) { testResult.StartTime = rockSteadyTestResult.StartTime.UtcDateTime; } if (rockSteadyTestResult.Duration != null) { testResult.Duration = rockSteadyTestResult.Duration; } return(testResult); }
/// <summary> /// Called when a test result is received. /// </summary> /// <param name="sender"> /// The sender. /// </param> /// <param name="e"> /// The eventArgs. /// </param> internal void TestResultHandler(object sender, TestResultEventArgs e) { // Create test run if (this.testRun == null) { CreateTestRun(); } // Convert skipped test to a log entry as that is the behavior of mstest. if (e.Result.Outcome == ObjectModel.TestOutcome.Skipped) { this.HandleSkippedTest(e.Result); } var testType = this.converter.GetTestType(e.Result); var executionId = this.converter.GetExecutionId(e.Result); // Setting parent properties like parent result, parent test element, parent execution id. var parentExecutionId = this.converter.GetParentExecutionId(e.Result); var parentTestResult = GetTestResult(parentExecutionId); var parentTestElement = (parentTestResult != null) ? GetTestElement(parentTestResult.Id.TestId) : null; // Switch to flat test results in case any parent related information is missing. if (parentTestResult == null || parentTestElement == null || parentExecutionId == Guid.Empty) { parentTestResult = null; parentTestElement = null; parentExecutionId = Guid.Empty; } // Create trx test element from rocksteady test case var testElement = GetOrCreateTestElement(executionId, parentExecutionId, testType, parentTestElement, e.Result); // Update test links. Test Links are updated in case of Ordered test. UpdateTestLinks(testElement, parentTestElement); // Convert the rocksteady result to trx test result var testResult = CreateTestResult(executionId, parentExecutionId, testType, testElement, parentTestElement, parentTestResult, e.Result); // Update test entries UpdateTestEntries(executionId, parentExecutionId, testElement, parentTestElement); // Set various counts (passed tests, failed tests, total tests) this.totalTests++; if (testResult.Outcome == TrxLoggerObjectModel.TestOutcome.Failed) { this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Failed; this.failTests++; } else if (testResult.Outcome == TrxLoggerObjectModel.TestOutcome.Passed) { this.passTests++; } }
private TrxLoggerObjectModel.TestOutcome changeTestOutcomeIfNecessary(TrxLoggerObjectModel.TestOutcome outcome) { // If no tests discovered/executed and TreatNoTestsAsError was set to True // We will return ResultSummary as Failed // Note : we only send the value of TreatNoTestsAsError if it is "True" if (totalTests == 0 && parametersDictionary.ContainsKey(ObjectModelConstants.TreatNoTestsAsError)) { outcome = TrxLoggerObjectModel.TestOutcome.Failed; } return(outcome); }
public void SaveCounters(XmlElement xml, string location, int[] counters) { xml = (XmlElement)LocationToXmlNode(xml, location); for (int i = 0; i < counters.Length; i++) { TrxObjectModel.TestOutcome outcome = (TrxObjectModel.TestOutcome)i; string attributeName = outcome.ToString(); attributeName = attributeName.Substring(0, 1).ToLowerInvariant() + attributeName.Substring(1); xml.SetAttribute(attributeName, counters[i].ToString(CultureInfo.InvariantCulture)); } }
/// <summary> /// Create test result. /// Currently trx supports only UnitTest and OrderedTest. All tests except OrderedTest all converted to unit test result. /// </summary> /// <param name="runId"></param> /// <param name="testId"></param> /// <param name="executionId"></param> /// <param name="parentExecutionId"></param> /// <param name="resultName"></param> /// <param name="outcome"></param> /// <param name="testType"></param> /// <param name="testCategoryId"></param> /// <returns>Trx test result</returns> private TrxObjectModel.TestResult CreateTestResult( Guid runId, Guid testId, Guid executionId, Guid parentExecutionId, string resultName, TrxObjectModel.TestOutcome outcome, TestType testType, TestListCategoryId testCategoryId) { return(testType.Equals(Constants.OrderedTestType) ? new TestResultAggregation(runId, testId, executionId, parentExecutionId, resultName, Environment.MachineName, outcome, testType, testCategoryId, trxFileHelper) : new UnitTestResult(runId, testId, executionId, parentExecutionId, resultName, Environment.MachineName, outcome, testType, testCategoryId, trxFileHelper)); }
/// <summary> /// Converts the rockSteady result to unit test result /// </summary> /// <param name="testId"></param> /// <param name="executionId"></param> /// <param name="parentExecutionId"></param> /// <param name="testName"></param> /// <param name="trxFileDirectory"></param> /// <param name="testType"></param> /// <param name="testCategoryId"></param> /// <param name="testOutcome"></param> /// <param name="testRun"></param> /// <param name="rockSteadyTestResult"></param> /// <returns>Trx test result object</returns> public ITestResult ToTestResult( Guid testId, Guid executionId, Guid parentExecutionId, string testName, string trxFileDirectory, TestType testType, TestListCategoryId testCategoryId, TrxObjectModel.TestOutcome testOutcome, TestRun testRun, ObjectModel.TestResult rockSteadyTestResult) { var resultName = !string.IsNullOrEmpty(rockSteadyTestResult.DisplayName) ? rockSteadyTestResult.DisplayName : testName; var testResult = CreateTestResult(testRun.Id, testId, executionId, parentExecutionId, resultName, testOutcome, testType, testCategoryId); if (rockSteadyTestResult.ErrorMessage != null) { testResult.ErrorMessage = rockSteadyTestResult.ErrorMessage; } if (rockSteadyTestResult.ErrorStackTrace != null) { testResult.ErrorStackTrace = rockSteadyTestResult.ErrorStackTrace; } if (rockSteadyTestResult.EndTime != null) { testResult.EndTime = rockSteadyTestResult.EndTime.UtcDateTime; } if (rockSteadyTestResult.StartTime != null) { testResult.StartTime = rockSteadyTestResult.StartTime.UtcDateTime; } if (rockSteadyTestResult.Duration != null) { testResult.Duration = rockSteadyTestResult.Duration; } // Clear existing messages and store rocksteady result messages. testResult.TextMessages = null; UpdateResultMessages(testResult, rockSteadyTestResult); // Save result attachments to target location. UpdateTestResultAttachments(rockSteadyTestResult, testResult, testRun, trxFileDirectory, true); return(testResult); }
/// <summary> /// Converts the rockSteady result to unit test result /// </summary> /// <param name="rockSteadyTestResult"> rock steady test result</param> /// <param name="testElement"> testElement of that test</param> /// <param name="testOutcome"> Test outcome </param> /// <param name="testRun"> test run object </param> /// <param name="trxFileDirectory"> TRX file directory</param> /// <returns> TestResult object </returns> internal static TrxObjectModel.UnitTestResult ToUnitTestResult( ObjectModel.TestResult rockSteadyTestResult, TrxObjectModel.UnitTestElement testElement, TrxObjectModel.TestOutcome testOutcome, TrxObjectModel.TestRun testRun, string trxFileDirectory) { TrxObjectModel.UnitTestResult qtoolsResult = GetQToolsTestResultFromTestResult(rockSteadyTestResult, testElement, testOutcome, testRun); // Clear exsting messages and store rocksteady result messages. qtoolsResult.TextMessages = null; UpdateResultMessages(qtoolsResult, rockSteadyTestResult); // Save result attachments to target location. UpdateTestResultAttachments(rockSteadyTestResult, qtoolsResult, testRun, trxFileDirectory, true); return(qtoolsResult); }
/// <summary> /// Called when a test run is completed. /// </summary> /// <param name="sender"> /// The sender. /// </param> /// <param name="e"> /// Test run complete events arguments. /// </param> internal void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) { // Create test run // If abort occurs there is no call to TestResultHandler which results in testRun not created. // This happens when some test aborts in the first batch of execution. if (this.testRun == null) { CreateTestRun(); } XmlPersistence helper = new XmlPersistence(); XmlTestStoreParameters parameters = XmlTestStoreParameters.GetParameters(); XmlElement rootElement = helper.CreateRootElement("TestRun"); // Save runId/username/creation time etc. this.testRun.Finished = DateTime.UtcNow; helper.SaveSingleFields(rootElement, this.testRun, parameters); // Save test settings helper.SaveObject(this.testRun.RunConfiguration, rootElement, "TestSettings", parameters); // Save test results helper.SaveIEnumerable(this.results.Values, rootElement, "Results", ".", null, parameters); // Save test definitions helper.SaveIEnumerable(this.testElements.Values, rootElement, "TestDefinitions", ".", null, parameters); // Save test entries helper.SaveIEnumerable(this.entries.Values, rootElement, "TestEntries", ".", "TestEntry", parameters); // Save default categories List <TestListCategory> categories = new List <TestListCategory>(); categories.Add(TestListCategory.UncategorizedResults); categories.Add(TestListCategory.AllResults); helper.SaveList(categories, rootElement, "TestLists", ".", "TestList", parameters); // Save summary if (this.testRunOutcome == TrxLoggerObjectModel.TestOutcome.Passed) { this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Completed; } testRunOutcome = changeTestOutcomeIfNecessary(testRunOutcome); List <string> errorMessages = new List <string>(); List <CollectorDataEntry> collectorEntries = this.converter.ToCollectionEntries(e.AttachmentSets, this.testRun, this.testResultsDirPath); IList <string> resultFiles = this.converter.ToResultFiles(e.AttachmentSets, this.testRun, this.testResultsDirPath, errorMessages); if (errorMessages.Count > 0) { // Got some errors while attaching files, report them and set the outcome of testrun to be Error... this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Error; foreach (string msg in errorMessages) { RunInfo runMessage = new RunInfo(msg, null, Environment.MachineName, TrxLoggerObjectModel.TestOutcome.Error); this.runLevelErrorsAndWarnings.Add(runMessage); } } TestRunSummary runSummary = new TestRunSummary( this.totalTests, this.passTests + this.failTests, this.passTests, this.failTests, this.testRunOutcome, this.runLevelErrorsAndWarnings, this.runLevelStdOut.ToString(), resultFiles, collectorEntries); helper.SaveObject(runSummary, rootElement, "ResultSummary", parameters); this.ReserveTrxFilePath(); this.PopulateTrxFile(this.trxFilePath, rootElement); }
/// <summary> /// Called when a test result is received. /// </summary> /// <param name="sender"> /// The sender. /// </param> /// <param name="e"> /// The eventArgs. /// </param> internal void TestResultHandler(object sender, ObjectModel.Logging.TestResultEventArgs e) { if (this.testRun == null) { Guid runId = Guid.NewGuid(); this.testRun = new TestRun(runId); // We cannot rely on the StartTime for the first test result // In case of parallel, first test result is the fastest test and not the one which started first. // Setting Started to DateTime.Now in Intialize will make sure we include the startup cost, which was being ignored earlier. // This is in parity with the way we set this.testRun.Finished this.testRun.Started = this.testRunStartTime; // Save default test settings string runDeploymentRoot = FileHelper.ReplaceInvalidFileNameChars(this.testRun.Name); TestRunConfiguration testrunConfig = new TestRunConfiguration("default"); testrunConfig.RunDeploymentRootDirectory = runDeploymentRoot; this.testRun.RunConfiguration = testrunConfig; } // Convert skipped test to a log entry as that is the behaviour of mstest. if (e.Result.Outcome == ObjectModel.TestOutcome.Skipped) { this.HandleSkippedTest(e.Result); } // Create MSTest test element from rocksteady test case UnitTestElement testElement = Converter.ToUnitTestElement(e.Result); // Conver the rocksteady result to MSTest result TrxLoggerObjectModel.TestOutcome testOutcome = Converter.ToOutcome(e.Result.Outcome); TrxLoggerObjectModel.UnitTestResult testResult = Converter.ToUnitTestResult(e.Result, testElement, testOutcome, this.testRun, this.testResultsDirPath); // Set various counts (passtests, failed tests, total tests) this.totalTests++; if (testResult.Outcome == TrxLoggerObjectModel.TestOutcome.Failed) { this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Failed; this.failTests++; } else if (testResult.Outcome == TrxLoggerObjectModel.TestOutcome.Passed) { this.passTests++; } // Add results to in-memory lists that are saved to the xml at completion. this.results.Add(testResult); if (!this.testElements.Contains(testElement)) { this.testElements.Add(testElement); } // create a test entry TestEntry te = new TestEntry(testElement.Id, TestListCategory.UncategorizedResults.Id); te.ExecId = testElement.ExecutionId; this.entries.Add(te); }