public void TestResultHandlerShouldAddHierarchicalResultsIfParentTestResultIsPresent() { ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); Guid parentExecutionId = Guid.NewGuid(); ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1); result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId); ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase1); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase1); result3.Outcome = ObjectModel.TestOutcome.Failed; result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1); Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2); Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object); Assert.AreEqual(1, this.testableTrxLogger.TestResultCount, "TestResultHandler is not creating hierarchical results when parent result is present."); Assert.AreEqual(3, this.testableTrxLogger.TotalTestCount, "TestResultHandler is not adding all inner results in parent test result."); }
public void TestResultHandlerShouldAddSingleTestEntryForDataDrivenTests() { ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); Guid parentExecutionId = Guid.NewGuid(); ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1); result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId); ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase1); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase1); result3.Outcome = ObjectModel.TestOutcome.Failed; result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1); Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2); Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object); Assert.AreEqual(1, this.testableTrxLogger.TestEntryCount, "TestResultHandler is adding multiple test entries for data driven tests."); }
public void OutcomeOfRunWillBeCompletedIfNoTestsFails() { ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1"); ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2"); ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1"); ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1); passResult1.Outcome = ObjectModel.TestOutcome.Passed; ObjectModel.TestResult passResult2 = new ObjectModel.TestResult(passTestCase2); passResult2.Outcome = ObjectModel.TestOutcome.Passed; ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1); skipResult1.Outcome = ObjectModel.TestOutcome.Skipped; Mock <TestResultEventArgs> pass1 = new Mock <TestResultEventArgs>(passResult1); Mock <TestResultEventArgs> pass2 = new Mock <TestResultEventArgs>(passResult2); Mock <TestResultEventArgs> skip1 = new Mock <TestResultEventArgs>(skipResult1); this.testableTrxLogger.TestResultHandler(new object(), pass1.Object); this.testableTrxLogger.TestResultHandler(new object(), pass2.Object); this.testableTrxLogger.TestResultHandler(new object(), skip1.Object); var testRunCompleteEventArgs = CreateTestRunCompleteEventArgs(); this.testableTrxLogger.TestRunCompleteHandler(new object(), testRunCompleteEventArgs); Assert.AreEqual(TrxLoggerObjectModel.TestOutcome.Completed, this.testableTrxLogger.TestResultOutcome); }
public void TestResultHandlerKeepingTheTrackOfTotalTests() { ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1"); ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2"); ObjectModel.TestCase failTestCase1 = CreateTestCase("Fail1"); ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1"); ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1); passResult1.Outcome = ObjectModel.TestOutcome.Passed; ObjectModel.TestResult passResult2 = new ObjectModel.TestResult(passTestCase2); passResult2.Outcome = ObjectModel.TestOutcome.Passed; ObjectModel.TestResult failResult1 = new ObjectModel.TestResult(failTestCase1); failResult1.Outcome = ObjectModel.TestOutcome.Failed; ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1); skipResult1.Outcome = ObjectModel.TestOutcome.Skipped; Mock <TestResultEventArgs> pass1 = new Mock <TestResultEventArgs>(passResult1); Mock <TestResultEventArgs> pass2 = new Mock <TestResultEventArgs>(passResult2); Mock <TestResultEventArgs> fail1 = new Mock <TestResultEventArgs>(failResult1); Mock <TestResultEventArgs> skip1 = new Mock <TestResultEventArgs>(skipResult1); this.testableTrxLogger.TestResultHandler(new object(), pass1.Object); this.testableTrxLogger.TestResultHandler(new object(), pass2.Object); this.testableTrxLogger.TestResultHandler(new object(), fail1.Object); this.testableTrxLogger.TestResultHandler(new object(), skip1.Object); Assert.AreEqual(4, this.testableTrxLogger.TotalTestCount, "Passed Tests"); }
/// <summary> /// Asserts general test properties /// </summary> /// <param name="tests">The enumeration of discovered tests</param> /// <param name="qualifiedName">The qualified test name which is to be tested</param> /// <param name="source">The source from which the test should have been discovered</param> /// <param name="info">Optional source file information related to the test under question</param> private void AssertVSTestCaseProperties(IEnumerable <VSTestCase> tests, QualifiedNameBuilder qualifiedName, string source, SourceFileInfo info) { VSTestCase test = tests.FirstOrDefault((_test) => (_test.FullyQualifiedName == qualifiedName.ToString())); Assert.That(test, Is.Not.Null); Assert.That(test.DisplayName, Is.EqualTo(qualifiedName.Peek())); Assert.That(test.Source, Is.EqualTo(source)); Assert.That(test.ExecutorUri, Is.EqualTo(BoostTestExecutor.ExecutorUri)); if (info != null) { Assert.That(test.CodeFilePath, Is.EqualTo(info.File)); Assert.That(test.LineNumber, Is.EqualTo(info.LineNumber)); } Assert.That(test.Traits.Count(), Is.EqualTo(1)); Trait trait = test.Traits.First(); Assert.That(trait.Name, Is.EqualTo(VSTestModel.TestSuiteTrait)); string suite = qualifiedName.Pop().ToString(); if (string.IsNullOrEmpty(suite)) { suite = qualifiedName.MasterTestSuite; } Assert.That(trait.Value, Is.EqualTo(suite)); }
/// <summary> /// Converts platform test case to trx test element. /// </summary> /// <param name="testId"></param> /// <param name="executionId"></param> /// <param name="parentExecutionId"></param> /// <param name="name"></param> /// <param name="testType"></param> /// <param name="rockSteadyTestCase"></param> /// <returns>Trx test element</returns> public static ITestElement ToTestElement( Guid testId, Guid executionId, Guid parentExecutionId, String testName, TestType testType, ObjectModel.TestCase rockSteadyTestCase) { var testElement = CreateTestElement(testId, testName, rockSteadyTestCase.FullyQualifiedName, rockSteadyTestCase.ExecutorUri.ToString(), rockSteadyTestCase.Source, testType); testElement.Storage = rockSteadyTestCase.Source; testElement.Priority = GetPriority(rockSteadyTestCase); testElement.Owner = GetOwner(rockSteadyTestCase); testElement.ExecutionId = new TestExecId(executionId); testElement.ParentExecutionId = new TestExecId(parentExecutionId); var testCategories = GetCustomPropertyValueFromTestCase(rockSteadyTestCase, "MSTestDiscoverer.TestCategory"); foreach (string testCategory in testCategories) { testElement.TestCategories.Add(testCategory); } return(testElement); }
public void TestResultHandlerShouldAddMultipleTestElementsForOrderedTest() { ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); ObjectModel.TestCase testCase2 = CreateTestCase("TestCase2"); ObjectModel.TestCase testCase3 = CreateTestCase("TestCase3"); Guid parentExecutionId = Guid.NewGuid(); ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1); result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId); result1.SetPropertyValue <Guid>(TrxLoggerConstants.TestTypeProperty, TrxLoggerConstants.OrderedTestTypeGuid); ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase2); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase3); result3.Outcome = ObjectModel.TestOutcome.Failed; result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1); Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2); Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object); Assert.AreEqual(3, this.testableTrxLogger.UnitTestElementCount, "TestResultHandler is not adding multiple test elements for ordered test."); }
/// <summary> /// Creates a Visual Studio TestCase based on the provided information /// </summary> /// <param name="fullyQualifiedName">The fully qualified name of the test case</param> /// <param name="source">The test case source</param> /// <returns>A Visual Studio TestCase intended for BoostTestExecutor execution</returns> private VSTestCase CreateTestCase(string fullyQualifiedName, string source) { VSTestCase test = new VSTestCase(fullyQualifiedName, BoostTestExecutor.ExecutorUri, source); test.Traits.Add(VSTestModel.TestSuiteTrait, QualifiedNameBuilder.FromString(fullyQualifiedName).Pop().ToString()); return(test); }
public void RecordStart(VsTestCase testCase) { if (startTime == DateTime.MinValue) { startTime = DateTime.UtcNow; } Interlocked.Increment(ref total); }
/// <summary> /// Enumerates a sample collection of tests. /// </summary> /// <returns>An enumeration of sample test cases</returns> private IEnumerable <VSTestCase> GetDefaultTests() { VSTestCase test = CreateTestCase( DefaultTestCase, DefaultSource ); return(new VSTestCase[] { test }); }
private void HandleSkippedTest(Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult rsTestResult) { Debug.Assert(rsTestResult.Outcome == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Skipped, "Test Result should be skipped but it is " + rsTestResult.Outcome); Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase testCase = rsTestResult.TestCase; string testCaseName = !string.IsNullOrEmpty(testCase.DisplayName) ? testCase.DisplayName : testCase.FullyQualifiedName; string message = String.Format(CultureInfo.CurrentCulture, TrxLoggerResources.MessageForSkippedTests, testCaseName); this.AddRunLevelInformationalMessage(message); }
private void ValidateTestMethodProperties(string testName, string fullyQualifiedName, string expectedClassName) { TestPlatformObjectModel.TestCase testCase = CreateTestCase(fullyQualifiedName); TestPlatformObjectModel.TestResult result = new TestPlatformObjectModel.TestResult(testCase); var unitTestElement = Converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testName, TrxLoggerConstants.UnitTestType, testCase) as UnitTestElement; Assert.AreEqual(expectedClassName, unitTestElement.TestMethod.ClassName); Assert.AreEqual(testName, unitTestElement.TestMethod.Name); }
public void TestResultHandlerShouldCaptureStartTimeInSummaryWithTimeStampDuringIntialize() { ObjectModel.TestCase testCase = CreateTestCase("dummy string"); ObjectModel.TestResult testResult = new ObjectModel.TestResult(testCase); Mock <TestResultEventArgs> e = new Mock <TestResultEventArgs>(testResult); this.testableTrxLogger.TestResultHandler(new object(), e.Object); Assert.AreEqual(this.testableTrxLogger.TestRunStartTime, this.testableTrxLogger.LoggerTestRun.Started); }
public void Visit(TestCase testCase) { Code.Require(testCase, "testCase"); if (ShouldVisit(testCase)) { VSTestCase test = GenerateTestCase(testCase); TestCaseUtils.AddTestCase(test, this.DiscoverySink); } }
private void ValidateTestMethodProperties(string testName, string fullyQualifiedName, string expectedClassName) { TestPlatformObjectModel.TestCase testCase = CreateTestCase(fullyQualifiedName); TestPlatformObjectModel.TestResult result = new TestPlatformObjectModel.TestResult(testCase); var unitTestElement = this.converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testName, TrxLoggerConstants.UnitTestType, testCase) as UnitTestElement; var expectedTestName = fullyQualifiedName.StartsWith(expectedClassName) ? fullyQualifiedName.Remove(0, $"{expectedClassName}.".Length) : fullyQualifiedName; Assert.AreEqual(expectedClassName, unitTestElement.TestMethod.ClassName); Assert.AreEqual(expectedTestName, unitTestElement.TestMethod.Name); }
/// <summary> /// Return TMI Test id when available for TestPlatform TestCase. /// </summary> /// <param name="rockSteadyTestCase"> /// The rock Steady Test Case. /// </param> /// <returns> /// The <see cref="Guid"/>. /// </returns> private static Guid GetTmiTestId(ObjectModel.TestCase rockSteadyTestCase) { Guid tmiTestId = Guid.Empty; ObjectModel.TestProperty tmiTestIdProperty = rockSteadyTestCase.Properties.FirstOrDefault(property => property.Id.Equals(TmiTestIdPropertyIdentifier)); if (null != tmiTestIdProperty) { tmiTestId = rockSteadyTestCase.GetPropertyValue(tmiTestIdProperty, Guid.Empty); } return(tmiTestId); }
public void ToTestElementShouldNotFailWhenThereIsNoTestCategoreis() { ObjectModel.TestCase testCase = CreateTestCase("TestCase1"); ObjectModel.TestResult result = new ObjectModel.TestResult(testCase); var unitTestElement = Converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testCase.DisplayName, TrxLoggerConstants.UnitTestType, testCase); object[] expected = Enumerable.Empty <Object>().ToArray(); CollectionAssert.AreEqual(expected, unitTestElement.TestCategories.ToArray()); }
public void GetQToolsTestElementFromTestCaseShouldNotFailWhenThereIsNoTestCategoreis() { ObjectModel.TestCase testCase = CreateTestCase("TestCase1"); ObjectModel.TestResult result = new ObjectModel.TestResult(testCase); TrxLoggerObjectModel.UnitTestElement unitTestElement = Converter.GetQToolsTestElementFromTestCase(result); object[] expected = Enumerable.Empty <Object>().ToArray(); CollectionAssert.AreEqual(expected, unitTestElement.TestCategories.ToArray()); }
/// <summary> /// Generates a default TestResult for a 'test not found' exception. /// </summary> /// <param name="test">The test which failed due to a timeout.</param> /// <returns>A timed-out, failed TestResult related to the provided test.</returns> private static VSTestResult GenerateNotFoundResult(VSTestCase test) { VSTestResult result = new VSTestResult(test); result.ComputerName = Environment.MachineName; result.Outcome = TestOutcome.Skipped; result.ErrorMessage = GetNotFoundErrorMessage(test); return(result); }
/// <summary> /// Gets priority of test. /// </summary> /// <param name="rockSteadyTestCase"></param> /// <returns>Priority</returns> private static int GetPriority(ObjectModel.TestCase rockSteadyTestCase) { int priority = int.MaxValue; ObjectModel.Trait priorityTrait = rockSteadyTestCase.Traits?.FirstOrDefault(t => t.Name.Equals("Priority")); if (priorityTrait != null && Int32.TryParse(priorityTrait.Value, out int priorityValue)) { priority = priorityValue; } return(priority); }
/// <summary> /// Gets owner of test. /// </summary> /// <param name="rockSteadyTestCase"></param> /// <returns>Owner</returns> private static string GetOwner(ObjectModel.TestCase rockSteadyTestCase) { string owner = null; ObjectModel.Trait ownerTrait = rockSteadyTestCase.Traits?.FirstOrDefault(t => t.Name.Equals("Owner")); if (ownerTrait != null) { owner = ownerTrait.Value; } return(owner ?? string.Empty); }
/// <summary> /// Determine the Visual Studio test case for which this BoostTestResult is associated with. /// </summary> /// <param name="test">The Visual Studio test case to associate with the generated result</param> /// <returns>this</returns> public BoostTestResultBuilder For(VSTestCase test) { TestSuite parent = new TestSuite("Master Test Suite"); string[] fragments = test.FullyQualifiedName.Split('/'); for (int i = 0; i < (fragments.Length - 1); ++i) { parent = new TestSuite(fragments[i], parent); } return For(new TestCase(fragments[(fragments.Length - 1)], parent)); }
/// <summary> /// Determine the Visual Studio test case for which this BoostTestResult is associated with. /// </summary> /// <param name="test">The Visual Studio test case to associate with the generated result</param> /// <returns>this</returns> public BoostTestResultBuilder For(VSTestCase test) { TestSuite parent = new TestSuite("Master Test Suite"); string[] fragments = test.FullyQualifiedName.Split('/'); for (int i = 0; i < (fragments.Length - 1); ++i) { parent = new TestSuite(fragments[i], parent); } return(For(new TestCase(fragments[(fragments.Length - 1)], parent))); }
/// <summary> /// Visits the provided TestCase /// </summary> /// <param name="testCase">The TestCase which is to be visited</param> /// <param name="displayName">The test case display name to use (overrides the test case name)</param> private void Visit(TestCase testCase, string displayName) { Code.Require(testCase, "testCase"); VSTestCase test = GenerateTestCase(testCase); test.DisplayName = string.IsNullOrEmpty(displayName) ? test.DisplayName : displayName; // Send to discovery sink Logger.Info("Found test: {0}", test.FullyQualifiedName); this.DiscoverySink.SendTestCase(test); }
/// <summary> /// Generates a Visual Studio equivalent test case structure. /// </summary> /// <param name="testCase">The Boost.Test.TestCase to convert.</param> /// <returns>An equivalent Visual Studio TestCase structure to the one provided.</returns> private VSTestCase GenerateTestCase(TestCase testCase) { VSTestCase test = new VSTestCase( testCase.FullyQualifiedName, BoostTestExecutor.ExecutorUri, this.Source ); test.DisplayName = testCase.Name; if (testCase.Source != null) { // NOTE As of Boost 1.61, this warning might be triggered when BOOST_DATA_TEST_CASEs are used due to irregular DOT output if (!Path.IsPathRooted(testCase.Source.File) && this.OutputLog) { Logger.Info("Relative Paths are being used. Please note that test navigation from the Test Explorer window will not be available. To enable such functionality, the Use Full Paths setting under C++ -> Advanced in the project's Property Page must be set to Yes (/FC)."); this.OutputLog = false; } test.CodeFilePath = testCase.Source.File; test.LineNumber = testCase.Source.LineNumber; } // Register the test suite as a trait test.Traits.Add(new Trait(VSTestModel.TestSuiteTrait, GetParentFullyQualifiedName(testCase))); // Register enabled and disabled as traits test.Traits.Add(new Trait(VSTestModel.StatusTrait, (testCase.DefaultEnabled ? VSTestModel.TestEnabled : VSTestModel.TestDisabled))); TestUnit unit = testCase; while (unit != null) { foreach (string label in unit.Labels) { // Register each and every label as an individual trait test.Traits.Add(new Trait(label, string.Empty)); } // Test cases inherit the labels of parent test units // Reference: http://www.boost.org/doc/libs/1_60_0/libs/test/doc/html/boost_test/tests_organization/tests_grouping.html unit = unit.Parent; } // Record Boost version if available if (!string.IsNullOrEmpty(this.Version)) { test.SetPropertyValue(VSTestModel.VersionProperty, this.Version); } return(test); }
/// <summary> /// Provides a suitable message in case the provided test is not found. /// </summary> /// <param name="test">The test which was not found.</param> /// <returns>A suitable 'not-found' for the provided test case.</returns> private static string GetNotFoundErrorMessage(VSTestCase test) { if (test.FullyQualifiedName.Contains(' ')) { return(TestNotFound + " (Test name contains spaces)"); } else if (test.FullyQualifiedName.Contains(',')) { return(TestNotFound + " (Test name contains commas)"); } return(TestNotFound); }
/// <summary> /// Converts a Boost Test Result into an equivalent Visual Studio Test result /// </summary> /// <param name="test">The test case under consideration</param> /// <param name="result">The Boost test result for the test case under consideration</param> /// <param name="start">The test starting time</param> /// <param name="end">The test ending time</param> /// <returns>A Visual Studio test result equivalent to the Boost Test result</returns> private static VSTestResult GenerateResult(VSTestCase test, BoostTestResult result, DateTimeOffset start, DateTimeOffset end) { Code.Require(test, "test"); Code.Require(result, "result"); // Convert the Boost.Test.Result data structure into an equivalent Visual Studio model VSTestResult vsResult = result.AsVSTestResult(test); vsResult.StartTime = start; vsResult.EndTime = end; return(vsResult); }
public void ToTestElementShouldAssignTestCategoryOfUnitTestElement() { ObjectModel.TestCase testCase = CreateTestCase("TestCase1"); ObjectModel.TestResult result = new ObjectModel.TestResult(testCase); TestProperty testProperty = TestProperty.Register("MSTestDiscoverer.TestCategory", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject)); testCase.SetPropertyValue(testProperty, new[] { "AsmLevel", "ClassLevel", "MethodLevel" }); var unitTestElement = Converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testCase.DisplayName, TrxLoggerConstants.UnitTestType, testCase); object[] expected = new[] { "MethodLevel", "ClassLevel", "AsmLevel" }; CollectionAssert.AreEqual(expected, unitTestElement.TestCategories.ToArray().OrderByDescending(x => x.ToString()).ToArray()); }
public void ToTestElementShouldAssignWorkitemOfUnitTestElement() { TestPlatformObjectModel.TestCase testCase = CreateTestCase("TestCase1"); TestPlatformObjectModel.TestResult result = new TestPlatformObjectModel.TestResult(testCase); TestProperty testProperty = TestProperty.Register("WorkItemIds", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject)); testCase.SetPropertyValue(testProperty, new[] { "3", "99999", "0" }); var unitTestElement = this.converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testCase.DisplayName, TrxLoggerConstants.UnitTestType, testCase); int[] expected = new[] { 0, 3, 99999 }; CollectionAssert.AreEquivalent(expected, unitTestElement.WorkItems.ToArray()); }
public void TestResultHandlerLockingAMessageForSkipTest() { ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1"); ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1); skipResult1.Outcome = ObjectModel.TestOutcome.Skipped; Mock <TestResultEventArgs> skip1 = new Mock <TestResultEventArgs>(skipResult1); this.testableTrxLogger.TestResultHandler(new object(), skip1.Object); string expectedMessage = String.Format(CultureInfo.CurrentCulture, TrxLoggerResources.MessageForSkippedTests, "Skip1"); Assert.AreEqual(expectedMessage + Environment.NewLine, this.testableTrxLogger.GetRunLevelInformationalMessage()); }
public void GetCustomPropertyValueFromTestCaseShouldReadCategoyrAttributesFromTestCase() { ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); TestProperty testProperty = TestProperty.Register("MSTestDiscoverer.TestCategory", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject)); testCase1.SetPropertyValue(testProperty, new[] { "ClassLevel", "AsmLevel" }); List <String> listCategoriesActual = Converter.GetCustomPropertyValueFromTestCase(testCase1, "MSTestDiscoverer.TestCategory"); List <String> listCategoriesExpected = new List <string>(); listCategoriesExpected.Add("ClassLevel"); listCategoriesExpected.Add("AsmLevel"); CollectionAssert.AreEqual(listCategoriesExpected, listCategoriesActual); }
/// <summary> /// Converts a Boost.Test.Result.TestResult model into an equivalent /// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model. /// </summary> /// <param name="result">The Boost.Test.Result.TestResult model to convert.</param> /// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param> /// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns> public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test) { Utility.Code.Require(result, "result"); Utility.Code.Require(test, "test"); VSTestResult vsResult = new VSTestResult(test); vsResult.ComputerName = Environment.MachineName; vsResult.Outcome = GetTestOutcome(result.Result); // Boost.Test.Result.TestResult.Duration is in microseconds // 1 millisecond = 10,000 ticks // => 1 microsecond = 10 ticks // Reference: https://msdn.microsoft.com/en-us/library/zz841zbz(v=vs.110).aspx long ticks = (long) Math.Min((result.Duration * 10), long.MaxValue); // Clamp tick count to 1 in case Boost duration is listed as 0 vsResult.Duration = new TimeSpan(Math.Max(ticks, 1)); if (result.LogEntries.Count > 0) { foreach (TestResultMessage message in GetTestMessages(result)) { vsResult.Messages.Add(message); } // Test using the TestOutcome type since elements from the // Boost Result type may be collapsed into a particular value if (vsResult.Outcome == VSTestOutcome.Failed) { LogEntry error = GetLastError(result); if (error != null) { vsResult.ErrorMessage = GetErrorMessage(result); vsResult.ErrorStackTrace = ((error.Source == null) ? null : error.Source.ToString()); } } } return vsResult; }
/// <summary> /// Converts a Boost.Test.Result.TestResult model into an equivalent /// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model. /// </summary> /// <param name="result">The Boost.Test.Result.TestResult model to convert.</param> /// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param> /// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns> public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test) { Utility.Code.Require(result, "result"); Utility.Code.Require(test, "test"); VSTestResult vsResult = new VSTestResult(test); vsResult.ComputerName = Environment.MachineName; vsResult.Outcome = GetTestOutcome(result.Result); // Boost.Test.Result.TestResult.Duration is in microseconds vsResult.Duration = TimeSpan.FromMilliseconds(result.Duration / 1000); if (result.LogEntries.Count > 0) { foreach (TestResultMessage message in GetTestMessages(result)) { vsResult.Messages.Add(message); } // Test using the TestOutcome type since elements from the // Boost Result type may be collapsed into a particular value if (vsResult.Outcome == VSTestOutcome.Failed) { LogEntry error = GetLastError(result); if (error != null) { vsResult.ErrorMessage = GetErrorMessage(result); vsResult.ErrorStackTrace = ((error.Source == null) ? null : error.Source.ToString()); } } } return vsResult; }
private void RecordResult(string testFullyQualifiedName, string testDisplayName, DateTime startTime, DateTime endTime, string errorMessage, string errorStackTrace, int outCome) { TestCase testCase = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase(testFullyQualifiedName, uri, sourcePath) { DisplayName = testDisplayName }; this.frameworkHandle.RecordResult(new TestResult(testCase) { ComputerName = Environment.MachineName, Duration = endTime - startTime, EndTime = endTime, ErrorMessage = errorMessage, ErrorStackTrace = errorStackTrace, Outcome = (TestOutcome)outCome, StartTime = startTime }); }
/// <summary> /// Generates a Visual Studio equivalent test case structure. /// </summary> /// <param name="testCase">The Boost.Test.TestCase to convert.</param> /// <returns>An equivalent Visual Studio TestCase structure to the one provided.</returns> private VSTestCase GenerateTestCase(TestCase testCase) { VSTestCase test = new VSTestCase( GetFullyQualifiedName(testCase), BoostTestExecutor.ExecutorUri, this.Source ); test.DisplayName = testCase.Name; if (testCase.Source != null) { // NOTE As of Boost 1.61, this warning might be triggered when BOOST_DATA_TEST_CASEs are used due to irregular DOT output if (!Path.IsPathRooted(testCase.Source.File) && this.OutputLog) { Logger.Info("Relative Paths are being used. Please note that test navigation from the Test Explorer window will not be available. To enable such functionality, the Use Full Paths setting under C++ -> Advanced in the project's Property Page must be set to Yes (/FC)."); this.OutputLog = false; } test.CodeFilePath = testCase.Source.File; test.LineNumber = testCase.Source.LineNumber; } // Register the test suite as a trait test.Traits.Add(new Trait(VSTestModel.TestSuiteTrait, GetParentFullyQualifiedName(testCase))); // Register enabled and disabled as traits test.Traits.Add(new Trait(VSTestModel.StatusTrait, (testCase.DefaultEnabled ? VSTestModel.TestEnabled : VSTestModel.TestDisabled))); TestUnit unit = testCase; while (unit != null) { foreach (string label in unit.Labels) { // Register each and every label as an individual trait test.Traits.Add(new Trait(label, "")); } // Test cases inherit the labels of parent test units // Reference: http://www.boost.org/doc/libs/1_60_0/libs/test/doc/html/boost_test/tests_organization/tests_grouping.html unit = unit.Parent; } return test; }
private static VSTestResult GenerateResult(VSTestCase test, Boost.Results.TestResult result, DateTimeOffset start, DateTimeOffset end) { Code.Require(test, "test"); Code.Require(result, "result"); // Convert the Boost.Test.Result data structure into an equivalent Visual Studio model VSTestResult vsResult = result.AsVSTestResult(test); vsResult.StartTime = start; vsResult.EndTime = end; return vsResult; }
/// <summary> /// Generates a Visual Studio equivalent test case structure. /// </summary> /// <param name="testCase">The Boost.Test.TestCase to convert.</param> /// <returns>An equivalent Visual Studio TestCase structure to the one provided.</returns> private VSTestCase GenerateTestCase(TestCase testCase) { // Temporarily push TestCase on TestSuite name builder to acquire the fully qualified name of the TestCase this.TestSuite.Push(testCase); VSTestCase test = new VSTestCase( this.TestSuite.ToString(), BoostTestExecutor.ExecutorUri, this.Source ); // Reset TestSuite QualifiedNameBuilder to original value this.TestSuite.Pop(); if (testCase.Source != null) { test.CodeFilePath = testCase.Source.File; test.LineNumber = testCase.Source.LineNumber; } test.DisplayName = testCase.Name; // Register the test suite as a trait test.Traits.Add(new Trait(VSTestModel.TestSuiteTrait, GetCurrentTestSuite())); return test; }
/// <summary> /// Converts a Boost.Test.Result.TestResult model into an equivalent /// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model. /// </summary> /// <param name="result">The Boost.Test.Result.TestResult model to convert.</param> /// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param> /// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns> public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test) { Utility.Code.Require(result, "result"); Utility.Code.Require(test, "test"); VSTestResult vsResult = new VSTestResult(test); vsResult.ComputerName = Environment.MachineName; vsResult.Outcome = GetTestOutcome(result.Result); // Boost.Test.Result.TestResult.Duration is in microseconds // 1 millisecond = 10,000 ticks // => 1 microsecond = 10 ticks // Reference: https://msdn.microsoft.com/en-us/library/zz841zbz(v=vs.110).aspx long ticks = (long) Math.Min((result.Duration * 10), long.MaxValue); // Clamp tick count to 1 in case Boost duration is listed as 0 vsResult.Duration = new TimeSpan(Math.Max(ticks, 1)); if (result.LogEntries.Count > 0) { foreach (TestResultMessage message in GetTestMessages(result)) { vsResult.Messages.Add(message); } // Test using the TestOutcome type since elements from the // Boost Result type may be collapsed into a particular value if (vsResult.Outcome == VSTestOutcome.Failed) { LogEntry error = GetLastError(result); if (error != null) { vsResult.ErrorMessage = GetErrorMessage(result); if (error.Source != null) { //String format for a hyper linkable Stack Trace //Reference: NUnit3 Test Adapter. vsResult.ErrorStackTrace = string.Format(CultureInfo.InvariantCulture, "at {0}() in {1}:line {2}", vsResult.TestCase.DisplayName, ConvertSlashes(error.Source.File), error.Source.LineNumber); } } } } return vsResult; }
public void SetUp() { this.TestCase = new VSTestCase(DefaultTestName, ExecutorUri, DefaultSource); }
/// <summary> /// Asserts general Visual Studio TestResult properties for the provided TestCase. /// </summary> /// <param name="result">The Visual Studio TestResult to test</param> /// <param name="test">The Visual Studio TestCase for which the result is based on</param> private void AssertVSTestModelProperties(VSTestResult result, VSTestCase test) { Assert.That(result.TestCase, Is.EqualTo(test)); Assert.That(result.ComputerName, Is.EqualTo(Environment.MachineName)); }
/// <summary> /// Generates a default TestResult for a timeout exception. /// </summary> /// <param name="test">The test which failed due to a timeout.</param> /// <param name="ex">The exception related to this timeout.</param> /// <returns>A timed-out, failed TestResult related to the provided test.</returns> private static VSTestResult GenerateTimeoutResult(VSTestCase test, Boost.Runner.TimeoutException ex) { VSTestResult result = new VSTestResult(test); result.ComputerName = Environment.MachineName; result.Outcome = TestOutcome.Failed; result.Duration = TimeSpan.FromMilliseconds(ex.Timeout); result.ErrorMessage = "Timeout exceeded. Test ran for more than " + ex.Timeout + " ms."; if (!string.IsNullOrEmpty(test.CodeFilePath)) { result.ErrorStackTrace = new SourceFileInfo(test.CodeFilePath, test.LineNumber).ToString(); } return result; }
/// <summary> /// Generates a default TestResult for a 'test not found' exception. /// </summary> /// <param name="test">The test which failed due to a timeout.</param> /// <returns>A timed-out, failed TestResult related to the provided test.</returns> private static VSTestResult GenerateNotFoundResult(VSTestCase test) { VSTestResult result = new VSTestResult(test); result.ComputerName = Environment.MachineName; result.Outcome = TestOutcome.Skipped; result.ErrorMessage = GetNotFoundErrorMessage(test); return result; }
/// <summary> /// Provides a suitable message in case the provided test is not found. /// </summary> /// <param name="test">The test which was not found.</param> /// <returns>A suitable 'not-found' for the provided test case.</returns> private static string GetNotFoundErrorMessage(VSTestCase test) { if (test.FullyQualifiedName.Contains(' ')) { return TestNotFound + " (Test name contains spaces)"; } else if (test.FullyQualifiedName.Contains(',')) { return TestNotFound + " (Test name contains commas)"; } return TestNotFound; }