public void TestResultHandlerShouldAddHierarchicalResultsIfParentTestResultIsPresent()
{
ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1");
Guid parentExecutionId = Guid.NewGuid();
ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1);
result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId);
ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase1);
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase1);
result3.Outcome = ObjectModel.TestOutcome.Failed;
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1);
Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2);
Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object);
Assert.AreEqual(1, this.testableTrxLogger.TestResultCount, "TestResultHandler is not creating hierarchical results when parent result is present.");
Assert.AreEqual(3, this.testableTrxLogger.TotalTestCount, "TestResultHandler is not adding all inner results in parent test result.");
}
public void TestResultHandlerShouldAddSingleTestEntryForDataDrivenTests()
{
ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1");
Guid parentExecutionId = Guid.NewGuid();
ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1);
result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId);
ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase1);
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase1);
result3.Outcome = ObjectModel.TestOutcome.Failed;
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1);
Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2);
Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object);
Assert.AreEqual(1, this.testableTrxLogger.TestEntryCount, "TestResultHandler is adding multiple test entries for data driven tests.");
}
public void OutcomeOfRunWillBeCompletedIfNoTestsFails()
{
ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1");
ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2");
ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1");
ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1);
passResult1.Outcome = ObjectModel.TestOutcome.Passed;
ObjectModel.TestResult passResult2 = new ObjectModel.TestResult(passTestCase2);
passResult2.Outcome = ObjectModel.TestOutcome.Passed;
ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1);
skipResult1.Outcome = ObjectModel.TestOutcome.Skipped;
Mock <TestResultEventArgs> pass1 = new Mock <TestResultEventArgs>(passResult1);
Mock <TestResultEventArgs> pass2 = new Mock <TestResultEventArgs>(passResult2);
Mock <TestResultEventArgs> skip1 = new Mock <TestResultEventArgs>(skipResult1);
this.testableTrxLogger.TestResultHandler(new object(), pass1.Object);
this.testableTrxLogger.TestResultHandler(new object(), pass2.Object);
this.testableTrxLogger.TestResultHandler(new object(), skip1.Object);
var testRunCompleteEventArgs = CreateTestRunCompleteEventArgs();
this.testableTrxLogger.TestRunCompleteHandler(new object(), testRunCompleteEventArgs);
Assert.AreEqual(TrxLoggerObjectModel.TestOutcome.Completed, this.testableTrxLogger.TestResultOutcome);
}
public void TestResultHandlerKeepingTheTrackOfTotalTests()
{
ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1");
ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2");
ObjectModel.TestCase failTestCase1 = CreateTestCase("Fail1");
ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1");
ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1);
passResult1.Outcome = ObjectModel.TestOutcome.Passed;
ObjectModel.TestResult passResult2 = new ObjectModel.TestResult(passTestCase2);
passResult2.Outcome = ObjectModel.TestOutcome.Passed;
ObjectModel.TestResult failResult1 = new ObjectModel.TestResult(failTestCase1);
failResult1.Outcome = ObjectModel.TestOutcome.Failed;
ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1);
skipResult1.Outcome = ObjectModel.TestOutcome.Skipped;
Mock <TestResultEventArgs> pass1 = new Mock <TestResultEventArgs>(passResult1);
Mock <TestResultEventArgs> pass2 = new Mock <TestResultEventArgs>(passResult2);
Mock <TestResultEventArgs> fail1 = new Mock <TestResultEventArgs>(failResult1);
Mock <TestResultEventArgs> skip1 = new Mock <TestResultEventArgs>(skipResult1);
this.testableTrxLogger.TestResultHandler(new object(), pass1.Object);
this.testableTrxLogger.TestResultHandler(new object(), pass2.Object);
this.testableTrxLogger.TestResultHandler(new object(), fail1.Object);
this.testableTrxLogger.TestResultHandler(new object(), skip1.Object);
Assert.AreEqual(4, this.testableTrxLogger.TotalTestCount, "Passed Tests");
}
/// <summary>
/// Asserts general test properties
/// </summary>
/// <param name="tests">The enumeration of discovered tests</param>
/// <param name="qualifiedName">The qualified test name which is to be tested</param>
/// <param name="source">The source from which the test should have been discovered</param>
/// <param name="info">Optional source file information related to the test under question</param>
private void AssertVSTestCaseProperties(IEnumerable <VSTestCase> tests, QualifiedNameBuilder qualifiedName, string source, SourceFileInfo info)
{
VSTestCase test = tests.FirstOrDefault((_test) => (_test.FullyQualifiedName == qualifiedName.ToString()));
Assert.That(test, Is.Not.Null);
Assert.That(test.DisplayName, Is.EqualTo(qualifiedName.Peek()));
Assert.That(test.Source, Is.EqualTo(source));
Assert.That(test.ExecutorUri, Is.EqualTo(BoostTestExecutor.ExecutorUri));
if (info != null)
{
Assert.That(test.CodeFilePath, Is.EqualTo(info.File));
Assert.That(test.LineNumber, Is.EqualTo(info.LineNumber));
}
Assert.That(test.Traits.Count(), Is.EqualTo(1));
Trait trait = test.Traits.First();
Assert.That(trait.Name, Is.EqualTo(VSTestModel.TestSuiteTrait));
string suite = qualifiedName.Pop().ToString();
if (string.IsNullOrEmpty(suite))
{
suite = qualifiedName.MasterTestSuite;
}
Assert.That(trait.Value, Is.EqualTo(suite));
}
/// <summary>
/// Converts platform test case to trx test element.
/// </summary>
/// <param name="testId"></param>
/// <param name="executionId"></param>
/// <param name="parentExecutionId"></param>
/// <param name="name"></param>
/// <param name="testType"></param>
/// <param name="rockSteadyTestCase"></param>
/// <returns>Trx test element</returns>
public static ITestElement ToTestElement(
Guid testId,
Guid executionId,
Guid parentExecutionId,
String testName,
TestType testType,
ObjectModel.TestCase rockSteadyTestCase)
{
var testElement = CreateTestElement(testId, testName, rockSteadyTestCase.FullyQualifiedName, rockSteadyTestCase.ExecutorUri.ToString(), rockSteadyTestCase.Source, testType);
testElement.Storage = rockSteadyTestCase.Source;
testElement.Priority = GetPriority(rockSteadyTestCase);
testElement.Owner = GetOwner(rockSteadyTestCase);
testElement.ExecutionId = new TestExecId(executionId);
testElement.ParentExecutionId = new TestExecId(parentExecutionId);
var testCategories = GetCustomPropertyValueFromTestCase(rockSteadyTestCase, "MSTestDiscoverer.TestCategory");
foreach (string testCategory in testCategories)
{
testElement.TestCategories.Add(testCategory);
}
return(testElement);
}
public void TestResultHandlerShouldAddMultipleTestElementsForOrderedTest()
{
ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1");
ObjectModel.TestCase testCase2 = CreateTestCase("TestCase2");
ObjectModel.TestCase testCase3 = CreateTestCase("TestCase3");
Guid parentExecutionId = Guid.NewGuid();
ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1);
result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId);
result1.SetPropertyValue <Guid>(TrxLoggerConstants.TestTypeProperty, TrxLoggerConstants.OrderedTestTypeGuid);
ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase2);
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase3);
result3.Outcome = ObjectModel.TestOutcome.Failed;
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1);
Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2);
Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object);
Assert.AreEqual(3, this.testableTrxLogger.UnitTestElementCount, "TestResultHandler is not adding multiple test elements for ordered test.");
}
/// <summary>
/// Creates a Visual Studio TestCase based on the provided information
/// </summary>
/// <param name="fullyQualifiedName">The fully qualified name of the test case</param>
/// <param name="source">The test case source</param>
/// <returns>A Visual Studio TestCase intended for BoostTestExecutor execution</returns>
private VSTestCase CreateTestCase(string fullyQualifiedName, string source)
{
VSTestCase test = new VSTestCase(fullyQualifiedName, BoostTestExecutor.ExecutorUri, source);
test.Traits.Add(VSTestModel.TestSuiteTrait, QualifiedNameBuilder.FromString(fullyQualifiedName).Pop().ToString());
return(test);
}
public void RecordStart(VsTestCase testCase)
{
if (startTime == DateTime.MinValue)
{
startTime = DateTime.UtcNow;
}
Interlocked.Increment(ref total);
}
/// <summary>
/// Enumerates a sample collection of tests.
/// </summary>
/// <returns>An enumeration of sample test cases</returns>
private IEnumerable <VSTestCase> GetDefaultTests()
{
VSTestCase test = CreateTestCase(
DefaultTestCase,
DefaultSource
);
return(new VSTestCase[] { test });
}
private void HandleSkippedTest(Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult rsTestResult)
{
Debug.Assert(rsTestResult.Outcome == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Skipped, "Test Result should be skipped but it is " + rsTestResult.Outcome);
Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase testCase = rsTestResult.TestCase;
string testCaseName = !string.IsNullOrEmpty(testCase.DisplayName) ? testCase.DisplayName : testCase.FullyQualifiedName;
string message = String.Format(CultureInfo.CurrentCulture, TrxLoggerResources.MessageForSkippedTests, testCaseName);
this.AddRunLevelInformationalMessage(message);
}
private void ValidateTestMethodProperties(string testName, string fullyQualifiedName, string expectedClassName)
{
TestPlatformObjectModel.TestCase testCase = CreateTestCase(fullyQualifiedName);
TestPlatformObjectModel.TestResult result = new TestPlatformObjectModel.TestResult(testCase);
var unitTestElement = Converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testName, TrxLoggerConstants.UnitTestType, testCase) as UnitTestElement;
Assert.AreEqual(expectedClassName, unitTestElement.TestMethod.ClassName);
Assert.AreEqual(testName, unitTestElement.TestMethod.Name);
}
public void TestResultHandlerShouldCaptureStartTimeInSummaryWithTimeStampDuringIntialize()
{
ObjectModel.TestCase testCase = CreateTestCase("dummy string");
ObjectModel.TestResult testResult = new ObjectModel.TestResult(testCase);
Mock <TestResultEventArgs> e = new Mock <TestResultEventArgs>(testResult);
this.testableTrxLogger.TestResultHandler(new object(), e.Object);
Assert.AreEqual(this.testableTrxLogger.TestRunStartTime, this.testableTrxLogger.LoggerTestRun.Started);
}
public void Visit(TestCase testCase)
{
Code.Require(testCase, "testCase");
if (ShouldVisit(testCase))
{
VSTestCase test = GenerateTestCase(testCase);
TestCaseUtils.AddTestCase(test, this.DiscoverySink);
}
}
private void ValidateTestMethodProperties(string testName, string fullyQualifiedName, string expectedClassName)
{
TestPlatformObjectModel.TestCase testCase = CreateTestCase(fullyQualifiedName);
TestPlatformObjectModel.TestResult result = new TestPlatformObjectModel.TestResult(testCase);
var unitTestElement = this.converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testName, TrxLoggerConstants.UnitTestType, testCase) as UnitTestElement;
var expectedTestName = fullyQualifiedName.StartsWith(expectedClassName) ? fullyQualifiedName.Remove(0, $"{expectedClassName}.".Length) : fullyQualifiedName;
Assert.AreEqual(expectedClassName, unitTestElement.TestMethod.ClassName);
Assert.AreEqual(expectedTestName, unitTestElement.TestMethod.Name);
}
/// <summary>
/// Return TMI Test id when available for TestPlatform TestCase.
/// </summary>
/// <param name="rockSteadyTestCase">
/// The rock Steady Test Case.
/// </param>
/// <returns>
/// The <see cref="Guid"/>.
/// </returns>
private static Guid GetTmiTestId(ObjectModel.TestCase rockSteadyTestCase)
{
Guid tmiTestId = Guid.Empty;
ObjectModel.TestProperty tmiTestIdProperty = rockSteadyTestCase.Properties.FirstOrDefault(property => property.Id.Equals(TmiTestIdPropertyIdentifier));
if (null != tmiTestIdProperty)
{
tmiTestId = rockSteadyTestCase.GetPropertyValue(tmiTestIdProperty, Guid.Empty);
}
return(tmiTestId);
}
public void ToTestElementShouldNotFailWhenThereIsNoTestCategoreis()
{
ObjectModel.TestCase testCase = CreateTestCase("TestCase1");
ObjectModel.TestResult result = new ObjectModel.TestResult(testCase);
var unitTestElement = Converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testCase.DisplayName, TrxLoggerConstants.UnitTestType, testCase);
object[] expected = Enumerable.Empty <Object>().ToArray();
CollectionAssert.AreEqual(expected, unitTestElement.TestCategories.ToArray());
}
public void GetQToolsTestElementFromTestCaseShouldNotFailWhenThereIsNoTestCategoreis()
{
ObjectModel.TestCase testCase = CreateTestCase("TestCase1");
ObjectModel.TestResult result = new ObjectModel.TestResult(testCase);
TrxLoggerObjectModel.UnitTestElement unitTestElement = Converter.GetQToolsTestElementFromTestCase(result);
object[] expected = Enumerable.Empty <Object>().ToArray();
CollectionAssert.AreEqual(expected, unitTestElement.TestCategories.ToArray());
}
/// <summary>
/// Generates a default TestResult for a 'test not found' exception.
/// </summary>
/// <param name="test">The test which failed due to a timeout.</param>
/// <returns>A timed-out, failed TestResult related to the provided test.</returns>
private static VSTestResult GenerateNotFoundResult(VSTestCase test)
{
VSTestResult result = new VSTestResult(test);
result.ComputerName = Environment.MachineName;
result.Outcome = TestOutcome.Skipped;
result.ErrorMessage = GetNotFoundErrorMessage(test);
return(result);
}
/// <summary>
/// Gets priority of test.
/// </summary>
/// <param name="rockSteadyTestCase"></param>
/// <returns>Priority</returns>
private static int GetPriority(ObjectModel.TestCase rockSteadyTestCase)
{
int priority = int.MaxValue;
ObjectModel.Trait priorityTrait = rockSteadyTestCase.Traits?.FirstOrDefault(t => t.Name.Equals("Priority"));
if (priorityTrait != null && Int32.TryParse(priorityTrait.Value, out int priorityValue))
{
priority = priorityValue;
}
return(priority);
}
/// <summary>
/// Gets owner of test.
/// </summary>
/// <param name="rockSteadyTestCase"></param>
/// <returns>Owner</returns>
private static string GetOwner(ObjectModel.TestCase rockSteadyTestCase)
{
string owner = null;
ObjectModel.Trait ownerTrait = rockSteadyTestCase.Traits?.FirstOrDefault(t => t.Name.Equals("Owner"));
if (ownerTrait != null)
{
owner = ownerTrait.Value;
}
return(owner ?? string.Empty);
}
/// <summary>
/// Determine the Visual Studio test case for which this BoostTestResult is associated with.
/// </summary>
/// <param name="test">The Visual Studio test case to associate with the generated result</param>
/// <returns>this</returns>
public BoostTestResultBuilder For(VSTestCase test)
{
TestSuite parent = new TestSuite("Master Test Suite");
string[] fragments = test.FullyQualifiedName.Split('/');
for (int i = 0; i < (fragments.Length - 1); ++i)
{
parent = new TestSuite(fragments[i], parent);
}
return For(new TestCase(fragments[(fragments.Length - 1)], parent));
}
/// <summary>
/// Determine the Visual Studio test case for which this BoostTestResult is associated with.
/// </summary>
/// <param name="test">The Visual Studio test case to associate with the generated result</param>
/// <returns>this</returns>
public BoostTestResultBuilder For(VSTestCase test)
{
TestSuite parent = new TestSuite("Master Test Suite");
string[] fragments = test.FullyQualifiedName.Split('/');
for (int i = 0; i < (fragments.Length - 1); ++i)
{
parent = new TestSuite(fragments[i], parent);
}
return(For(new TestCase(fragments[(fragments.Length - 1)], parent)));
}
/// <summary>
/// Visits the provided TestCase
/// </summary>
/// <param name="testCase">The TestCase which is to be visited</param>
/// <param name="displayName">The test case display name to use (overrides the test case name)</param>
private void Visit(TestCase testCase, string displayName)
{
Code.Require(testCase, "testCase");
VSTestCase test = GenerateTestCase(testCase);
test.DisplayName = string.IsNullOrEmpty(displayName) ? test.DisplayName : displayName;
// Send to discovery sink
Logger.Info("Found test: {0}", test.FullyQualifiedName);
this.DiscoverySink.SendTestCase(test);
}
/// <summary>
/// Generates a Visual Studio equivalent test case structure.
/// </summary>
/// <param name="testCase">The Boost.Test.TestCase to convert.</param>
/// <returns>An equivalent Visual Studio TestCase structure to the one provided.</returns>
private VSTestCase GenerateTestCase(TestCase testCase)
{
VSTestCase test = new VSTestCase(
testCase.FullyQualifiedName,
BoostTestExecutor.ExecutorUri,
this.Source
);
test.DisplayName = testCase.Name;
if (testCase.Source != null)
{
// NOTE As of Boost 1.61, this warning might be triggered when BOOST_DATA_TEST_CASEs are used due to irregular DOT output
if (!Path.IsPathRooted(testCase.Source.File) && this.OutputLog)
{
Logger.Info("Relative Paths are being used. Please note that test navigation from the Test Explorer window will not be available. To enable such functionality, the Use Full Paths setting under C++ -> Advanced in the project's Property Page must be set to Yes (/FC).");
this.OutputLog = false;
}
test.CodeFilePath = testCase.Source.File;
test.LineNumber = testCase.Source.LineNumber;
}
// Register the test suite as a trait
test.Traits.Add(new Trait(VSTestModel.TestSuiteTrait, GetParentFullyQualifiedName(testCase)));
// Register enabled and disabled as traits
test.Traits.Add(new Trait(VSTestModel.StatusTrait, (testCase.DefaultEnabled ? VSTestModel.TestEnabled : VSTestModel.TestDisabled)));
TestUnit unit = testCase;
while (unit != null)
{
foreach (string label in unit.Labels)
{
// Register each and every label as an individual trait
test.Traits.Add(new Trait(label, string.Empty));
}
// Test cases inherit the labels of parent test units
// Reference: http://www.boost.org/doc/libs/1_60_0/libs/test/doc/html/boost_test/tests_organization/tests_grouping.html
unit = unit.Parent;
}
// Record Boost version if available
if (!string.IsNullOrEmpty(this.Version))
{
test.SetPropertyValue(VSTestModel.VersionProperty, this.Version);
}
return(test);
}
/// <summary>
/// Provides a suitable message in case the provided test is not found.
/// </summary>
/// <param name="test">The test which was not found.</param>
/// <returns>A suitable 'not-found' for the provided test case.</returns>
private static string GetNotFoundErrorMessage(VSTestCase test)
{
if (test.FullyQualifiedName.Contains(' '))
{
return(TestNotFound + " (Test name contains spaces)");
}
else if (test.FullyQualifiedName.Contains(','))
{
return(TestNotFound + " (Test name contains commas)");
}
return(TestNotFound);
}
/// <summary>
/// Converts a Boost Test Result into an equivalent Visual Studio Test result
/// </summary>
/// <param name="test">The test case under consideration</param>
/// <param name="result">The Boost test result for the test case under consideration</param>
/// <param name="start">The test starting time</param>
/// <param name="end">The test ending time</param>
/// <returns>A Visual Studio test result equivalent to the Boost Test result</returns>
private static VSTestResult GenerateResult(VSTestCase test, BoostTestResult result, DateTimeOffset start, DateTimeOffset end)
{
Code.Require(test, "test");
Code.Require(result, "result");
// Convert the Boost.Test.Result data structure into an equivalent Visual Studio model
VSTestResult vsResult = result.AsVSTestResult(test);
vsResult.StartTime = start;
vsResult.EndTime = end;
return(vsResult);
}
public void ToTestElementShouldAssignTestCategoryOfUnitTestElement()
{
ObjectModel.TestCase testCase = CreateTestCase("TestCase1");
ObjectModel.TestResult result = new ObjectModel.TestResult(testCase);
TestProperty testProperty = TestProperty.Register("MSTestDiscoverer.TestCategory", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject));
testCase.SetPropertyValue(testProperty, new[] { "AsmLevel", "ClassLevel", "MethodLevel" });
var unitTestElement = Converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testCase.DisplayName, TrxLoggerConstants.UnitTestType, testCase);
object[] expected = new[] { "MethodLevel", "ClassLevel", "AsmLevel" };
CollectionAssert.AreEqual(expected, unitTestElement.TestCategories.ToArray().OrderByDescending(x => x.ToString()).ToArray());
}
public void ToTestElementShouldAssignWorkitemOfUnitTestElement()
{
TestPlatformObjectModel.TestCase testCase = CreateTestCase("TestCase1");
TestPlatformObjectModel.TestResult result = new TestPlatformObjectModel.TestResult(testCase);
TestProperty testProperty = TestProperty.Register("WorkItemIds", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject));
testCase.SetPropertyValue(testProperty, new[] { "3", "99999", "0" });
var unitTestElement = this.converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testCase.DisplayName, TrxLoggerConstants.UnitTestType, testCase);
int[] expected = new[] { 0, 3, 99999 };
CollectionAssert.AreEquivalent(expected, unitTestElement.WorkItems.ToArray());
}
public void TestResultHandlerLockingAMessageForSkipTest()
{
ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1");
ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1);
skipResult1.Outcome = ObjectModel.TestOutcome.Skipped;
Mock <TestResultEventArgs> skip1 = new Mock <TestResultEventArgs>(skipResult1);
this.testableTrxLogger.TestResultHandler(new object(), skip1.Object);
string expectedMessage = String.Format(CultureInfo.CurrentCulture, TrxLoggerResources.MessageForSkippedTests, "Skip1");
Assert.AreEqual(expectedMessage + Environment.NewLine, this.testableTrxLogger.GetRunLevelInformationalMessage());
}
public void GetCustomPropertyValueFromTestCaseShouldReadCategoyrAttributesFromTestCase()
{
ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1");
TestProperty testProperty = TestProperty.Register("MSTestDiscoverer.TestCategory", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject));
testCase1.SetPropertyValue(testProperty, new[] { "ClassLevel", "AsmLevel" });
List <String> listCategoriesActual = Converter.GetCustomPropertyValueFromTestCase(testCase1, "MSTestDiscoverer.TestCategory");
List <String> listCategoriesExpected = new List <string>();
listCategoriesExpected.Add("ClassLevel");
listCategoriesExpected.Add("AsmLevel");
CollectionAssert.AreEqual(listCategoriesExpected, listCategoriesActual);
}
/// <summary>
/// Converts a Boost.Test.Result.TestResult model into an equivalent
/// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model.
/// </summary>
/// <param name="result">The Boost.Test.Result.TestResult model to convert.</param>
/// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param>
/// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns>
public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test)
{
Utility.Code.Require(result, "result");
Utility.Code.Require(test, "test");
VSTestResult vsResult = new VSTestResult(test);
vsResult.ComputerName = Environment.MachineName;
vsResult.Outcome = GetTestOutcome(result.Result);
// Boost.Test.Result.TestResult.Duration is in microseconds
// 1 millisecond = 10,000 ticks
// => 1 microsecond = 10 ticks
// Reference: https://msdn.microsoft.com/en-us/library/zz841zbz(v=vs.110).aspx
long ticks = (long) Math.Min((result.Duration * 10), long.MaxValue);
// Clamp tick count to 1 in case Boost duration is listed as 0
vsResult.Duration = new TimeSpan(Math.Max(ticks, 1));
if (result.LogEntries.Count > 0)
{
foreach (TestResultMessage message in GetTestMessages(result))
{
vsResult.Messages.Add(message);
}
// Test using the TestOutcome type since elements from the
// Boost Result type may be collapsed into a particular value
if (vsResult.Outcome == VSTestOutcome.Failed)
{
LogEntry error = GetLastError(result);
if (error != null)
{
vsResult.ErrorMessage = GetErrorMessage(result);
vsResult.ErrorStackTrace = ((error.Source == null) ? null : error.Source.ToString());
}
}
}
return vsResult;
}
/// <summary>
/// Converts a Boost.Test.Result.TestResult model into an equivalent
/// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model.
/// </summary>
/// <param name="result">The Boost.Test.Result.TestResult model to convert.</param>
/// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param>
/// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns>
public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test)
{
Utility.Code.Require(result, "result");
Utility.Code.Require(test, "test");
VSTestResult vsResult = new VSTestResult(test);
vsResult.ComputerName = Environment.MachineName;
vsResult.Outcome = GetTestOutcome(result.Result);
// Boost.Test.Result.TestResult.Duration is in microseconds
vsResult.Duration = TimeSpan.FromMilliseconds(result.Duration / 1000);
if (result.LogEntries.Count > 0)
{
foreach (TestResultMessage message in GetTestMessages(result))
{
vsResult.Messages.Add(message);
}
// Test using the TestOutcome type since elements from the
// Boost Result type may be collapsed into a particular value
if (vsResult.Outcome == VSTestOutcome.Failed)
{
LogEntry error = GetLastError(result);
if (error != null)
{
vsResult.ErrorMessage = GetErrorMessage(result);
vsResult.ErrorStackTrace = ((error.Source == null) ? null : error.Source.ToString());
}
}
}
return vsResult;
}
private void RecordResult(string testFullyQualifiedName, string testDisplayName, DateTime startTime, DateTime endTime, string errorMessage, string errorStackTrace, int outCome)
{
TestCase testCase = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase(testFullyQualifiedName, uri, sourcePath) { DisplayName = testDisplayName };
this.frameworkHandle.RecordResult(new TestResult(testCase)
{
ComputerName = Environment.MachineName,
Duration = endTime - startTime,
EndTime = endTime,
ErrorMessage = errorMessage,
ErrorStackTrace = errorStackTrace,
Outcome = (TestOutcome)outCome,
StartTime = startTime
});
}
/// <summary>
/// Generates a Visual Studio equivalent test case structure.
/// </summary>
/// <param name="testCase">The Boost.Test.TestCase to convert.</param>
/// <returns>An equivalent Visual Studio TestCase structure to the one provided.</returns>
private VSTestCase GenerateTestCase(TestCase testCase)
{
VSTestCase test = new VSTestCase(
GetFullyQualifiedName(testCase),
BoostTestExecutor.ExecutorUri,
this.Source
);
test.DisplayName = testCase.Name;
if (testCase.Source != null)
{
// NOTE As of Boost 1.61, this warning might be triggered when BOOST_DATA_TEST_CASEs are used due to irregular DOT output
if (!Path.IsPathRooted(testCase.Source.File) && this.OutputLog)
{
Logger.Info("Relative Paths are being used. Please note that test navigation from the Test Explorer window will not be available. To enable such functionality, the Use Full Paths setting under C++ -> Advanced in the project's Property Page must be set to Yes (/FC).");
this.OutputLog = false;
}
test.CodeFilePath = testCase.Source.File;
test.LineNumber = testCase.Source.LineNumber;
}
// Register the test suite as a trait
test.Traits.Add(new Trait(VSTestModel.TestSuiteTrait, GetParentFullyQualifiedName(testCase)));
// Register enabled and disabled as traits
test.Traits.Add(new Trait(VSTestModel.StatusTrait, (testCase.DefaultEnabled ? VSTestModel.TestEnabled : VSTestModel.TestDisabled)));
TestUnit unit = testCase;
while (unit != null)
{
foreach (string label in unit.Labels)
{
// Register each and every label as an individual trait
test.Traits.Add(new Trait(label, ""));
}
// Test cases inherit the labels of parent test units
// Reference: http://www.boost.org/doc/libs/1_60_0/libs/test/doc/html/boost_test/tests_organization/tests_grouping.html
unit = unit.Parent;
}
return test;
}
private static VSTestResult GenerateResult(VSTestCase test, Boost.Results.TestResult result, DateTimeOffset start, DateTimeOffset end)
{
Code.Require(test, "test");
Code.Require(result, "result");
// Convert the Boost.Test.Result data structure into an equivalent Visual Studio model
VSTestResult vsResult = result.AsVSTestResult(test);
vsResult.StartTime = start;
vsResult.EndTime = end;
return vsResult;
}
/// <summary>
/// Generates a Visual Studio equivalent test case structure.
/// </summary>
/// <param name="testCase">The Boost.Test.TestCase to convert.</param>
/// <returns>An equivalent Visual Studio TestCase structure to the one provided.</returns>
private VSTestCase GenerateTestCase(TestCase testCase)
{
// Temporarily push TestCase on TestSuite name builder to acquire the fully qualified name of the TestCase
this.TestSuite.Push(testCase);
VSTestCase test = new VSTestCase(
this.TestSuite.ToString(),
BoostTestExecutor.ExecutorUri,
this.Source
);
// Reset TestSuite QualifiedNameBuilder to original value
this.TestSuite.Pop();
if (testCase.Source != null)
{
test.CodeFilePath = testCase.Source.File;
test.LineNumber = testCase.Source.LineNumber;
}
test.DisplayName = testCase.Name;
// Register the test suite as a trait
test.Traits.Add(new Trait(VSTestModel.TestSuiteTrait, GetCurrentTestSuite()));
return test;
}
/// <summary>
/// Converts a Boost.Test.Result.TestResult model into an equivalent
/// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model.
/// </summary>
/// <param name="result">The Boost.Test.Result.TestResult model to convert.</param>
/// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param>
/// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns>
public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test)
{
Utility.Code.Require(result, "result");
Utility.Code.Require(test, "test");
VSTestResult vsResult = new VSTestResult(test);
vsResult.ComputerName = Environment.MachineName;
vsResult.Outcome = GetTestOutcome(result.Result);
// Boost.Test.Result.TestResult.Duration is in microseconds
// 1 millisecond = 10,000 ticks
// => 1 microsecond = 10 ticks
// Reference: https://msdn.microsoft.com/en-us/library/zz841zbz(v=vs.110).aspx
long ticks = (long) Math.Min((result.Duration * 10), long.MaxValue);
// Clamp tick count to 1 in case Boost duration is listed as 0
vsResult.Duration = new TimeSpan(Math.Max(ticks, 1));
if (result.LogEntries.Count > 0)
{
foreach (TestResultMessage message in GetTestMessages(result))
{
vsResult.Messages.Add(message);
}
// Test using the TestOutcome type since elements from the
// Boost Result type may be collapsed into a particular value
if (vsResult.Outcome == VSTestOutcome.Failed)
{
LogEntry error = GetLastError(result);
if (error != null)
{
vsResult.ErrorMessage = GetErrorMessage(result);
if (error.Source != null)
{
//String format for a hyper linkable Stack Trace
//Reference: NUnit3 Test Adapter.
vsResult.ErrorStackTrace = string.Format(CultureInfo.InvariantCulture, "at {0}() in {1}:line {2}", vsResult.TestCase.DisplayName, ConvertSlashes(error.Source.File), error.Source.LineNumber);
}
}
}
}
return vsResult;
}
public void SetUp()
{
this.TestCase = new VSTestCase(DefaultTestName, ExecutorUri, DefaultSource);
}
/// <summary>
/// Asserts general Visual Studio TestResult properties for the provided TestCase.
/// </summary>
/// <param name="result">The Visual Studio TestResult to test</param>
/// <param name="test">The Visual Studio TestCase for which the result is based on</param>
private void AssertVSTestModelProperties(VSTestResult result, VSTestCase test)
{
Assert.That(result.TestCase, Is.EqualTo(test));
Assert.That(result.ComputerName, Is.EqualTo(Environment.MachineName));
}
/// <summary>
/// Generates a default TestResult for a timeout exception.
/// </summary>
/// <param name="test">The test which failed due to a timeout.</param>
/// <param name="ex">The exception related to this timeout.</param>
/// <returns>A timed-out, failed TestResult related to the provided test.</returns>
private static VSTestResult GenerateTimeoutResult(VSTestCase test, Boost.Runner.TimeoutException ex)
{
VSTestResult result = new VSTestResult(test);
result.ComputerName = Environment.MachineName;
result.Outcome = TestOutcome.Failed;
result.Duration = TimeSpan.FromMilliseconds(ex.Timeout);
result.ErrorMessage = "Timeout exceeded. Test ran for more than " + ex.Timeout + " ms.";
if (!string.IsNullOrEmpty(test.CodeFilePath))
{
result.ErrorStackTrace = new SourceFileInfo(test.CodeFilePath, test.LineNumber).ToString();
}
return result;
}
/// <summary>
/// Generates a default TestResult for a 'test not found' exception.
/// </summary>
/// <param name="test">The test which failed due to a timeout.</param>
/// <returns>A timed-out, failed TestResult related to the provided test.</returns>
private static VSTestResult GenerateNotFoundResult(VSTestCase test)
{
VSTestResult result = new VSTestResult(test);
result.ComputerName = Environment.MachineName;
result.Outcome = TestOutcome.Skipped;
result.ErrorMessage = GetNotFoundErrorMessage(test);
return result;
}
/// <summary>
/// Provides a suitable message in case the provided test is not found.
/// </summary>
/// <param name="test">The test which was not found.</param>
/// <returns>A suitable 'not-found' for the provided test case.</returns>
private static string GetNotFoundErrorMessage(VSTestCase test)
{
if (test.FullyQualifiedName.Contains(' '))
{
return TestNotFound + " (Test name contains spaces)";
}
else if (test.FullyQualifiedName.Contains(','))
{
return TestNotFound + " (Test name contains commas)";
}
return TestNotFound;
}
