static VsTestFilter()
{
// Initialize the property cache
SupportedPropertiesCache = new Dictionary <string, TestProperty>(StringComparer.OrdinalIgnoreCase)
{
["FullyQualifiedName"] = TestCaseProperties.FullyQualifiedName,
["Name"] = TestCaseProperties.DisplayName,
["TestCategory"] = CategoryList.NUnitTestCategoryProperty,
["Category"] = CategoryList.NUnitTestCategoryProperty,
};
// Initialize the trait cache
var priorityTrait = new NTrait("Priority", "");
var categoryTrait = new NTrait("Category", "");
SupportedTraitCache = new Dictionary <string, NTrait>(StringComparer.OrdinalIgnoreCase)
{
["Priority"] = priorityTrait,
["TestCategory"] = categoryTrait,
["Category"] = categoryTrait
};
// Initialize the trait property map, since TFS doesnt know about traits
TraitPropertyMap = new Dictionary <NTrait, TestProperty>(new NTraitNameComparer());
var priorityProperty = TestProperty.Find("Priority") ??
TestProperty.Register("Priority", "Priority", typeof(string), typeof(TestCase));
TraitPropertyMap[priorityTrait] = priorityProperty;
var categoryProperty = TestProperty.Find("TestCategory") ??
TestProperty.Register("TestCategory", "TestCategory", typeof(string), typeof(TestCase));
TraitPropertyMap[categoryTrait] = categoryProperty;
// Initialize a merged list of properties and traits to fool TFS Build to think traits is properties
SupportedProperties = new List <string>();
SupportedProperties.AddRange(SupportedPropertiesCache.Keys);
SupportedProperties.AddRange(SupportedTraitCache.Keys);
}
static Constants()
{
TestClassProperty = TestProperty.Register("TestClass", "Test class", typeof(string),
TestPropertyAttributes.Hidden, typeof(Constants));
TestMethodProperty = TestProperty.Register("TestMethod", "Test method", typeof(string),
TestPropertyAttributes.Hidden, typeof(Constants));
}
/// <summary>
/// Initializes a new instance of the <see cref="DataCollectionTestCaseEventManager"/> class.
/// </summary>
/// <param name="testRunCache">
/// The test run cache.
/// </param>
public DataCollectionTestCaseEventManager(ITestRunCache testRunCache)
{
this.testRunCache = testRunCache;
this.testResultDictionary = new Dictionary <Guid, List <TestResult> >();
this.testCaseEndStatusMap = new HashSet <Guid>();
FlushResultTestResultPoperty = TestProperty.Register(id: "allowTestResultFlush", label: "allowTestResultFlush", category: string.Empty, description: string.Empty, valueType: typeof(bool), validateValueCallback: null, attributes: TestPropertyAttributes.None, owner: typeof(TestCase));
}
public void GetPropertiesShouldReturnListOfPropertiesInStore()
{
TestProperty tp = TestProperty.Register("dummyId", "dummyLabel", typeof(int), typeof(TestObjectTests));
var kvp = new KeyValuePair <TestProperty, object>(tp, 123);
testCase.SetPropertyValue(kvp.Key, kvp.Value);
var properties = testCase.GetProperties().ToList();
Assert.IsTrue(properties.Contains(kvp));
}
public void TestObjectSetPropertyValueShouldNotConvertIfValueMatchesPropertyDataType()
{
var property = TestProperty.Register("98", "p1", typeof(bool), typeof(TestObject));
var testobj = new TestableTestObject();
// This should not throw even if the runtime type of boolean where as specified
// type is object
testobj.SetPropertyValue <object>(property, false);
Assert.AreEqual(false, testobj.GetPropertyValue(property));
}
public GallioAdapter()
{
LoaderManager.InitializeAndSetupRuntimeIfNeeded();
testIdProperty = TestProperty.Register("Gallio.TestId", "Test id", typeof(string), typeof(TestCase));
testCaseFactory = new TestCaseFactory(testIdProperty);
cachingTestCaseFactory = new CachingTestCaseFactory(testCaseFactory, testIdProperty);
testResultFactory = new TestResultFactory();
testExplorer = new TestExplorer(cachingTestCaseFactory);
testRunner = new TestRunner(cachingTestCaseFactory, testResultFactory, testIdProperty);
}
public void IdentifyNonCoveredMutants()
{
var options = new StrykerOptions();
using (var endProcess = new EventWaitHandle(false, EventResetMode.ManualReset))
{
var mockVsTest = BuildVsTestRunner(options, endProcess, out var runner, OptimizationFlags.SkipUncoveredMutants);
var coverageProperty = TestProperty.Register("Stryker.Coverage", "Coverage", typeof(string), typeof(TestResult));
mockVsTest.Setup(x => x.AbortTestRun()).Verifiable();
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
It.Is <IEnumerable <string> >(t => t.Any(source => source == _testAssemblyPath)),
It.IsAny <string>(),
It.IsAny <ITestRunEventsHandler>(),
It.IsAny <ITestHostLauncher>())).Callback(
(IEnumerable <string> sources, string settings, ITestRunEventsHandler testRunEvents,
ITestHostLauncher host) =>
{
Task.Run(() =>
{
// generate coverage data covering only mutant (1)
settings.ShouldContain("DataCollector");
var results = new TestResult[_testCases.Length];
for (var i = 0; i < _testCases.Length; i++)
{
results[i] = new TestResult(_testCases[i])
{
Outcome = TestOutcome.Passed,
ComputerName = "."
};
results[i].SetPropertyValue(coverageProperty, "1;");
}
MoqTestRun(testRunEvents, results);
endProcess.Set();
});
});
var result = runner.CaptureCoverage(false, false);
// one mutant is covered
runner.CoverageMutants.IsCovered(1).ShouldBeTrue();
// it is covered by both tests
runner.CoverageMutants.GetTests(new Mutant()
{
Id = 1
}).ShouldBe(_testCases.Select(x => (TestDescription)x));
// verify Abort has been called
result.Success.ShouldBe(true);
}
}
private void onTestCaseFoundHandler(object sender, TestCaseFoundEventArgs e)
{
if (this.settings.JavaScriptTestFramework == JSTestFramework.Jest)
{
string package = "";
using (IEnumerator <string> enumer = sources.GetEnumerator())
{
enumer.MoveNext();
package = enumer.Current;
}
e.TestCase.SetPropertyValue(TestProperty.Register("jestConfigPath", "jestConfigPath", typeof(string), typeof(JSTest.TestRunner)), package);
}
this.discoverySink.SendTestCase(e.TestCase);
}
public void TestObjectShouldSerializeDateTimeOffsetForProperty()
{
var test = new TestableTestObject();
var testProperty1 = TestProperty.Register("12", "label1", typeof(DateTimeOffset), typeof(TestableTestObject));
var testPropertyData1 = DateTimeOffset.MaxValue;
test.SetPropertyValue(testProperty1, testPropertyData1);
var json = Serialize(test);
var expectedJson = "{\"Properties\":[{\"Key\":{\"Id\":\"12\",\"Label\":\"label1\",\"Category\":\"\",\"Description\":\"\",\"Attributes\":0,\"ValueType\":\"System.DateTimeOffset\"},\"Value\":\"9999-12-31T23:59:59.9999999+00:00\"}]}";
Assert.AreEqual(expectedJson, json);
}
public void TestObjectShouldSerializeStringArrayValueForProperty()
{
var test = new TestableTestObject();
var testProperty1 = TestProperty.Register("11", "label1", typeof(string[]), typeof(TestableTestObject));
var testPropertyData1 = new[] { "val1", "val2" };
test.SetPropertyValue(testProperty1, testPropertyData1);
var json = Serialize(test);
var expectedJson = "{\"Properties\":[{\"Key\":{\"Id\":\"11\",\"Label\":\"label1\",\"Category\":\"\",\"Description\":\"\",\"Attributes\":0,\"ValueType\":\"System.String[]\"},\"Value\":[\"val1\",\"val2\"]}]}";
Assert.AreEqual(expectedJson, json);
}
/// <summary>
/// Set the data sent via datacollection sink in the testresult property for upstream applications to read.
/// And removes the data from the dictionary.
/// </summary>
/// <param name="testResult">
/// The test Result.
/// </param>
private void SetInProcDataCollectionDataInTestResult(TestResult testResult)
{
// Loops through each datacollector reads the data collection data and sets as TestResult property.
foreach (var entry in this.InProcDataCollectors)
{
var dataCollectionData = ((InProcDataCollectionSink)this.inProcDataCollectionSink).GetDataCollectionDataSetForTestCase(testResult.TestCase.Id);
foreach (var keyValuePair in dataCollectionData)
{
var testProperty = TestProperty.Register(id: keyValuePair.Key, label: keyValuePair.Key, category: string.Empty, description: string.Empty, valueType: typeof(string), validateValueCallback: null, attributes: TestPropertyAttributes.None, owner: typeof(TestCase));
testResult.SetPropertyValue(testProperty, keyValuePair.Value);
}
}
}
public void ToTestElementShouldAssignTestCategoryOfUnitTestElement()
{
ObjectModel.TestCase testCase = CreateTestCase("TestCase1");
ObjectModel.TestResult result = new ObjectModel.TestResult(testCase);
TestProperty testProperty = TestProperty.Register("MSTestDiscoverer.TestCategory", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject));
testCase.SetPropertyValue(testProperty, new[] { "AsmLevel", "ClassLevel", "MethodLevel" });
var unitTestElement = Converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testCase.DisplayName, TrxLoggerConstants.UnitTestType, testCase);
object[] expected = new[] { "MethodLevel", "ClassLevel", "AsmLevel" };
CollectionAssert.AreEqual(expected, unitTestElement.TestCategories.ToArray().OrderByDescending(x => x.ToString()).ToArray());
}
public void ToTestElementShouldAssignWorkitemOfUnitTestElement()
{
TestPlatformObjectModel.TestCase testCase = CreateTestCase("TestCase1");
TestPlatformObjectModel.TestResult result = new TestPlatformObjectModel.TestResult(testCase);
TestProperty testProperty = TestProperty.Register("WorkItemIds", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject));
testCase.SetPropertyValue(testProperty, new[] { "3", "99999", "0" });
var unitTestElement = this.converter.ToTestElement(testCase.Id, Guid.Empty, Guid.Empty, testCase.DisplayName, TrxLoggerConstants.UnitTestType, testCase);
int[] expected = new[] { 0, 3, 99999 };
CollectionAssert.AreEquivalent(expected, unitTestElement.WorkItems.ToArray());
}
public void GetCustomPropertyValueFromTestCaseShouldReadCategoyrAttributesFromTestCase()
{
ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1");
TestProperty testProperty = TestProperty.Register("MSTestDiscoverer.TestCategory", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject));
testCase1.SetPropertyValue(testProperty, new[] { "ClassLevel", "AsmLevel" });
List <String> listCategoriesActual = Converter.GetCustomPropertyValueFromTestCase(testCase1, "MSTestDiscoverer.TestCategory");
List <String> listCategoriesExpected = new List <string>();
listCategoriesExpected.Add("ClassLevel");
listCategoriesExpected.Add("AsmLevel");
CollectionAssert.AreEqual(listCategoriesExpected, listCategoriesActual);
}
public void CaptureCoverageWhenSkippingUncovered()
{
var options = new StrykerOptions();
using (var endProcess = new EventWaitHandle(false, EventResetMode.ManualReset))
{
var mockVsTest = BuildVsTestRunner(options, endProcess, out var runner, OptimizationFlags.SkipUncoveredMutants);
var coverageProperty = TestProperty.Register("Stryker.Coverage", "Coverage", typeof(string), typeof(TestResult));
mockVsTest.Setup(x => x.AbortTestRun()).Verifiable();
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
It.Is <IEnumerable <string> >(t => t.Any(source => source == _testAssemblyPath)),
It.IsAny <string>(),
It.IsAny <ITestRunEventsHandler>(),
It.IsAny <ITestHostLauncher>())).Callback(
(IEnumerable <string> sources, string settings, ITestRunEventsHandler testRunEvents,
ITestHostLauncher host) =>
{
Task.Run(() =>
{
// ensure coverage capture is properly configured
settings.ShouldContain("DataCollector");
var results = new TestResult[_testCases.Length];
for (var i = 0; i < _testCases.Length; i++)
{
results[i] = new TestResult(_testCases[i])
{
Outcome = TestOutcome.Passed,
ComputerName = "."
};
}
results[0].SetPropertyValue(coverageProperty, "1;");
MoqTestRun(testRunEvents, results);
endProcess.Set();
});
});
var result = runner.CaptureCoverage(false, false);
// only one mutant is covered
runner.CoveredMutants.ShouldHaveSingleItem();
// capture when ok
result.Success.ShouldBe(true);
}
}
private void InitProperties(ISet <string> traitNames)
{
_testPropertiesMap[nameof(TestCaseProperties.FullyQualifiedName)] = TestCaseProperties.FullyQualifiedName;
_testPropertiesMap[nameof(TestCaseProperties.DisplayName)] = TestCaseProperties.DisplayName;
_testPropertiesMap[nameof(TestCaseProperties.LineNumber)] = TestCaseProperties.LineNumber;
_testPropertiesMap[nameof(TestCaseProperties.CodeFilePath)] = TestCaseProperties.CodeFilePath;
_testPropertiesMap[nameof(TestCaseProperties.ExecutorUri)] = TestCaseProperties.ExecutorUri;
_testPropertiesMap[nameof(TestCaseProperties.Id)] = TestCaseProperties.Id;
_testPropertiesMap[nameof(TestCaseProperties.Source)] = TestCaseProperties.Source;
foreach (string traitName in traitNames)
{
var traitTestProperty = TestProperty.Find(traitName) ??
TestProperty.Register(traitName, traitName, typeof(string), typeof(TestCase));
_traitPropertiesMap[traitName] = traitTestProperty;
}
}
public void GetCustomPropertyValueFromTestCaseShouldReadWorkItemAttributesFromTestCase()
{
ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1");
TestProperty testProperty = TestProperty.Register("WorkItemIds", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject));
testCase1.SetPropertyValue(testProperty, new[] { "99999", "0" });
var converter = new Converter(new Mock <IFileHelper>().Object, new TrxFileHelper());
List <string> listWorkItemsActual = converter.GetCustomPropertyValueFromTestCase(testCase1, "WorkItemIds");
List <string> listWorkItemsExpected = new List <string>();
listWorkItemsExpected.Add("99999");
listWorkItemsExpected.Add("0");
CollectionAssert.AreEqual(listWorkItemsExpected, listWorkItemsActual);
}
private static TestProperty Convert(this Common.Models.TestProperty testProperty)
{
var result = TestProperty.Find(testProperty.Id);
if (result == null)
{
var type = GetType(testProperty.ValueType);
if (type == null)
{
return(null);
}
result = TestProperty.Register(testProperty.Id, testProperty.Label, type, testProperty.Attributes.Convert(), typeof(ConversionExtensions));
result.Description = testProperty.Description;
result.Category = testProperty.Category;
}
return(result);
}
public void TestCaseObjectShouldSerializeCustomProperties()
{
var test = new TestableTestObject();
var testProperty1 = TestProperty.Register("1", "label1", typeof(Guid), typeof(TestableTestObject));
var testPropertyData1 = Guid.Parse("02048dfd-3da7-475d-a011-8dd1121855ec");
var testProperty2 = TestProperty.Register("2", "label2", typeof(int), typeof(TestableTestObject));
var testPropertyData2 = 29;
test.SetPropertyValue(testProperty1, testPropertyData1);
test.SetPropertyValue(testProperty2, testPropertyData2);
var json = Serialize(test);
// Use raw deserialization to validate basic properties
var expectedJson = "{\"Properties\":[{\"Key\":{\"Id\":\"1\",\"Label\":\"label1\",\"Category\":\"\",\"Description\":\"\",\"Attributes\":0,\"ValueType\":\"System.Guid\"},\"Value\":\"02048dfd-3da7-475d-a011-8dd1121855ec\"},{\"Key\":{\"Id\":\"2\",\"Label\":\"label2\",\"Category\":\"\",\"Description\":\"\",\"Attributes\":0,\"ValueType\":\"System.Int32\"},\"Value\":29}]}";
Assert.AreEqual(expectedJson, json);
}
private void InitProperties(ISet <string> traitNames)
{
_testPropertiesMap[nameof(TestCaseProperties.FullyQualifiedName)] = TestCaseProperties.FullyQualifiedName;
_testPropertiesMap[nameof(TestCaseProperties.DisplayName)] = TestCaseProperties.DisplayName;
_testPropertiesMap[nameof(TestCaseProperties.LineNumber)] = TestCaseProperties.LineNumber;
_testPropertiesMap[nameof(TestCaseProperties.CodeFilePath)] = TestCaseProperties.CodeFilePath;
_testPropertiesMap[nameof(TestCaseProperties.ExecutorUri)] = TestCaseProperties.ExecutorUri;
_testPropertiesMap[nameof(TestCaseProperties.Id)] = TestCaseProperties.Id;
_testPropertiesMap[nameof(TestCaseProperties.Source)] = TestCaseProperties.Source;
foreach (string traitName in traitNames)
{
if (_testPropertiesMap.Keys.Contains(traitName))
{
_logger.LogWarning(String.Format(Resources.TraitIgnoreMessage, traitName));
continue;
}
var traitTestProperty = TestProperty.Find(traitName) ??
TestProperty.Register(traitName, traitName, "", "", typeof(string),
ValidateTraitValue, TestPropertyAttributes.None, typeof(TestCase));
_traitPropertiesMap[traitName] = traitTestProperty;
}
}
private void InitProperties(ISet <string> traitNames)
{
_testPropertiesMap[nameof(TestCaseProperties.FullyQualifiedName)] = TestCaseProperties.FullyQualifiedName;
_testPropertiesMap[nameof(TestCaseProperties.DisplayName)] = TestCaseProperties.DisplayName;
_testPropertiesMap[nameof(TestCaseProperties.LineNumber)] = TestCaseProperties.LineNumber;
_testPropertiesMap[nameof(TestCaseProperties.CodeFilePath)] = TestCaseProperties.CodeFilePath;
_testPropertiesMap[nameof(TestCaseProperties.ExecutorUri)] = TestCaseProperties.ExecutorUri;
_testPropertiesMap[nameof(TestCaseProperties.Id)] = TestCaseProperties.Id;
_testPropertiesMap[nameof(TestCaseProperties.Source)] = TestCaseProperties.Source;
foreach (string traitName in traitNames)
{
if (_testPropertiesMap.Keys.Contains(traitName))
{
_logger.LogWarning($"Trait has same name as base test property and will thus be ignored for test case filtering: {traitName}");
continue;
}
var traitTestProperty = TestProperty.Find(traitName) ??
TestProperty.Register(traitName, traitName, "", "", typeof(string),
ValidateTraitValue, TestPropertyAttributes.None, typeof(TestCase));
_traitPropertiesMap[traitName] = traitTestProperty;
}
}
// initialize the test context and mock objects
public VsTestRunnersShould()
{
var currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
var filesystemRoot = Path.GetPathRoot(currentDirectory);
var sourceFile = File.ReadAllText(currentDirectory + "/TestResources/ExampleSourceFile.cs");
var testProjectPath = FilePathUtils.NormalizePathSeparators(Path.Combine(filesystemRoot, "TestProject", "TestProject.csproj"));
var projectUnderTestPath = FilePathUtils.NormalizePathSeparators(Path.Combine(filesystemRoot, "ExampleProject", "ExampleProject.csproj"));
const string defaultTestProjectFileContents = @"<Project Sdk=""Microsoft.NET.Sdk"">
<PropertyGroup>
<TargetFramework>netcoreapp2.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include=""Microsoft.NET.Test.Sdk"" Version = ""15.5.0"" />
<PackageReference Include=""xunit"" Version=""2.3.1"" />
<PackageReference Include=""xunit.runner.visualstudio"" Version=""2.3.1"" />
<DotNetCliToolReference Include=""dotnet-xunit"" Version=""2.3.1"" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include=""..\ExampleProject\ExampleProject.csproj"" />
</ItemGroup>
</Project>";
_testAssemblyPath = Path.Combine(filesystemRoot, "_firstTest", "bin", "Debug", "TestApp.dll");
_executorUri = new Uri("exec://nunit");
var firstTest = new TestCase("T0", _executorUri, _testAssemblyPath);
var secondTest = new TestCase("T1", _executorUri, _testAssemblyPath);
var content = new FolderComposite();
_coverageProperty = TestProperty.Register("Stryker.Coverage", "Coverage", typeof(string), typeof(TestResult));
_mutant = new Mutant {
Id = 0
};
_otherMutant = new Mutant {
Id = 1
};
_projectContents = content;
_projectContents.Add(new FileLeaf {
Mutants = new[] { _otherMutant, _mutant }
});
_targetProject = new ProjectInfo()
{
TestProjectAnalyzerResults = new List <IAnalyzerResult> {
TestHelper.SetupProjectAnalyzerResult(
properties: new Dictionary <string, string>()
{
{ "TargetDir", Path.GetDirectoryName(_testAssemblyPath) },
{ "TargetFileName", Path.GetFileName(_testAssemblyPath) }
},
targetFramework: "toto").Object
},
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(
properties: new Dictionary <string, string>()
{
{ "TargetDir", Path.Combine(filesystemRoot, "app", "bin", "Debug") },
{ "TargetFileName", "AppToTest.dll" }
}).Object,
ProjectContents = _projectContents
};
//CodeInjection.HelperNamespace = "Stryker.Core.UnitTest.TestRunners";
_fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ projectUnderTestPath, new MockFileData(defaultTestProjectFileContents) },
{ Path.Combine(filesystemRoot, "ExampleProject", "Recursive.cs"), new MockFileData(sourceFile) },
{ Path.Combine(filesystemRoot, "ExampleProject", "OneFolderDeeper", "Recursive.cs"), new MockFileData(sourceFile) },
{ testProjectPath, new MockFileData(defaultTestProjectFileContents) },
{ _testAssemblyPath, new MockFileData("Bytecode") },
{ Path.Combine(filesystemRoot, "app", "bin", "Debug", "AppToTest.dll"), new MockFileData("Bytecode") },
});
_testCases = new List <TestCase> {
firstTest, secondTest
};
}
internal static void AddTelemetry(TestResult testResult, IDictionary <string, int> adapterTelemetry)
{
var executorUri = testResult?.TestCase?.ExecutorUri;
// add additional info for mstestadapter/v1
if (IsMSTestV1Adapter(executorUri))
{
if (testTypeProperty == null)
{
// this is present when the legacy runner is used, and contains a guid which
// is the test type.
// GenericTestType 982B8C01-1A8A-48F5-B98A-67EE64BC8687
// OrderedTestType ec4800e8-40e5-4ab3-8510-b8bf29b1904d
// UnitTestType 13CDC9D9-DDB5-4fa4-A97D-D965CCFC6D4B
// WebTestType 4e7599fa-5ecb-43e9-a887-cd63cf72d207
// CodedWebTestType 37e36796-fb51-4610-8d5c-e00ceaa68b9f
testTypeProperty = TestProperty.Register("TestType", "TestType", typeof(Guid), typeof(TestResult));
}
if (extensionIdProperty == null)
{
extensionIdProperty = TestProperty.Register("ExtensionId", "ExtensionId", typeof(string), typeof(TestResult));
}
// Get addional data from test result passed by MSTestv1
// Only legacy tests have testtype.
var testType = testResult.GetPropertyValue <Guid>(testTypeProperty, Guid.Empty);
var hasTestType = testType != Guid.Empty;
string key;
if (hasTestType)
{
var testExtension = testResult.GetPropertyValue <string>(extensionIdProperty, null);
var hasExtension = !string.IsNullOrWhiteSpace(testExtension);
if (hasExtension && testExtension.StartsWith("urn:"))
{
// remove urn: prefix
testExtension = testExtension.Remove(0, 4);
}
if (hasExtension)
{
key = $"{testResult.TestCase.ExecutorUri.AbsoluteUri}.legacy.extension.{testExtension}.count";
}
else
{
key = $"{testResult.TestCase.ExecutorUri.AbsoluteUri}.legacy.count";
}
}
else
{
key = $"{testResult.TestCase.ExecutorUri.AbsoluteUri}.count";
}
if (adapterTelemetry.ContainsKey(key))
{
adapterTelemetry[key]++;
}
else
{
adapterTelemetry[key] = 1;
}
}
}
static TestProperty GetTestProperty()
{
return(TestProperty.Register("XunitTestCase", "xUnit.net Test Case", typeof(string), typeof(VsTestRunner)));
}
/// <inheritdoc/>
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
var testCase = new TestCase();
var data = JObject.Load(reader);
var properties = data["Properties"];
if (properties != null && properties.HasValues)
{
// Every class that inherits from TestObject uses a properties store for <Property, Object>
// key value pairs.
foreach (var property in properties.Values <JToken>())
{
var testProperty = property["Key"].ToObject <TestProperty>();
// Let the null values be passed in as null data
var token = property["Value"];
string propertyData = null;
if (token.Type != JTokenType.Null)
{
// If the property is already a string. No need to convert again.
if (token.Type == JTokenType.String)
{
propertyData = token.ToObject <string>();
}
else
{
// On deserialization, the value for each TestProperty is always a string. It is up
// to the consumer to deserialize it further as appropriate.
propertyData = token.ToString(Formatting.None).Trim('"');
}
}
switch (testProperty.Id)
{
case "TestCase.Id":
testCase.Id = Guid.Parse(propertyData); break;
case "TestCase.ExecutorUri":
testCase.ExecutorUri = new Uri(propertyData); break;
case "TestCase.FullyQualifiedName":
testCase.FullyQualifiedName = propertyData; break;
case "TestCase.DisplayName":
testCase.DisplayName = propertyData; break;
case "TestCase.Source":
testCase.Source = propertyData; break;
case "TestCase.CodeFilePath":
testCase.CodeFilePath = propertyData; break;
case "TestCase.LineNumber":
testCase.LineNumber = int.Parse(propertyData); break;
default:
// No need to register member properties as they get registered as part of TestCaseProperties class.
testProperty = TestProperty.Register(testProperty.Id, testProperty.Label, testProperty.GetValueType(), testProperty.Attributes, typeof(TestObject));
testCase.SetPropertyValue(testProperty, propertyData);
break;
}
}
}
return(testCase);
}
public void RunRelevantTestsOnStaticWhenPerTestCoverage()
{
var options = new StrykerOptions();
using (var endProcess = new EventWaitHandle(false, EventResetMode.ManualReset))
{
var mockVsTest = BuildVsTestRunner(options, endProcess, out var runner, OptimizationFlags.CoverageBasedTest | OptimizationFlags.CaptureCoveragePerTest);
var coverageProperty = TestProperty.Register("Stryker.Coverage", "Coverage", typeof(string), typeof(TestResult));
mockVsTest.Setup(x => x.AbortTestRun()).Verifiable();
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
It.Is <IEnumerable <string> >(t => t.Any(source => source == _testAssemblyPath)),
It.IsAny <string>(),
It.IsAny <ITestRunEventsHandler>(),
It.IsAny <ITestHostLauncher>())).Callback(
(IEnumerable <string> sources, string settings, ITestRunEventsHandler testRunEvents,
ITestHostLauncher host) =>
{
settings.ShouldContain("DataCollector");
var results = new TestResult[_testCases.Length];
for (var i = 0; i < _testCases.Length; i++)
{
results[i] = new TestResult(_testCases[i])
{
Outcome = TestOutcome.Passed,
ComputerName = "."
};
}
results[0].SetPropertyValue(coverageProperty, "0,1;1");
MoqTestRun(testRunEvents, results);
endProcess.Set();
});
runner.CaptureCoverage(false, false);
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
It.Is <IEnumerable <TestCase> >(t => t.Count() == 1),
It.IsAny <string>(),
It.IsAny <ITestRunEventsHandler>(),
It.IsAny <ITestHostLauncher>())).Callback(
(IEnumerable <TestCase> sources, string settings, ITestRunEventsHandler testRunEvents,
ITestHostLauncher host) =>
{
settings.ShouldNotContain("DataCollector");
var results = new List <TestResult>();
foreach (var test in sources)
{
results.Add(new TestResult(test)
{
Outcome = TestOutcome.Passed,
ComputerName = "."
});
}
results.ShouldHaveSingleItem();
MoqTestRun(testRunEvents, results);
endProcess.Set();
});
var mutant = new Mutant {
Id = 1
};
runner.CoverageMutants.GetTests(mutant).ShouldHaveSingleItem();
var otherMutant = new Mutant {
Id = 0
};
foreach (var testDescription in runner.CoverageMutants.GetTests(otherMutant))
{
otherMutant.CoveringTest[testDescription.Guid] = false;
}
var result = runner.RunAll(0, otherMutant);
// verify Abort has been called
result.Success.ShouldBe(true);
}
}
public void RunAllTestsOnStatic()
{
var options = new StrykerOptions();
using (var endProcess = new EventWaitHandle(false, EventResetMode.ManualReset))
{
var mockVsTest = BuildVsTestRunner(options, endProcess, out var runner, OptimizationFlags.CoverageBasedTest);
var coverageProperty = TestProperty.Register("Stryker.Coverage", "Coverage", typeof(string), typeof(TestResult));
mockVsTest.Setup(x => x.AbortTestRun()).Verifiable();
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
It.Is <IEnumerable <string> >(t => t.Any(source => source == _testAssemblyPath)),
It.Is <string>(settings => settings.Contains("DataCollector")),
It.IsAny <ITestRunEventsHandler>(),
It.IsAny <ITestHostLauncher>())).Callback(
(IEnumerable <string> sources, string settings, ITestRunEventsHandler testRunEvents,
ITestHostLauncher host) =>
{
Task.Run(() =>
{
// provides coverage result with static mutant
var results = new TestResult[_testCases.Length];
for (var i = 0; i < _testCases.Length; i++)
{
results[i] = new TestResult(_testCases[i])
{
Outcome = TestOutcome.Passed,
ComputerName = "."
};
}
results[0].SetPropertyValue(coverageProperty, "1;1");
MoqTestRun(testRunEvents, results);
endProcess.Set();
});
});
runner.CaptureCoverage(false, false);
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
It.IsAny <IEnumerable <string> >(),
It.Is <string>(settings => !settings.Contains("DataCollector")),
It.IsAny <ITestRunEventsHandler>(),
It.IsAny <ITestHostLauncher>())).Callback(
(IEnumerable <string> sources, string settings, ITestRunEventsHandler testRunEvents,
ITestHostLauncher host) =>
{
var results = _testCases.Select(test => new TestResult(test)
{
Outcome = TestOutcome.Failed, ComputerName = "."
}).ToList();
// we expect only one test
results.Count.ShouldBe(2);
MoqTestRun(testRunEvents, results);
endProcess.Set();
});
var mutant = new Mutant {
Id = 1
};
var otherMutant = new Mutant {
Id = 0
};
var mutants = new List <Mutant> {
mutant, otherMutant
};
// process coverage info
runner.CoverageMutants.UpdateMutants(mutants, _testCases.Length);
// mutant 1 is covered
mutant.MustRunAllTests.ShouldBeTrue();
var result = runner.RunAll(0, mutant);
// mutant is killed
result.Success.ShouldBe(false);
// mutant 0 is not covered
result = runner.RunAll(0, otherMutant);
// tests are ok
result.Success.ShouldBe(true);
}
}
public void RunOnlyUsefulTest()
{
var options = new StrykerOptions();
using (var endProcess = new EventWaitHandle(false, EventResetMode.ManualReset))
{
var mockVsTest = BuildVsTestRunner(options, endProcess, out var runner, OptimizationFlags.CoverageBasedTest);
var coverageProperty = TestProperty.Register("Stryker.Coverage", "Coverage", typeof(string), typeof(TestResult));
mockVsTest.Setup(x => x.AbortTestRun()).Verifiable();
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
It.Is <IEnumerable <string> >(t => t.Any(source => source == _testAssemblyPath)),
It.IsAny <string>(),
It.IsAny <ITestRunEventsHandler>(),
It.IsAny <ITestHostLauncher>())).Callback(
(IEnumerable <string> sources, string settings, ITestRunEventsHandler testRunEvents,
ITestHostLauncher host) =>
{
Task.Run(() =>
{
// generate coverage results
settings.ShouldContain("DataCollector");
var results = new TestResult[_testCases.Length];
for (var i = 0; i < _testCases.Length; i++)
{
results[i] = new TestResult(_testCases[i])
{
Outcome = TestOutcome.Passed,
ComputerName = "."
};
}
// only first test covers one mutant
results[0].SetPropertyValue(coverageProperty, "1;");
MoqTestRun(testRunEvents, results);
endProcess.Set();
});
});
runner.CaptureCoverage(false, false);
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
// we expect a run with only one test
It.Is <IEnumerable <TestCase> >(t => t.Count() == 1),
It.IsAny <string>(),
It.IsAny <ITestRunEventsHandler>(),
It.IsAny <ITestHostLauncher>())).Callback(
(IEnumerable <TestCase> sources, string settings, ITestRunEventsHandler testRunEvents,
ITestHostLauncher host) =>
{
settings.ShouldNotContain("DataCollector");
// setup a normal test run
var results = sources.Select(test => new TestResult(test)
{
Outcome = TestOutcome.Passed, ComputerName = "."
}).ToList();
results.ShouldHaveSingleItem();
MoqTestRun(testRunEvents, results);
endProcess.Set();
});
var mutant = new Mutant {
Id = 1
};
var mutants = new List <Mutant> {
mutant
};
// process coverage information
runner.CoverageMutants.UpdateMutants(mutants, _testCases.Length);
var result = runner.RunAll(0, mutant);
// verify Abort has been called
result.Success.ShouldBe(true);
}
}
/// <inheritdoc/>
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
var data = JObject.Load(reader);
var testCase = data["TestCase"].ToObject <TestCase>(serializer);
var testResult = new TestResult(testCase);
// Add attachments for the result
var attachments = data["Attachments"];
if (attachments != null && attachments.HasValues)
{
foreach (var attachment in attachments.Values <JToken>())
{
if (attachment.Type != JTokenType.Null)
{
testResult.Attachments.Add(attachment.ToObject <AttachmentSet>(serializer));
}
}
}
// Add messages for the result
var messages = data["Messages"];
if (messages != null && messages.HasValues)
{
foreach (var message in messages.Values <JToken>())
{
if (message.Type != JTokenType.Null)
{
testResult.Messages.Add(message.ToObject <TestResultMessage>(serializer));
}
}
}
JToken properties = data["Properties"];
if (properties != null && properties.HasValues)
{
// Every class that inherits from TestObject uses a properties store for <Property, Object>
// key value pairs.
foreach (var property in properties.Values <JToken>())
{
var testProperty = property["Key"].ToObject <TestProperty>(serializer);
// Let the null values be passed in as null data
var token = property["Value"];
string propertyData = null;
if (token.Type != JTokenType.Null)
{
// If the property is already a string. No need to convert again.
if (token.Type == JTokenType.String)
{
propertyData = token.ToObject <string>(serializer);
}
else
{
// On deserialization, the value for each TestProperty is always a string. It is up
// to the consumer to deserialize it further as appropriate.
propertyData = token.ToString(Formatting.None).Trim('"');
}
}
switch (testProperty.Id)
{
case "TestResult.DisplayName":
testResult.DisplayName = propertyData; break;
case "TestResult.ComputerName":
testResult.ComputerName = propertyData ?? string.Empty; break;
case "TestResult.Outcome":
testResult.Outcome = (TestOutcome)Enum.Parse(typeof(TestOutcome), propertyData); break;
case "TestResult.Duration":
testResult.Duration = TimeSpan.Parse(propertyData); break;
case "TestResult.StartTime":
testResult.StartTime = DateTimeOffset.Parse(propertyData); break;
case "TestResult.EndTime":
testResult.EndTime = DateTimeOffset.Parse(propertyData); break;
case "TestResult.ErrorMessage":
testResult.ErrorMessage = propertyData; break;
case "TestResult.ErrorStackTrace":
testResult.ErrorStackTrace = propertyData; break;
default:
// No need to register member properties as they get registered as part of TestResultProperties class.
testProperty = TestProperty.Register(testProperty.Id, testProperty.Label, testProperty.GetValueType(), testProperty.Attributes, typeof(TestObject));
testResult.SetPropertyValue(testProperty, propertyData);
break;
}
}
}
return(testResult);
}