public static async ValueTask TestCaseOrdererWhichThrowsLogsMessageAndDoesNotReorderTestCollections()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var collection1 = Mocks.TestCollection(displayName: "AAA", uniqueID: "collection-1");
var testCase1 = TestCaseForTestCollection(collection1);
var collection2 = Mocks.TestCollection(displayName: "ZZZZ", uniqueID: "collection-2");
var testCase2 = TestCaseForTestCollection(collection2);
var collection3 = Mocks.TestCollection(displayName: "MM", uniqueID: "collection-3");
var testCase3 = TestCaseForTestCollection(collection3);
var testCases = new[] { testCase1, testCase2, testCase3 };
var runner = TestableTestAssemblyRunner.Create(testCases: testCases, testCollectionOrderer: new ThrowingCollectionOrderer());
await runner.RunAsync();
Assert.Collection(
runner.CollectionsRun,
collection => Assert.Same(collection1, collection.Item1),
collection => Assert.Same(collection2, collection.Item1),
collection => Assert.Same(collection3, collection.Item1)
);
var diagnosticMessage = Assert.Single(spy.Messages.OfType <_DiagnosticMessage>());
Assert.StartsWith("Test collection orderer 'TestAssemblyRunnerTests+TestCollectionOrderer+ThrowingCollectionOrderer' threw 'System.DivideByZeroException' during ordering: Attempted to divide by zero.", diagnosticMessage.Message);
}
public void NoAttribute()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var assembly = Mocks.AssemblyInfo();
var framework = ExtensibilityPointFactory.GetTestFramework(assembly);
Assert.IsType <XunitTestFramework>(framework);
Assert.Empty(spy.Messages);
}
public void Attribute_WithDiscoverer_NoMessageSink()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var attribute = Mocks.TestFrameworkAttribute <AttributeWithDiscoverer>();
var assembly = Mocks.AssemblyInfo(attributes: new[] { attribute });
ExtensibilityPointFactory.GetTestFramework(assembly);
Assert.Empty(spy.Messages);
}
public void Attribute_NoDiscoverer()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var attribute = Mocks.TestFrameworkAttribute <AttributeWithoutDiscoverer>();
var assembly = Mocks.AssemblyInfo(attributes: new[] { attribute });
var framework = ExtensibilityPointFactory.GetTestFramework(assembly);
Assert.IsType <XunitTestFramework>(framework);
AssertSingleDiagnosticMessage(spy, "Assembly-level test framework attribute was not decorated with [TestFrameworkDiscoverer]");
}
public static async void SettingTestCaseOrdererWithThrowingConstructorLogsDiagnosticMessage()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var testCase = TestData.XunitTestCase <TestClassWithCtorThrowingTestCaseOrder>("Passing");
var runner = TestableXunitTestClassRunner.Create(testCase);
await runner.RunAsync();
Assert.IsType <MockTestCaseOrderer>(runner.RunTestMethodsAsync_TestCaseOrderer);
var diagnosticMessage = Assert.Single(spy.Messages.Cast <_DiagnosticMessage>());
Assert.StartsWith("Class-level test case orderer 'XunitTestClassRunnerTests+TestCaseOrderer+MyCtorThrowingTestCaseOrderer' for test class 'XunitTestClassRunnerTests+TestCaseOrderer+TestClassWithCtorThrowingTestCaseOrder' threw 'System.DivideByZeroException' during construction: Attempted to divide by zero.", diagnosticMessage.Message);
}
public static async void SettingUnknownTestCaseOrderLogsDiagnosticMessage()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var testCase = TestData.XunitTestCase <TestClassWithUnknownTestCaseOrderer>("Passing");
var runner = TestableXunitTestClassRunner.Create(testCase);
await runner.RunAsync();
Assert.IsType <MockTestCaseOrderer>(runner.RunTestMethodsAsync_TestCaseOrderer);
var diagnosticMessage = Assert.Single(spy.Messages.Cast <_DiagnosticMessage>());
Assert.Equal("Could not find type 'UnknownType' in UnknownAssembly for class-level test case orderer on test class 'XunitTestClassRunnerTests+TestCaseOrderer+TestClassWithUnknownTestCaseOrderer'", diagnosticMessage.Message);
}
public void Attribute_ThrowingDiscovererMethod()
{
CultureInfo.CurrentUICulture = CultureInfo.InvariantCulture;
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var attribute = Mocks.TestFrameworkAttribute <AttributeWithThrowingDiscovererMethod>();
var assembly = Mocks.AssemblyInfo(attributes: new[] { attribute });
var framework = ExtensibilityPointFactory.GetTestFramework(assembly);
Assert.IsType <XunitTestFramework>(framework);
AssertSingleDiagnosticMessage(spy, "Exception thrown during test framework discoverer construction: System.DivideByZeroException: Attempted to divide by zero.");
}
public static async void SettingTestCaseOrdererWithThrowingConstructorLogsDiagnosticMessage()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var collection = new TestCollection(Mocks.TestAssembly(), Reflector.Wrap(typeof(CollectionWithCtorThrowingTestCaseOrderer)), "TestCollectionDisplayName");
var testCase = TestData.XunitTestCase <XunitTestCollectionRunnerTests>("DisposesFixtures", collection);
var runner = TestableXunitTestCollectionRunner.Create(testCase);
await runner.RunAsync();
Assert.IsType <MockTestCaseOrderer>(runner.RunTestClassesAsync_TestCaseOrderer);
var diagnosticMessage = Assert.Single(spy.Messages.Cast <_DiagnosticMessage>());
Assert.StartsWith("Collection-level test case orderer 'XunitTestCollectionRunnerTests+TestCaseOrderer+MyCtorThrowingTestCaseOrderer' for test collection 'TestCollectionDisplayName' threw 'System.DivideByZeroException' during construction: Attempted to divide by zero.", diagnosticMessage.Message);
}
public static async void SettingUnknownTestCaseOrderLogsDiagnosticMessage()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var collection = new TestCollection(Mocks.TestAssembly(), Reflector.Wrap(typeof(CollectionWithUnknownTestCaseOrderer)), "TestCollectionDisplayName");
var testCase = TestData.XunitTestCase <XunitTestCollectionRunnerTests>("DisposesFixtures", collection);
var runner = TestableXunitTestCollectionRunner.Create(testCase);
await runner.RunAsync();
Assert.IsType <MockTestCaseOrderer>(runner.RunTestClassesAsync_TestCaseOrderer);
var diagnosticMessage = Assert.Single(spy.Messages.Cast <_DiagnosticMessage>());
Assert.Equal("Could not find type 'UnknownType' in UnknownAssembly for collection-level test case orderer on test collection 'TestCollectionDisplayName'", diagnosticMessage.Message);
}
public static void InvalidCustomFactoryFallsBackToDefault()
{
var spyMessageSink = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spyMessageSink;
var behaviorAttribute = Mocks.CollectionBehaviorAttribute <object>();
var assembly = Mocks.AssemblyInfo(attributes: new[] { behaviorAttribute });
var discoverer = TestableXunitTestFrameworkDiscoverer.Create(assembly);
Assert.IsType <CollectionPerClassTestCollectionFactory>(discoverer.TestCollectionFactory);
var message = Assert.Single(spyMessageSink.Messages);
var diagMessage = Assert.IsType <_DiagnosticMessage>(message);
Assert.Equal("Test collection factory type 'System.Object' does not implement IXunitTestCollectionFactory", diagMessage.Message);
}
public static void CustomTraitWithoutDiscoverer()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var trait = Mocks.TraitAttribute <BadTraitAttribute>();
var testMethod = Mocks.TestMethod(classAttributes: new[] { trait });
var testCase = new TestableXunitTestCase(testMethod);
Assert.Empty(testCase.Traits);
var diagnosticMessages = spy.Messages.OfType <_DiagnosticMessage>();
var diagnosticMessage = Assert.Single(diagnosticMessages);
Assert.Equal($"Trait attribute on '{testCase.TestCaseDisplayName}' did not have [TraitDiscoverer]", diagnosticMessage.Message);
}
public async ValueTask ExceptionDuringFindTestsForType_ReportsExceptionAsDiagnosticMessage()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var mockType = Mocks.TypeInfo("MockType");
var discoverer = TestableTestFrameworkDiscoverer.Create(mockType);
discoverer.FindTestsForType_Exception = new DivideByZeroException();
await discoverer.Find();
var message = Assert.Single(spy.Messages.OfType <_DiagnosticMessage>());
Assert.StartsWith($"Exception during discovery:{Environment.NewLine}System.DivideByZeroException: Attempted to divide by zero.", message.Message);
}
public static void MultiplyDeclaredCollectionsRaisesEnvironmentalWarning()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var testType = Mocks.TypeInfo("type", attributes: new[] { Mocks.CollectionAttribute("This is a test collection") });
var collectionDefinition1 = Mocks.TypeInfo("collectionDefinition1", attributes: new[] { Mocks.CollectionDefinitionAttribute("This is a test collection") });
var collectionDefinition2 = Mocks.TypeInfo("collectionDefinition2", attributes: new[] { Mocks.CollectionDefinitionAttribute("This is a test collection") });
var assembly = Mocks.TestAssembly(@"C:\Foo\bar.dll", types: new[] { collectionDefinition1, collectionDefinition2 });
var factory = new CollectionPerAssemblyTestCollectionFactory(assembly);
factory.Get(testType);
var msg = Assert.Single(spy.Messages.OfType <_DiagnosticMessage>().Select(m => m.Message));
Assert.Equal("Multiple test collections declared with name 'This is a test collection': collectionDefinition1, collectionDefinition2", msg);
}
public void IncompatibleOrInvalidTypesGetDefaultBehavior(string factoryTypeName, string expectedMessage)
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
#if BUILD_X86
expectedMessage = expectedMessage.Replace("xunit.v3.core.tests", "xunit.v3.core.tests.x86");
var attr = Mocks.CollectionBehaviorAttribute(factoryTypeName, "xunit.v3.core.tests.x86");
#else
var attr = Mocks.CollectionBehaviorAttribute(factoryTypeName, "xunit.v3.core.tests");
#endif
var assembly = Mocks.TestAssembly();
var result = ExtensibilityPointFactory.GetXunitTestCollectionFactory(attr, assembly);
AssertSingleDiagnosticMessage(spy, expectedMessage);
}
public static async ValueTask SettingUnknownTestCollectionOrderLogsDiagnosticMessage()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var ordererAttribute = Mocks.TestCollectionOrdererAttribute("UnknownType", "UnknownAssembly");
var assembly = Mocks.TestAssembly("assembly.dll", assemblyAttributes: new[] { ordererAttribute });
await using var context = TestableXunitTestAssemblyRunnerContext.Create(assembly: assembly);
await context.InitializeAsync();
var result = context.AssemblyTestCollectionOrderer;
Assert.Null(result);
var diagnosticMessage = Assert.Single(spy.Messages.OfType <_DiagnosticMessage>());
Assert.Equal("Could not find type 'UnknownType' in UnknownAssembly for assembly-level test collection orderer", diagnosticMessage.Message);
}
public static async ValueTask SettingTestCollectionOrdererWithThrowingConstructorLogsDiagnosticMessage()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var ordererAttribute = Mocks.TestCollectionOrdererAttribute <CtorThrowingCollectionOrderer>();
var assembly = Mocks.TestAssembly("assembly.dll", assemblyAttributes: new[] { ordererAttribute });
await using var context = TestableXunitTestAssemblyRunnerContext.Create(assembly: assembly);
await context.InitializeAsync();
var result = context.AssemblyTestCollectionOrderer;
Assert.Null(result);
var diagnosticMessage = Assert.Single(spy.Messages.OfType <_DiagnosticMessage>());
Assert.StartsWith($"Assembly-level test collection orderer '{typeof(CtorThrowingCollectionOrderer).FullName}' threw 'System.DivideByZeroException' during construction: Attempted to divide by zero.", diagnosticMessage.Message);
}
public static async void TestCaseOrdererWhichThrowsLogsMessageAndDoesNotReorderTests()
{
var spy = SpyMessageSink.Capture();
TestContext.Current !.DiagnosticMessageSink = spy;
var passing1 = Mocks.TestCase <ClassUnderTest>("Passing");
var passing2 = Mocks.TestCase <ClassUnderTest>("Passing");
var other1 = Mocks.TestCase <ClassUnderTest>("Other");
var other2 = Mocks.TestCase <ClassUnderTest>("Other");
var runner = TestableTestClassRunner.Create(testCases: new[] { passing1, other1, passing2, other2 }, orderer: new ThrowingOrderer());
await runner.RunAsync();
Assert.Collection(
runner.MethodsRun,
tuple =>
{
Assert.Equal("Passing", tuple.Item1?.Name);
Assert.Collection(tuple.Item2,
testCase => Assert.Same(passing1, testCase),
testCase => Assert.Same(passing2, testCase)
);
},
tuple =>
{
Assert.Equal("Other", tuple.Item1?.Name);
Assert.Collection(tuple.Item2,
testCase => Assert.Same(other1, testCase),
testCase => Assert.Same(other2, testCase)
);
}
);
var diagnosticMessage = Assert.Single(spy.Messages.Cast <_DiagnosticMessage>());
Assert.StartsWith("Test case orderer 'TestClassRunnerTests+TestCaseOrderer+ThrowingOrderer' threw 'System.DivideByZeroException' during ordering: Attempted to divide by zero.", diagnosticMessage.Message);
}
