public static async void Cancellation_TestAssemblyStarting_DoesNotCallExtensibilityCallbacks() { var messageSink = SpyMessageSink.Create(msg => !(msg is ITestAssemblyStarting)); var runner = TestableTestAssemblyRunner.Create(messageSink); await runner.RunAsync(); Assert.False(runner.AfterTestAssemblyStarting_Called); Assert.False(runner.BeforeTestAssemblyFinished_Called); }
public static async ValueTask Cancellation_TestAssemblyFinished_CallsCallExtensibilityCallbacks() { var messageSink = SpyMessageSink.Create(msg => !(msg is _TestAssemblyFinished)); var runner = TestableTestAssemblyRunner.Create(messageSink); await runner.RunAsync(); Assert.True(runner.AfterTestAssemblyStarting_Called); Assert.True(runner.BeforeTestAssemblyFinished_Called); }
public static void SyncMessageBusOption() { var runner = TestableTestAssemblyRunner.Create(options: new XunitExecutionOptions { SynchronousMessageReporting = true }); var messageBus = runner.CreateMessageBus_Public(); Assert.IsType <SynchronousMessageBus>(messageBus); }
public static void SyncMessageBusOption() { var executionOptions = TestFrameworkOptions.ForExecution(); executionOptions.SetSynchronousMessageReporting(true); var runner = TestableTestAssemblyRunner.Create(executionOptions: executionOptions); var messageBus = runner.CreateMessageBus_Public(); Assert.IsType <SynchronousMessageBus>(messageBus); }
public static async void SignalingCancellationStopsRunningCollections() { var collection1 = Mocks.TestCollection(); var testCase1 = Mocks.TestCase(collection1); var collection2 = Mocks.TestCollection(); var testCase2 = Mocks.TestCase(collection2); var runner = TestableTestAssemblyRunner.Create(testCases: new[] { testCase1, testCase2 }, cancelInRunTestCollectionAsync: true); await runner.RunAsync(); Assert.Single(runner.CollectionsRun); }
public static async void FailureInAfterTestAssemblyStarting_GivesErroredAggregatorToTestCollectionRunner_NoCleanupFailureMessage() { var messages = new List <IMessageSinkMessage>(); var messageSink = SpyMessageSink.Create(messages: messages); var runner = TestableTestAssemblyRunner.Create(messageSink); var ex = new DivideByZeroException(); runner.AfterTestAssemblyStarting_Callback = aggregator => aggregator.Add(ex); await runner.RunAsync(); Assert.Same(ex, runner.RunTestCollectionAsync_AggregatorResult); Assert.Empty(messages.OfType <ITestAssemblyCleanupFailure>()); }
public static async ValueTask Messages() { var summary = new RunSummary { Total = 4, Failed = 2, Skipped = 1, Time = 21.12m }; var messages = new List <_MessageSinkMessage>(); var messageSink = SpyMessageSink.Create(messages: messages); await using var runner = TestableTestAssemblyRunner.Create(messageSink, summary); var thisAssembly = Assembly.GetExecutingAssembly(); var result = await runner.RunAsync(); Assert.Equal(4, result.Total); Assert.Equal(2, result.Failed); Assert.Equal(1, result.Skipped); Assert.NotEqual(21.12m, result.Time); // Uses clock time, not result time Assert.Collection( messages, msg => { var starting = Assert.IsAssignableFrom <_TestAssemblyStarting>(msg); #if NETFRAMEWORK Assert.Equal(thisAssembly.GetLocalCodeBase(), starting.AssemblyPath); Assert.Equal(runner.TestAssembly.ConfigFileName, starting.ConfigFilePath); Assert.Equal(".NETFramework,Version=v4.7.2", starting.TargetFramework); #else Assert.Equal(".NETCoreApp,Version=v2.1", starting.TargetFramework); #endif Assert.InRange(starting.StartTime, DateTime.Now.AddMinutes(-15), DateTime.Now); Assert.Equal("The test framework environment", starting.TestEnvironment); Assert.Equal("The test framework display name", starting.TestFrameworkDisplayName); Assert.Equal("assembly-id", starting.AssemblyUniqueID); }, msg => { var finished = Assert.IsAssignableFrom <_TestAssemblyFinished>(msg); Assert.Equal("assembly-id", finished.AssemblyUniqueID); Assert.Equal(result.Time, finished.ExecutionTime); Assert.Equal(2, finished.TestsFailed); Assert.Equal(4, finished.TestsRun); Assert.Equal(1, finished.TestsSkipped); } ); }
public static async void Messages() { var summary = new RunSummary { Total = 4, Failed = 2, Skipped = 1, Time = 21.12m }; var messages = new List <IMessageSinkMessage>(); var messageSink = SpyMessageSink.Create(messages: messages); var runner = TestableTestAssemblyRunner.Create(messageSink, summary); var thisAssembly = Assembly.GetExecutingAssembly(); var thisAppDomain = AppDomain.CurrentDomain; var result = await runner.RunAsync(); Assert.Equal(4, result.Total); Assert.Equal(2, result.Failed); Assert.Equal(1, result.Skipped); Assert.NotEqual(21.12m, result.Time); // Uses clock time, not result time Assert.Collection(messages, msg => { var starting = Assert.IsAssignableFrom <ITestAssemblyStarting>(msg); #if NETFRAMEWORK Assert.Equal(thisAssembly.GetLocalCodeBase(), starting.TestAssembly.Assembly.AssemblyPath); Assert.Equal(thisAppDomain.SetupInformation.ConfigurationFile, starting.TestAssembly.ConfigFileName); #endif Assert.InRange(starting.StartTime, DateTime.Now.AddMinutes(-15), DateTime.Now); Assert.Equal("The test framework environment", starting.TestEnvironment); Assert.Equal("The test framework display name", starting.TestFrameworkDisplayName); }, msg => { var finished = Assert.IsAssignableFrom <ITestAssemblyFinished>(msg); Assert.Equal(4, finished.TestsRun); Assert.Equal(2, finished.TestsFailed); Assert.Equal(1, finished.TestsSkipped); Assert.Equal(result.Time, finished.ExecutionTime); } ); }
public static async void FailureInBeforeTestAssemblyFinished_ReportsCleanupFailure_DoesNotIncludeExceptionsFromAfterTestAssemblyStarting() { var thisAssembly = Assembly.GetExecutingAssembly(); var thisAppDomain = AppDomain.CurrentDomain; var messages = new List <IMessageSinkMessage>(); var messageSink = SpyMessageSink.Create(messages: messages); var testCases = new[] { Mocks.TestCase() }; var runner = TestableTestAssemblyRunner.Create(messageSink, testCases: testCases); var startingException = new DivideByZeroException(); var finishedException = new InvalidOperationException(); runner.AfterTestAssemblyStarting_Callback = aggregator => aggregator.Add(startingException); runner.BeforeTestAssemblyFinished_Callback = aggregator => aggregator.Add(finishedException); await runner.RunAsync(); var cleanupFailure = Assert.Single(messages.OfType <ITestAssemblyCleanupFailure>()); Assert.Equal(thisAssembly.GetLocalCodeBase(), cleanupFailure.TestAssembly.Assembly.AssemblyPath); Assert.Equal(thisAppDomain.SetupInformation.ConfigurationFile, cleanupFailure.TestAssembly.ConfigFileName); Assert.Equal(testCases, cleanupFailure.TestCases); Assert.Equal(typeof(InvalidOperationException).FullName, cleanupFailure.ExceptionTypes.Single()); }
public static async ValueTask OrdererUsedToOrderTestCases() { var collection1 = Mocks.TestCollection(displayName: "AAA", uniqueID: "collection-1"); var testCase1a = TestCaseForTestCollection(collection1); var testCase1b = TestCaseForTestCollection(collection1); var collection2 = Mocks.TestCollection(displayName: "ZZZZ", uniqueID: "collection-2"); var testCase2a = TestCaseForTestCollection(collection2); var testCase2b = TestCaseForTestCollection(collection2); var collection3 = Mocks.TestCollection(displayName: "MM", uniqueID: "collection-3"); var testCase3a = TestCaseForTestCollection(collection3); var testCase3b = TestCaseForTestCollection(collection3); var testCases = new[] { testCase1a, testCase3a, testCase2a, testCase3b, testCase2b, testCase1b }; await using var runner = TestableTestAssemblyRunner.Create(testCases: testCases); runner.TestCollectionOrderer = new MyTestCollectionOrderer(); await runner.RunAsync(); Assert.Collection( runner.CollectionsRun, collection => { Assert.Same(collection2, collection.Item1); Assert.Equal(new[] { testCase2a, testCase2b }, collection.Item2); }, collection => { Assert.Same(collection3, collection.Item1); Assert.Equal(new[] { testCase3a, testCase3b }, collection.Item2); }, collection => { Assert.Same(collection1, collection.Item1); Assert.Equal(new[] { testCase1a, testCase1b }, collection.Item2); } ); }
public static async void TestCaseOrdererWhichThrowsLogsMessageAndDoesNotReorderTests() { var collection1 = Mocks.TestCollection(displayName: "AAA"); var testCase1 = Mocks.TestCase(collection1); var collection2 = Mocks.TestCollection(displayName: "ZZZZ"); var testCase2 = Mocks.TestCase(collection2); var collection3 = Mocks.TestCollection(displayName: "MM"); var testCase3 = Mocks.TestCase(collection3); var testCases = new[] { testCase1, testCase2, testCase3 }; var runner = TestableTestAssemblyRunner.Create(testCases: testCases); runner.TestCollectionOrderer = new ThrowingOrderer(); await runner.RunAsync(); Assert.Collection(runner.CollectionsRun, collection => Assert.Same(collection1, collection.Item1), collection => Assert.Same(collection2, collection.Item1), collection => Assert.Same(collection3, collection.Item1) ); var diagnosticMessage = Assert.Single(runner.DiagnosticMessages.Cast <IDiagnosticMessage>()); Assert.StartsWith("Test collection orderer 'TestAssemblyRunnerTests+TestCollectionOrderer+ThrowingOrderer' threw 'System.DivideByZeroException' during ordering: Attempted to divide by zero.", diagnosticMessage.Message); }
public static async ValueTask FailureInBeforeTestAssemblyFinished_ReportsCleanupFailure_DoesNotIncludeExceptionsFromAfterTestAssemblyStarting() { var thisAssembly = Assembly.GetExecutingAssembly(); var messages = new List <_MessageSinkMessage>(); var messageSink = SpyMessageSink.Create(messages: messages); var testCases = new[] { TestCaseForTestCollection() }; var runner = TestableTestAssemblyRunner.Create(messageSink, testCases: testCases); var startingException = new DivideByZeroException(); var finishedException = new InvalidOperationException(); runner.AfterTestAssemblyStarting_Callback = aggregator => aggregator.Add(startingException); runner.BeforeTestAssemblyFinished_Callback = aggregator => aggregator.Add(finishedException); await runner.RunAsync(); var assemblyStarting = Assert.Single(messages.OfType <_TestAssemblyStarting>()); var cleanupFailure = Assert.Single(messages.OfType <_TestAssemblyCleanupFailure>()); #if NETFRAMEWORK Assert.Equal(thisAssembly.GetLocalCodeBase(), assemblyStarting.AssemblyPath); Assert.Equal(runner.TestAssembly.ConfigFileName, assemblyStarting.ConfigFilePath); #endif Assert.Equal(typeof(InvalidOperationException).FullName, cleanupFailure.ExceptionTypes.Single()); }
public static void DefaultTestCaseOrderer() { var runner = TestableTestAssemblyRunner.Create(); Assert.IsType <DefaultTestCollectionOrderer>(runner.TestCollectionOrderer); }
public static async ValueTask DefaultTestCaseOrderer() { await using var runner = TestableTestAssemblyRunner.Create(); Assert.IsType <DefaultTestCollectionOrderer>(runner.TestCollectionOrderer); }